From 4c0aea50aef8bd4821a64b6049a4dc51dd0f47a9 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 15:49:35 +0000 Subject: [PATCH 01/10] Remove redundant Docker tasks Nothing and no one uses these. --- Makefile | 49 ----------------------------------------------- docker/Dockerfile | 39 ------------------------------------- 2 files changed, 88 deletions(-) delete mode 100644 docker/Dockerfile diff --git a/Makefile b/Makefile index 63fd612e8..a306b6405 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,6 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master - -BUILD_TAG ?= notifications-api-manual -BUILD_NUMBER ?= 0 -DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER} -BUILD_URL ?= - -DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} - CF_API ?= api.cloud.service.gov.uk CF_ORG ?= govuk-notify CF_SPACE ?= ${DEPLOY_ENV} @@ -61,46 +52,6 @@ test-requirements: echo "Run 'make freeze-requirements' to update."; exit 1; } \ || { echo "requirements.txt is up to date"; exit 0; } -.PHONY: prepare-docker-build-image -prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image - docker build -f docker/Dockerfile \ - --build-arg HTTP_PROXY="${HTTP_PROXY}" \ - --build-arg HTTPS_PROXY="${HTTP_PROXY}" \ - --build-arg NO_PROXY="${NO_PROXY}" \ - -t ${DOCKER_BUILDER_IMAGE_NAME} \ - . - -.PHONY: test-with-docker -test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container - @docker run -it --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-test" \ - --link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \ - -e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \ - -e GIT_COMMIT=${GIT_COMMIT} \ - -e BUILD_NUMBER=${BUILD_NUMBER} \ - -e BUILD_URL=${BUILD_URL} \ - -e http_proxy="${HTTP_PROXY}" \ - -e HTTP_PROXY="${HTTP_PROXY}" \ - -e https_proxy="${HTTPS_PROXY}" \ - -e HTTPS_PROXY="${HTTPS_PROXY}" \ - -e NO_PROXY="${NO_PROXY}" \ - ${DOCKER_BUILDER_IMAGE_NAME} \ - make test - -.PHONY: create-docker-test-db -create-docker-test-db: ## Start the test database in a Docker container - docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true - @docker run -d \ - --name "${DOCKER_CONTAINER_PREFIX}-db" \ - -e POSTGRES_PASSWORD="postgres" \ - -e POSTGRES_DB=test_notification_api \ - postgres:9.5 - sleep 3 - -.PHONY: clean-docker-containers -clean-docker-containers: ## Clean up any remaining docker containers - docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true - .PHONY: clean clean: rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH} diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index c4dd66460..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM python:3.6-slim - -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV PYTHONUNBUFFERED=1 \ - DEBIAN_FRONTEND=noninteractive - -RUN \ - echo "Install base packages" \ - && ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - make \ - curl \ - git \ - build-essential \ - zip \ - libpq-dev \ - && echo "Clean up" \ - && rm -rf /var/lib/apt/lists/* /tmp/* - -RUN \ - echo "Install Cloud Foundry CLI" \ - && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ - && dpkg -i /tmp/cloudfoundry-cli.deb - -# these are declared statically here so that they're cached by the docker image - if we run after the `COPY` command -# they won't be cached so it'll re-download every time. But these don't involve the filesystem -COPY requirements.txt . -COPY requirements_for_test.txt . -RUN \ - echo "Installing python dependencies" \ - && pip install -r requirements_for_test.txt - -WORKDIR /var/project - -COPY . . From ba4d399982c4976199f83d0ebae57457dbf38851 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 16:49:05 +0000 Subject: [PATCH 02/10] Switch to 'make' for running app processes These are simple enough that they don't need their own scripts. --- Makefile | 18 ++++++++++++++++++ README.md | 7 +++---- scripts/run_app.sh | 6 ------ scripts/run_celery.sh | 6 ------ scripts/run_celery_beat.sh | 6 ------ 5 files changed, 21 insertions(+), 22 deletions(-) delete mode 100755 scripts/run_app.sh delete mode 100755 scripts/run_celery.sh delete mode 100755 scripts/run_celery_beat.sh diff --git a/Makefile b/Makefile index a306b6405..f70dd4a3b 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,24 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials ## DEVELOPMENT +.PHONY: run-flask +run-flask: + . environment.sh && flask run -p 6011 + +.PHONY: run-celery +run-celery: + . environment.sh && celery \ + -A run_celery.notify_celery worker \ + --pidfile="/tmp/celery.pid" \ + --loglevel=INFO \ + --concurrency=4 + +.PHONY: run-celery-beat +run-celery-beat: + . environment.sh && celery \ + -A run_celery.notify_celery beat \ + --loglevel=INFO + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/README.md b/README.md index db6b1e5ce..ff3b1cc86 100644 --- a/README.md +++ b/README.md @@ -71,20 +71,19 @@ You need to run the api application and a local celery instance. There are two run scripts for running all the necessary parts. ``` -scripts/run_app.sh +make run-flask ``` ``` -scripts/run_celery.sh +make run-celery ``` Optionally you can also run this script to run the scheduled tasks: ``` -scripts/run_celery_beat.sh +make run-celery-beat ``` - ## To test the application First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database. diff --git a/scripts/run_app.sh b/scripts/run_app.sh deleted file mode 100755 index 9997d1072..000000000 --- a/scripts/run_app.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -flask run -p 6011 diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh deleted file mode 100755 index 8830d2491..000000000 --- a/scripts/run_celery.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=4 diff --git a/scripts/run_celery_beat.sh b/scripts/run_celery_beat.sh deleted file mode 100755 index 1d5278e0d..000000000 --- a/scripts/run_celery_beat.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -celery -A run_celery.notify_celery beat --loglevel=INFO From 89dc7218feb16b54bb46a5618d4e93a907e8472c Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 16:50:30 +0000 Subject: [PATCH 03/10] Remove section about virtualenv This is covered generically in the Wiki [1]. [1]: https://github.com/alphagov/notifications-manuals/wiki/Getting-Started#development-environment --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index ff3b1cc86..33f3892f9 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,6 @@ To run the API you will need appropriate AWS credentials. You should receive the Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly. -### Virtualenv - -``` -mkvirtualenv -p /usr/local/bin/python3 notifications-api -``` - ### `environment.sh` Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account). From de6ed47239f92f2d431e77de7bcf1d22ed67be6a Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 16:56:20 +0000 Subject: [PATCH 04/10] Revise section about AWS setup This is covered in Wiki in more detail. --- README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33f3892f9..7ebd611b8 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,11 @@ At the moment we run Python 3.6 in production. You will run into problems if you ### AWS credentials -To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key. - -Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly. +To run the API you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details. ### `environment.sh` -Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account). +Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Create a local environment.sh file containing the following: From 273fd339aceb9a9cba92bd2a7c835479adcf9c6b Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 17:04:28 +0000 Subject: [PATCH 05/10] Clarify setup instructions for environment.sh --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7ebd611b8..0d9360081 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ To run the API you will need appropriate AWS credentials. See the [Wiki](https:/ ### `environment.sh` -Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. - -Create a local environment.sh file containing the following: +Creating and edit an environment.sh file. ``` echo " @@ -35,11 +33,14 @@ export WERKZEUG_DEBUG_PIN=off "> environment.sh ``` -NOTES: +Things to change: - * Replace the placeholder key and prefix values as appropriate - * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app. - * The unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface. +* Replace `YOUR_OWN_PREFIX` with `local_dev_`. +* Run the following in the credentials repo to get the API keys. + +``` +notify-pass credentials/providers/api_keys +``` ### Postgres From 5ad821428b89bf2a12b8c61f7ea04d888e777c2a Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 17:06:58 +0000 Subject: [PATCH 06/10] Make instructions to run app/tests more concise --- README.md | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 0d9360081..d613d33cb 100644 --- a/README.md +++ b/README.md @@ -57,40 +57,29 @@ To switch redis on you'll need to install it locally. On a OSX we've used brew f ## To run the application -First, run `scripts/bootstrap.sh` to install dependencies and create the databases. - -You need to run the api application and a local celery instance. - -There are two run scripts for running all the necessary parts. - ``` +# install dependencies, etc. +./scripts/bootstrap.sh + +# run the web app make run-flask -``` -``` +# run the background tasks make run-celery -``` -Optionally you can also run this script to run the scheduled tasks: - -``` +# run scheduled tasks (optional) make run-celery-beat ``` ## To test the application -First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database. - -Then simply run - ``` +# install dependencies, etc. +./scripts/bootstrap.sh + make test ``` -That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the -[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository. - - ## To update application dependencies `requirements.txt` file is generated from the `requirements-app.txt` in order to pin From dc6fb1d1f2d40df0bb028ce741e31a36884668e1 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 17:15:22 +0000 Subject: [PATCH 07/10] Remove unused single test script --- scripts/run_single_test.sh | 4 ---- 1 file changed, 4 deletions(-) delete mode 100755 scripts/run_single_test.sh diff --git a/scripts/run_single_test.sh b/scripts/run_single_test.sh deleted file mode 100755 index 0d1a6bfb9..000000000 --- a/scripts/run_single_test.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# run a single unit test, pass in the unit test name for example: tests/app/service/test_rest.py::test_get_template_list -source environment_test.sh -py.test $@ From af95ad68eace2f808fb0809dc761330fbae69397 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Wed, 17 Feb 2021 17:22:45 +0000 Subject: [PATCH 08/10] Move bootstrap tasks into the Makefile This is more consistent with how we run all other tasks. Note that the virtual env setup is not generally applicable, and developers of this repo should follow the guidance in the README. --- Makefile | 6 ++++++ README.md | 4 ++-- scripts/bootstrap.sh | 39 --------------------------------------- 3 files changed, 8 insertions(+), 41 deletions(-) delete mode 100755 scripts/bootstrap.sh diff --git a/Makefile b/Makefile index f70dd4a3b..372ed2c3f 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,12 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials ## DEVELOPMENT +.PHONY: bootstrap +bootstrap: generate-version-file + pip3 install -r requirements_for_test.txt + createdb notification_api + . environment.sh && flask db upgrade + .PHONY: run-flask run-flask: . environment.sh && flask run -p 6011 diff --git a/README.md b/README.md index d613d33cb..8109018cb 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ To switch redis on you'll need to install it locally. On a OSX we've used brew f ``` # install dependencies, etc. -./scripts/bootstrap.sh +make bootstrap # run the web app make run-flask @@ -75,7 +75,7 @@ make run-celery-beat ``` # install dependencies, etc. -./scripts/bootstrap.sh +make bootstrap make test ``` diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh deleted file mode 100755 index c46d03aa6..000000000 --- a/scripts/bootstrap.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# -# Bootstrap virtualenv environment and postgres databases locally. -# -# NOTE: This script expects to be run from the project root with -# ./scripts/bootstrap.sh - -set -o pipefail - -function display_result { - RESULT=$1 - EXIT_STATUS=$2 - TEST=$3 - - if [ $RESULT -ne 0 ]; then - echo -e "\033[31m$TEST failed\033[0m" - exit $EXIT_STATUS - else - echo -e "\033[32m$TEST passed\033[0m" - fi -} - -if [ ! $VIRTUAL_ENV ]; then - virtualenv -p python3 ./venv - . ./venv/bin/activate -fi - -# we need the version file to exist otherwise the app will blow up -make generate-version-file - -# Install Python development dependencies -pip3 install -r requirements_for_test.txt - -# Create Postgres databases -createdb notification_api - -# Upgrade databases -source environment.sh -flask db upgrade From 5ff7c2013e544edb4feced9133a39bb2175f385b Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Thu, 18 Feb 2021 09:10:13 +0000 Subject: [PATCH 09/10] Swap FLASK_DEBUG for FLASK_ENV This achieves the same thing and gets rid of the warning about being in a production environment when the app starts up. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8109018cb..c3f17a99a 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY' export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX' export FLASK_APP=application.py -export FLASK_DEBUG=1 +export FLASK_ENV=development export WERKZEUG_DEBUG_PIN=off "> environment.sh ``` From f2c6a49ab64cedccebe66f9cf96de6696fa6f51a Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Thu, 18 Feb 2021 15:19:40 +0000 Subject: [PATCH 10/10] Update install instructions for Postgres This apps works with v11, but not with v13. Adding '|| true' in the Makefile means the 'bootstrap' rule can be run multiple times, even if the DB already exists. --- Makefile | 2 +- README.md | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 372ed2c3f..cfbac125b 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials .PHONY: bootstrap bootstrap: generate-version-file pip3 install -r requirements_for_test.txt - createdb notification_api + createdb notification_api || true . environment.sh && flask db upgrade .PHONY: run-flask diff --git a/README.md b/README.md index c3f17a99a..2c7467bc5 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,15 @@ notify-pass credentials/providers/api_keys ### Postgres -Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this. +Install [Postgres.app](http://postgresapp.com/). -Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server. +Currently the API works with PostgreSQL 11. After installation, open the Postgres app, open the sidebar, and update or replace the default server with a compatible version. + +**Note:** you may need to add the following directory to your PATH in order to bootstrap the app. + +``` +export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ +``` ### Redis