diff --git a/Makefile b/Makefile index 63fd612e8..cfbac125b 100644 --- a/Makefile +++ b/Makefile @@ -7,15 +7,6 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master - -BUILD_TAG ?= notifications-api-manual -BUILD_NUMBER ?= 0 -DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER} -BUILD_URL ?= - -DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} - CF_API ?= api.cloud.service.gov.uk CF_ORG ?= govuk-notify CF_SPACE ?= ${DEPLOY_ENV} @@ -30,6 +21,30 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials ## DEVELOPMENT +.PHONY: bootstrap +bootstrap: generate-version-file + pip3 install -r requirements_for_test.txt + createdb notification_api || true + . environment.sh && flask db upgrade + +.PHONY: run-flask +run-flask: + . environment.sh && flask run -p 6011 + +.PHONY: run-celery +run-celery: + . environment.sh && celery \ + -A run_celery.notify_celery worker \ + --pidfile="/tmp/celery.pid" \ + --loglevel=INFO \ + --concurrency=4 + +.PHONY: run-celery-beat +run-celery-beat: + . environment.sh && celery \ + -A run_celery.notify_celery beat \ + --loglevel=INFO + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -61,46 +76,6 @@ test-requirements: echo "Run 'make freeze-requirements' to update."; exit 1; } \ || { echo "requirements.txt is up to date"; exit 0; } -.PHONY: prepare-docker-build-image -prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image - docker build -f docker/Dockerfile \ - --build-arg HTTP_PROXY="${HTTP_PROXY}" \ - --build-arg HTTPS_PROXY="${HTTP_PROXY}" \ - --build-arg NO_PROXY="${NO_PROXY}" \ - -t ${DOCKER_BUILDER_IMAGE_NAME} \ - . - -.PHONY: test-with-docker -test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container - @docker run -it --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-test" \ - --link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \ - -e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \ - -e GIT_COMMIT=${GIT_COMMIT} \ - -e BUILD_NUMBER=${BUILD_NUMBER} \ - -e BUILD_URL=${BUILD_URL} \ - -e http_proxy="${HTTP_PROXY}" \ - -e HTTP_PROXY="${HTTP_PROXY}" \ - -e https_proxy="${HTTPS_PROXY}" \ - -e HTTPS_PROXY="${HTTPS_PROXY}" \ - -e NO_PROXY="${NO_PROXY}" \ - ${DOCKER_BUILDER_IMAGE_NAME} \ - make test - -.PHONY: create-docker-test-db -create-docker-test-db: ## Start the test database in a Docker container - docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true - @docker run -d \ - --name "${DOCKER_CONTAINER_PREFIX}-db" \ - -e POSTGRES_PASSWORD="postgres" \ - -e POSTGRES_DB=test_notification_api \ - postgres:9.5 - sleep 3 - -.PHONY: clean-docker-containers -clean-docker-containers: ## Clean up any remaining docker containers - docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true - .PHONY: clean clean: rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH} diff --git a/README.md b/README.md index db6b1e5ce..2c7467bc5 100644 --- a/README.md +++ b/README.md @@ -13,21 +13,11 @@ At the moment we run Python 3.6 in production. You will run into problems if you ### AWS credentials -To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key. - -Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly. - -### Virtualenv - -``` -mkvirtualenv -p /usr/local/bin/python3 notifications-api -``` +To run the API you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details. ### `environment.sh` -Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account). - -Create a local environment.sh file containing the following: +Creating and edit an environment.sh file. ``` echo " @@ -38,22 +28,31 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY' export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX' export FLASK_APP=application.py -export FLASK_DEBUG=1 +export FLASK_ENV=development export WERKZEUG_DEBUG_PIN=off "> environment.sh ``` -NOTES: +Things to change: - * Replace the placeholder key and prefix values as appropriate - * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app. - * The unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface. +* Replace `YOUR_OWN_PREFIX` with `local_dev_`. +* Run the following in the credentials repo to get the API keys. + +``` +notify-pass credentials/providers/api_keys +``` ### Postgres -Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this. +Install [Postgres.app](http://postgresapp.com/). -Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server. +Currently the API works with PostgreSQL 11. After installation, open the Postgres app, open the sidebar, and update or replace the default server with a compatible version. + +**Note:** you may need to add the following directory to your PATH in order to bootstrap the app. + +``` +export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/ +``` ### Redis @@ -64,41 +63,29 @@ To switch redis on you'll need to install it locally. On a OSX we've used brew f ## To run the application -First, run `scripts/bootstrap.sh` to install dependencies and create the databases. - -You need to run the api application and a local celery instance. - -There are two run scripts for running all the necessary parts. - -``` -scripts/run_app.sh ``` +# install dependencies, etc. +make bootstrap -``` -scripts/run_celery.sh -``` +# run the web app +make run-flask -Optionally you can also run this script to run the scheduled tasks: +# run the background tasks +make run-celery +# run scheduled tasks (optional) +make run-celery-beat ``` -scripts/run_celery_beat.sh -``` - ## To test the application -First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database. - -Then simply run - ``` +# install dependencies, etc. +make bootstrap + make test ``` -That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the -[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository. - - ## To update application dependencies `requirements.txt` file is generated from the `requirements-app.txt` in order to pin diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index c4dd66460..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -FROM python:3.6-slim - -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV PYTHONUNBUFFERED=1 \ - DEBIAN_FRONTEND=noninteractive - -RUN \ - echo "Install base packages" \ - && ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - make \ - curl \ - git \ - build-essential \ - zip \ - libpq-dev \ - && echo "Clean up" \ - && rm -rf /var/lib/apt/lists/* /tmp/* - -RUN \ - echo "Install Cloud Foundry CLI" \ - && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ - && dpkg -i /tmp/cloudfoundry-cli.deb - -# these are declared statically here so that they're cached by the docker image - if we run after the `COPY` command -# they won't be cached so it'll re-download every time. But these don't involve the filesystem -COPY requirements.txt . -COPY requirements_for_test.txt . -RUN \ - echo "Installing python dependencies" \ - && pip install -r requirements_for_test.txt - -WORKDIR /var/project - -COPY . . diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh deleted file mode 100755 index c46d03aa6..000000000 --- a/scripts/bootstrap.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# -# Bootstrap virtualenv environment and postgres databases locally. -# -# NOTE: This script expects to be run from the project root with -# ./scripts/bootstrap.sh - -set -o pipefail - -function display_result { - RESULT=$1 - EXIT_STATUS=$2 - TEST=$3 - - if [ $RESULT -ne 0 ]; then - echo -e "\033[31m$TEST failed\033[0m" - exit $EXIT_STATUS - else - echo -e "\033[32m$TEST passed\033[0m" - fi -} - -if [ ! $VIRTUAL_ENV ]; then - virtualenv -p python3 ./venv - . ./venv/bin/activate -fi - -# we need the version file to exist otherwise the app will blow up -make generate-version-file - -# Install Python development dependencies -pip3 install -r requirements_for_test.txt - -# Create Postgres databases -createdb notification_api - -# Upgrade databases -source environment.sh -flask db upgrade diff --git a/scripts/run_app.sh b/scripts/run_app.sh deleted file mode 100755 index 9997d1072..000000000 --- a/scripts/run_app.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -flask run -p 6011 diff --git a/scripts/run_celery.sh b/scripts/run_celery.sh deleted file mode 100755 index 8830d2491..000000000 --- a/scripts/run_celery.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=4 diff --git a/scripts/run_celery_beat.sh b/scripts/run_celery_beat.sh deleted file mode 100755 index 1d5278e0d..000000000 --- a/scripts/run_celery_beat.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -source environment.sh -celery -A run_celery.notify_celery beat --loglevel=INFO diff --git a/scripts/run_single_test.sh b/scripts/run_single_test.sh deleted file mode 100755 index 0d1a6bfb9..000000000 --- a/scripts/run_single_test.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# run a single unit test, pass in the unit test name for example: tests/app/service/test_rest.py::test_get_template_list -source environment_test.sh -py.test $@