Merge pull request #3150 from alphagov/refresh-dev-tasks

Simplify README and consolidate scripts
2025-12-24 09:21:39 -05:00 · 2021-02-22 13:20:49 +00:00
parent 965a2ff0cb f2c6a49ab6
commit 71bf89bf0c
8 changed files with 53 additions and 191 deletions
--- a/73
+++ b/73
@@ -7,15 +7,6 @@ APP_VERSION_FILE = app/version.py
 GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached")
 GIT_COMMIT ?= $(shell git rev-parse HEAD)

-DOCKER_BUILDER_IMAGE_NAME = govuk/notify-api-builder:master
-
-BUILD_TAG ?= notifications-api-manual
-BUILD_NUMBER ?= 0
-DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER}
-BUILD_URL ?=
-
-DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG}
-
 CF_API ?= api.cloud.service.gov.uk
 CF_ORG ?= govuk-notify
 CF_SPACE ?= ${DEPLOY_ENV}
@@ -30,6 +21,30 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials

 ## DEVELOPMENT

+.PHONY: bootstrap
+bootstrap: generate-version-file
+	pip3 install -r requirements_for_test.txt
+	createdb notification_api || true
+	. environment.sh && flask db upgrade
+
+.PHONY: run-flask
+run-flask:
+	. environment.sh && flask run -p 6011
+
+.PHONY: run-celery
+run-celery:
+	. environment.sh && celery \
+		-A run_celery.notify_celery worker \
+		--pidfile="/tmp/celery.pid" \
+		--loglevel=INFO \
+		--concurrency=4
+
+.PHONY: run-celery-beat
+run-celery-beat:
+	. environment.sh && celery \
+		-A run_celery.notify_celery beat \
+		--loglevel=INFO
+
 .PHONY: help
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -61,46 +76,6 @@ test-requirements:
 	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
 || { echo "requirements.txt is up to date"; exit 0; }

-.PHONY: prepare-docker-build-image
-prepare-docker-build-image: generate-version-file ## Prepare the Docker builder image
-	docker build -f docker/Dockerfile \
-		--build-arg HTTP_PROXY="${HTTP_PROXY}" \
-		--build-arg HTTPS_PROXY="${HTTP_PROXY}" \
-		--build-arg NO_PROXY="${NO_PROXY}" \
-		-t ${DOCKER_BUILDER_IMAGE_NAME} \
-		.
-
-.PHONY: test-with-docker
-test-with-docker: prepare-docker-build-image create-docker-test-db ## Run tests inside a Docker container
-	@docker run -it --rm \
-		--name "${DOCKER_CONTAINER_PREFIX}-test" \
-		--link "${DOCKER_CONTAINER_PREFIX}-db:postgres" \
-		-e SQLALCHEMY_DATABASE_URI=postgresql://postgres:postgres@postgres/test_notification_api \
-		-e GIT_COMMIT=${GIT_COMMIT} \
-		-e BUILD_NUMBER=${BUILD_NUMBER} \
-		-e BUILD_URL=${BUILD_URL} \
-		-e http_proxy="${HTTP_PROXY}" \
-		-e HTTP_PROXY="${HTTP_PROXY}" \
-		-e https_proxy="${HTTPS_PROXY}" \
-		-e HTTPS_PROXY="${HTTPS_PROXY}" \
-		-e NO_PROXY="${NO_PROXY}" \
-		${DOCKER_BUILDER_IMAGE_NAME} \
-		make test
-
-.PHONY: create-docker-test-db
-create-docker-test-db: ## Start the test database in a Docker container
-	docker rm -f ${DOCKER_CONTAINER_PREFIX}-db 2> /dev/null || true
-	@docker run -d \
-		--name "${DOCKER_CONTAINER_PREFIX}-db" \
-		-e POSTGRES_PASSWORD="postgres" \
-		-e POSTGRES_DB=test_notification_api \
-		postgres:9.5
-	sleep 3
-
-.PHONY: clean-docker-containers
-clean-docker-containers: ## Clean up any remaining docker containers
-	docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true
-
 .PHONY: clean
 clean:
 	rm -rf node_modules cache target venv .coverage build tests/.cache ${CF_MANIFEST_PATH}
--- a/README.md
+++ b/README.md
@@ -13,21 +13,11 @@ At the moment we run Python 3.6 in production. You will run into problems if you

 ### AWS credentials

-To run the API you will need appropriate AWS credentials. You should receive these from whoever administrates your AWS account. Make sure you've got both an access key id and a secret access key.
-
-Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
-
-### Virtualenv
-
-```
-mkvirtualenv -p /usr/local/bin/python3 notifications-api
-```
+To run the API you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details.

 ### `environment.sh`

-Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
-
-Create a local environment.sh file containing the following:
+Creating and edit an environment.sh file.

 ```
 echo "
@@ -38,22 +28,31 @@ export FIRETEXT_API_KEY='FIRETEXT_ACTUAL_KEY'
 export NOTIFICATION_QUEUE_PREFIX='YOUR_OWN_PREFIX'

 export FLASK_APP=application.py
-export FLASK_DEBUG=1
+export FLASK_ENV=development
 export WERKZEUG_DEBUG_PIN=off
 "> environment.sh
 ```

-NOTES:
+Things to change:

- * Replace the placeholder key and prefix values as appropriate
- * The SECRET_KEY and DANGEROUS_SALT should match those in the [notifications-admin](https://github.com/alphagov/notifications-admin) app.
- * The  unique prefix for the queue names prevents clashing with others' queues in shared amazon environment and enables filtering by queue name in the SQS interface.
+* Replace `YOUR_OWN_PREFIX` with `local_dev_<first name>`.
+* Run the following in the credentials repo to get the API keys.
+
+```
+notify-pass credentials/providers/api_keys
+```

 ### Postgres

-Install [Postgres.app](http://postgresapp.com/). You will need admin on your machine to do this.
+Install [Postgres.app](http://postgresapp.com/).

-Choose the version with Additional Releases - you want 9.6. Once you run the app, open the sidebar, remove the default v11 server and create and initialise a v9.6 server.
+Currently the API works with PostgreSQL 11. After installation, open the Postgres app, open the sidebar, and update or replace the default server with a compatible version.
+
+**Note:** you may need to add the following directory to your PATH in order to bootstrap the app.
+
+```
+export PATH=${PATH}:/Applications/Postgres.app/Contents/Versions/11/bin/
+```

 ### Redis

@@ -64,41 +63,29 @@ To switch redis on you'll need to install it locally. On a OSX we've used brew f

 ##  To run the application

-First, run `scripts/bootstrap.sh` to install dependencies and create the databases.
-
-You need to run the api application and a local celery instance.
-
-There are two run scripts for running all the necessary parts.
-
-```
-scripts/run_app.sh
 ```
+# install dependencies, etc.
+make bootstrap

-```
-scripts/run_celery.sh
-```
+# run the web app
+make run-flask

-Optionally you can also run this script to run the scheduled tasks:
+# run the background tasks
+make run-celery

+# run scheduled tasks (optional)
+make run-celery-beat
 ```
-scripts/run_celery_beat.sh
-```
-

 ##  To test the application

-First, ensure that `scripts/bootstrap.sh` has been run, as it creates the test database.
-
-Then simply run
-
 ```
+# install dependencies, etc.
+make bootstrap
+
 make test
 ```

-That will run flake8 for code analysis and our unit test suite. If you wish to run our functional tests, instructions can be found in the
-[notifications-functional-tests](https://github.com/alphagov/notifications-functional-tests) repository.
-
-
 ## To update application dependencies

 `requirements.txt` file is generated from the `requirements-app.txt` in order to pin
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,39 +0,0 @@
-FROM python:3.6-slim
-
-ARG HTTP_PROXY
-ARG HTTPS_PROXY
-ARG NO_PROXY
-
-ENV PYTHONUNBUFFERED=1 \
-	DEBIAN_FRONTEND=noninteractive
-
-RUN \
-	echo "Install base packages" \
-	&& ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \
-	&& apt-get update \
-	&& apt-get install -y --no-install-recommends \
-		make \
-		curl \
-		git \
-		build-essential \
-		zip \
-		libpq-dev \
-	&& echo "Clean up" \
-	&& rm -rf /var/lib/apt/lists/* /tmp/*
-
-RUN \
-	echo "Install Cloud Foundry CLI" \
-	&& curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \
-	&& dpkg -i /tmp/cloudfoundry-cli.deb
-
-# these are declared statically here so that they're cached by the docker image - if we run after the `COPY` command
-# they won't be cached so it'll re-download every time. But these don't involve the filesystem
-COPY requirements.txt .
-COPY requirements_for_test.txt .
-RUN \
-	echo "Installing python dependencies" \
-	&& pip install -r requirements_for_test.txt
-
-WORKDIR /var/project
-
-COPY . .
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@@ -1,39 +0,0 @@
-#!/bin/bash
-#
-# Bootstrap virtualenv environment and postgres databases locally.
-#
-# NOTE: This script expects to be run from the project root with
-# ./scripts/bootstrap.sh
-
-set -o pipefail
-
-function display_result {
-  RESULT=$1
-  EXIT_STATUS=$2
-  TEST=$3
-
-  if [ $RESULT -ne 0 ]; then
-    echo -e "\033[31m$TEST failed\033[0m"
-    exit $EXIT_STATUS
-  else
-    echo -e "\033[32m$TEST passed\033[0m"
-  fi
-}
-
-if [ ! $VIRTUAL_ENV ]; then
-  virtualenv -p python3 ./venv
-  . ./venv/bin/activate
-fi
-
-# we need the version file to exist otherwise the app will blow up
-make generate-version-file
-
-# Install Python development dependencies
-pip3 install -r requirements_for_test.txt
-
-# Create Postgres databases
-createdb notification_api
-
-# Upgrade databases
-source environment.sh
-flask db upgrade
--- a/scripts/run_app.sh
+++ b/scripts/run_app.sh
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source environment.sh
-flask run -p 6011
--- a/scripts/run_celery.sh
+++ b/scripts/run_celery.sh
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source environment.sh
-celery -A run_celery.notify_celery worker --pidfile="/tmp/celery.pid" --loglevel=INFO --concurrency=4
--- a/scripts/run_celery_beat.sh
+++ b/scripts/run_celery_beat.sh
@@ -1,6 +0,0 @@
-#!/bin/bash
-
-set -e
-
-source environment.sh
-celery -A run_celery.notify_celery beat --loglevel=INFO
--- a/scripts/run_single_test.sh
+++ b/scripts/run_single_test.sh
@@ -1,4 +0,0 @@
-#!/bin/bash
-# run a single unit test, pass in the unit test name for example: tests/app/service/test_rest.py::test_get_template_list
-source environment_test.sh
-py.test $@