diff --git a/Makefile b/Makefile index da06fad25..1c7c7f485 100644 --- a/Makefile +++ b/Makefile @@ -7,16 +7,6 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD 2> /dev/null || echo "") -DOCKER_IMAGE_TAG := $(shell cat docker/VERSION) -DOCKER_BUILDER_IMAGE_NAME = govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} - -BUILD_TAG ?= notifications-admin-manual -BUILD_NUMBER ?= 0 -DEPLOY_BUILD_NUMBER ?= ${BUILD_NUMBER} -BUILD_URL ?= - -DOCKER_CONTAINER_PREFIX = ${USER}-${BUILD_TAG} - CF_API ?= api.cloud.service.gov.uk CF_ORG ?= govuk-notify CF_SPACE ?= ${DEPLOY_ENV} @@ -32,6 +22,15 @@ VIRTUALENV_ROOT := $(shell [ -z $$VIRTUAL_ENV ] && echo $$(pwd)/venv || echo $$V ## DEVELOPMENT +.PHONY: bootstrap +bootstrap: generate-version-file + pip3 install -r requirements_for_test.txt + npm install && npm rebuild node-sass npm run build + +.PHONY: run-flask +run-flask: + . environment.sh && flask run -p 6012 + .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -44,28 +43,10 @@ virtualenv: upgrade-pip: virtualenv ${VIRTUALENV_ROOT}/bin/pip install --upgrade pip -.PHONY: requirements -requirements: upgrade-pip requirements.txt ## Install dependencies for running the app - ${VIRTUALENV_ROOT}/bin/pip install -r requirements.txt - -.PHONY: requirements-for-test -requirements-for-test: upgrade-pip requirements_for_test.txt ## Install all dependencies for running the app, development and testing - ${VIRTUALENV_ROOT}/bin/pip install -r requirements_for_test.txt - -.PHONY: frontend -frontend: - npm set progress=false - npm install - npm rebuild node-sass - .PHONY: generate-version-file generate-version-file: ## Generates the app version file @echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE} -.PHONY: build -build: frontend requirements-for-test generate-version-file ## Build project - npm run build - .PHONY: test test: ## Run tests ./scripts/run_tests.sh @@ -79,50 +60,6 @@ freeze-requirements: ## create static requirements.txt ${VIRTUALENV_ROOT}/bin/pip install pip-tools ${VIRTUALENV_ROOT}/bin/pip-compile requirements.in -.PHONY: prepare-docker-build-image -prepare-docker-build-image: ## Prepare the Docker builder image - make -C docker build - -define run_docker_container - @docker run -it --rm \ - --name "${DOCKER_CONTAINER_PREFIX}-${1}" \ - -v "`pwd`:/var/project" \ - -e UID=$(shell id -u) \ - -e GID=$(shell id -g) \ - -e GIT_COMMIT=${GIT_COMMIT} \ - -e BUILD_NUMBER=${BUILD_NUMBER} \ - -e BUILD_URL=${BUILD_URL} \ - -e http_proxy="${HTTP_PROXY}" \ - -e HTTP_PROXY="${HTTP_PROXY}" \ - -e https_proxy="${HTTPS_PROXY}" \ - -e HTTPS_PROXY="${HTTPS_PROXY}" \ - -e NO_PROXY="${NO_PROXY}" \ - -e CI_NAME=${CI_NAME} \ - -e CI_BUILD_NUMBER=${BUILD_NUMBER} \ - -e CI_BUILD_URL=${BUILD_URL} \ - -e CI_BRANCH=${GIT_BRANCH} \ - -e CI_PULL_REQUEST=${CI_PULL_REQUEST} \ - -e CF_API="${CF_API}" \ - -e CF_USERNAME="${CF_USERNAME}" \ - -e CF_PASSWORD="${CF_PASSWORD}" \ - -e CF_ORG="${CF_ORG}" \ - -e CF_SPACE="${CF_SPACE}" \ - ${DOCKER_BUILDER_IMAGE_NAME} \ - ${2} -endef - -.PHONY: build-with-docker -build-with-docker: prepare-docker-build-image ## Build inside a Docker container - $(call run_docker_container,build,gosu hostuser make build) - -.PHONY: test-with-docker -test-with-docker: prepare-docker-build-image ## Run tests inside a Docker container - $(call run_docker_container,test,gosu hostuser make test) - -.PHONY: clean-docker-containers -clean-docker-containers: ## Clean up any remaining docker containers - docker rm -f $(shell docker ps -q -f "name=${DOCKER_CONTAINER_PREFIX}") 2> /dev/null || true - .PHONY: clean clean: rm -rf node_modules cache target ${CF_MANIFEST_PATH} diff --git a/README.md b/README.md index f2f06becf..90b9e7ae2 100644 --- a/README.md +++ b/README.md @@ -2,109 +2,31 @@ GOV.UK Notify admin application - https://www.notifications.service.gov.uk/ -## Features of this application - - Register and manage users - Create and manage services - Send batch emails and SMS by uploading a CSV - Show history of notifications -## First-time setup +## Setting up -### 1. Install Homebrew +### Python version -Install [Homebrew](https://brew.sh), a package manager for OSX: +At the moment we run Python 3.6 in production. -```shell -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" -``` - -### 2. Make sure you're using correct language versions - -Languages needed -- Python 3.6.x -- [Node](https://nodejs.org/) 10.15.3 or greater -- [npm](https://www.npmjs.com/) 6.4.1 or greater - -Need to install node? Run: +### NPM packages ```shell brew install node ``` -#### 2.1. `pyenv` For Python version management - -[pyenv](https://github.com/pyenv/pyenv) is a program to manage and swap between different versions of Python. To install: - -```shell -brew install pyenv -``` - -And then follow the further installation instructions in https://github.com/pyenv/pyenv#installation to configure it. - -#### 2.2. `n` For Node version management - -[NPM](npmjs.org) is Node's package management tool. `n` is a tool for managing -different versions of Node. The following installs `n` and uses the long term support (LTS) -version of Node. +[NPM](npmjs.org) is Node's package management tool. `n` is a tool for managing different versions of Node. The following installs `n` and uses the long term support (LTS) version of Node. ```shell npm install -g n n lts ``` -### 3. Install NPM dependencies - -```shell -npm install -npm rebuild node-sass -``` - -### 4. Install and use `virtualenvwrapper` (optional) - -We suggest using a virtualenv to separate the python dependencies for this project from python dependencies for other projects. - -Install [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/install.html): - -```shell -pip install virtualenvwrapper -``` - -Then follow the [virtualenvwrapper installation instructions](https://virtualenvwrapper.readthedocs.io/en/latest/install.html) docs to configure virtualenvwrapper for your terminal. - -Set up your virtualenv: - -```shell -mkvirtualenv notifications-admin -``` - -If you need to specify a certain version of python you can do this using `-p`, for example: - -```shell -mkvirtualenv -p ~/.pyenv/versions/3.6.3/bin/python notifications-admin -``` - -Activate your virtualenv: - -```shell -workon notifications-admin -``` - -### 5. Install Python dependencies - -Install dependencies and build the frontend assets: - -```shell -./scripts/bootstrap.sh -``` - -**Note:** You may need versions of both Python 3 and Python 2 accessible to build the python dependencies. `pyenv` is great for that, and making both Python versions accessible can be done like so: - -```shell -pyenv global 3.6.3 2.7.15 -``` - -### 6. Create a local `environment.sh` file +### `environment.sh` In the root directory of the application, run: @@ -112,53 +34,50 @@ In the root directory of the application, run: echo " export NOTIFY_ENVIRONMENT='development' export FLASK_APP=application.py -export FLASK_DEBUG=1 +export FLASK_ENV=development export WERKZEUG_DEBUG_PIN=off "> environment.sh ``` -### 7. AWS credentials +### AWS credentials -Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly +To run parts of the app, such as uploading letters, you will need appropriate AWS credentials. See the [Wiki](https://github.com/alphagov/notifications-manuals/wiki/aws-accounts#how-to-set-up-local-development) for more details. - -### 8. Running the application - -In the root directory of the application, run: +## To run the application ```shell -./scripts/run_app.sh +# install dependencies, etc. +make bootstrap + +# run the web app +make run-flask ``` -Then visit [localhost:6012](http://localhost:6012) +Then visit [localhost:6012](http://localhost:6012). -## Updating application dependencies - -`requirements.txt` file is generated from the `requirements-app.txt` in order to pin -versions of all nested dependencies. If `requirements-app.txt` has been changed (or -we want to update the unpinned nested dependencies) `requirements.txt` should be -regenerated with +## To test the application ``` -make freeze-requirements +# install dependencies, etc. +make bootstrap + +make test ``` -`requirements.txt` should be committed alongside `requirements-app.txt` changes. +## Common tasks +### Updating application dependencies -## Automatically rebuild the frontend assets +`requirements.txt` is generated from the `requirements.in` in order to pin versions of all nested dependencies. If `requirements.in` has been changed, run `make freeze-requirements` to regenerate it. -If you want the front end assets to re-compile on changes, leave this running -in a separate terminal from the app +### Automatically rebuild the frontend assets + +If you want the front end assets to re-compile on changes, leave this running in a separate terminal from the app ```shell npm run watch ``` -## Working with static assets +## Further docs -When running locally static assets are served by Flask at http://localhost:6012/static/… - -When running on preview, staging and production there’s a bit more to it: - -![notify-static-after](https://user-images.githubusercontent.com/355079/50343595-6ea5de80-051f-11e9-85cf-2c20eb3cdefa.png) +- [Working with static assets](docs/static-assets.md) diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index c9464a30b..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,80 +0,0 @@ -FROM python:3.5-slim-jessie - -ARG HTTP_PROXY -ARG HTTPS_PROXY -ARG NO_PROXY - -ENV PYTHONUNBUFFERED=1 \ - DEBIAN_FRONTEND=noninteractive \ - NODEJS_VERSION=10.x \ - GOSU_VERSION=1.10 - -RUN \ - echo "Install base packages" \ - && ([ -z "$HTTP_PROXY" ] || echo "Acquire::http::Proxy \"${HTTP_PROXY}\";" > /etc/apt/apt.conf.d/99HttpProxy) \ - && apt-get update \ - && apt-get install -y --no-install-recommends \ - apt-transport-https \ - make \ - curl \ - git \ - build-essential \ - libxml2-dev \ - libxslt-dev \ - zlib1g-dev \ - zip \ - rlwrap \ - python-dev \ - libffi-dev \ - libssl-dev \ - libexif-dev \ - libfreetype6-dev \ - libjpeg-dev \ - liblcms2-2 \ - libtiff5-dev \ - zlib1g-dev \ - libpango1.0-dev \ - libcairo2-dev \ - libmagickwand-dev \ - ghostscript \ - jq \ - libjpeg62 \ - - && echo "Install nodejs" \ - && cd /tmp \ - && curl -x "$HTTP_PROXY" -sL https://deb.nodesource.com/setup_${NODEJS_VERSION} | bash - \ - && apt-get install -y --no-install-recommends nodejs \ - - && echo "Clean up" \ - && rm -rf /var/lib/apt/lists/* /tmp/* - -RUN \ - echo "Install global pip packages" \ - && pip install \ - virtualenv \ - awscli \ - wheel - -RUN \ - echo "Install Cloud Foundry CLI" \ - && curl -sSL "https://cli.run.pivotal.io/stable?release=debian64&source=github" -o /tmp/cloudfoundry-cli.deb \ - && dpkg -i /tmp/cloudfoundry-cli.deb - -COPY tianon.gpg /tmp/tianon.gpg - -RUN \ - echo "Install gosu" \ - && curl -sSL -o /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture)" \ - && curl -sSL -o /usr/local/bin/gosu.asc "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$(dpkg --print-architecture).asc" \ - && export GNUPGHOME="$(mktemp -d)" \ - && gpg --import /tmp/tianon.gpg \ - && gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu \ - && rm -r "$GNUPGHOME" /usr/local/bin/gosu.asc \ - && chmod +x /usr/local/bin/gosu \ - && gosu nobody true - -WORKDIR /var/project - -COPY entrypoint.sh /usr/local/bin/docker-entrypoint - -ENTRYPOINT ["/usr/local/bin/docker-entrypoint"] diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index 949b58b7b..000000000 --- a/docker/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -.DEFAULT_GOAL := help -SHELL := /bin/bash -DOCKER_IMAGE_TAG := $(shell cat VERSION) - -.PHONY: help -help: - @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' - -.PHONY: build -build: - docker pull `grep "FROM " Dockerfile | cut -d ' ' -f 2` || true - docker build \ - --build-arg HTTP_PROXY="${HTTP_PROXY}" \ - --build-arg HTTPS_PROXY="${HTTP_PROXY}" \ - --build-arg NO_PROXY="${NO_PROXY}" \ - -t govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ - . - -.PHONY: bash -bash: - docker run -it --rm \ - -e UID=$(shell id -u) \ - -e GID=$(shell id -g) \ - govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ - bash - -.PHONY: bash -bash-hostuser: - docker run -it --rm \ - -e UID=$(shell id -u) \ - -e GID=$(shell id -g) \ - govuk/notify-admin-builder:${DOCKER_IMAGE_TAG} \ - gosu hostuser bash diff --git a/docker/VERSION b/docker/VERSION deleted file mode 100644 index 0cfbf0888..000000000 --- a/docker/VERSION +++ /dev/null @@ -1 +0,0 @@ -2 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100755 index a57515244..000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -eo pipefail; [[ "$TRACE" ]] && set -x - -if [[ "$(id -u)" -ne 0 ]]; then - echo 'docker-entrypoint requires root' >&2 - exit 1 -fi - -if [ -z "$UID" ] || [ "$UID" = "0" ]; then - echo "UID must be specified as a positive integer" - exit 1 -fi - -if [ -z "$GID" ] || [ "$GID" = "0" ]; then - echo "GID must be specified as positive integer" - exit 1 -fi - -USER=$(id -un $UID 2>/dev/null || echo "hostuser") -GROUP=$(getent group $GID | cut -d: -f1 || echo "hostgroup") - -if [ "$USER" = "hostuser" ]; then - useradd -u $UID -s /bin/bash -m $USER -fi - -if [ "$GROUP" = "hostgroup" ]; then - groupadd -g $GID $GROUP -fi - -usermod -g $GROUP $USER - -exec "$@" diff --git a/docker/tianon.gpg b/docker/tianon.gpg deleted file mode 100644 index 415d54907..000000000 Binary files a/docker/tianon.gpg and /dev/null differ diff --git a/docs/static-assets.md b/docs/static-assets.md new file mode 100644 index 000000000..76c18b143 --- /dev/null +++ b/docs/static-assets.md @@ -0,0 +1,7 @@ +# Working with static assets + +When running locally static assets are served by Flask at http://localhost:6012/static/… + +When running on preview, staging and production there’s a bit more to it: + +![notify-static-after](https://user-images.githubusercontent.com/355079/50343595-6ea5de80-051f-11e9-85cf-2c20eb3cdefa.png) diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh deleted file mode 100755 index 28243ca2d..000000000 --- a/scripts/bootstrap.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# -# Bootstrap virtualenv environment and postgres databases locally. -# -# NOTE: This script expects to be run from the project root with -# ./scripts/bootstrap.sh - -set -o pipefail - -function display_result { - RESULT=$1 - EXIT_STATUS=$2 - TEST=$3 - - if [ $RESULT -ne 0 ]; then - echo -e "\033[31m$TEST failed\033[0m" - exit $EXIT_STATUS - else - echo -e "\033[32m$TEST passed\033[0m" - fi -} - -if [ ! $VIRTUAL_ENV ]; then - virtualenv -p python3 ./venv - . ./venv/bin/activate -fi - -# we need the version file to exist otherwise the app will blow up -make generate-version-file - -# Install Python development dependencies -pip3 install -r requirements_for_test.txt - -npm install && npm run build diff --git a/scripts/push-tag.sh b/scripts/push-tag.sh deleted file mode 100644 index 5a74e9a05..000000000 --- a/scripts/push-tag.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash - -function pretty() { - local blue="\033[34m" - local reset="\033[0m" - while read line; do - echo -e "${blue}[publishing]${reset} ${line}" - done -} - -function warn() { - local orange="\033[33m" - local reset="\033[0m" - while read line; do - echo -e "${orange}[warning]${reset} ${line}" - done -} - -function get_sha(){ - REF=$(git log -n 1 -- VERSION --name-only) - SHA=$(echo $REF | awk '{ print $2 }') - - echo "checking out ${SHA}" | pretty - - git checkout $SHA -} - -function get_version(){ - VERSION=$(python setup.py --version) - echo "latest version is ${VERSION}" | pretty -} - -function push_tag_or_die(){ - TAG_EXISTS=$(git tag | grep -G "^${VERSION}$") - if [ "$TAG_EXISTS" ]; then - echo "Tag already exists, exiting" | warn - exit 0 - else - push_tag $VERSION - fi -} - -function push_tag(){ - git tag -a $VERSION -m "Version tag for ${VERSION}" - echo "Pushing tags to github ${VERSION} to Github" | pretty - git push origin --tags -} - -function main(){ - get_sha - get_version - push_tag_or_die -} - -main \ No newline at end of file diff --git a/scripts/run_app.sh b/scripts/run_app.sh deleted file mode 100755 index 67eb6dd17..000000000 --- a/scripts/run_app.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -source environment.sh -flask run -p 6012