From 6786b892fde1240561905c13cf6984b951f16b91 Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:07:22 +0000
Subject: [PATCH 1/7] Copy requirements files to .in
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We’re going to move to using pip-tools for freezing requirements.

pip-tools uses `.in` files for the un-frozen list of requirements, and
then generates `.txt` equivalents.

This commit just copies our existing `.txt` files, keeping the same name
but giving them a `.in` extension ready for pip-tools to use.
---
 requirements.in          | 32 ++++++++++++++++++++++++++++++++
 requirements_for_test.in | 15 +++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 requirements.in
 create mode 100644 requirements_for_test.in

diff --git a/requirements.in b/requirements.in
new file mode 100644
index 000000000..36413d59c
--- /dev/null
+++ b/requirements.in
@@ -0,0 +1,32 @@
+# Run `make freeze-requirements` to update requirements.txt
+# with package version changes made in requirements-app.txt
+
+ago==0.0.93
+govuk-bank-holidays==0.8
+humanize==3.2.0
+Flask==1.1.2
+Flask-WTF==0.14.3
+Flask-Login==0.5.0
+
+blinker==1.4
+pyexcel==0.6.6
+pyexcel-io==0.6.4
+pyexcel-xls==0.6.2
+pyexcel-xlsx==0.6.0
+pyexcel-ods3==0.6.0
+pytz==2020.5
+gunicorn==20.0.4
+eventlet==0.30.0
+notifications-python-client==5.7.1
+Shapely==1.7.1
+
+# PaaS
+awscli-cwlogs>=1.4,<1.5
+itsdangerous==1.1.0
+
+git+https://github.com/alphagov/notifications-utils.git@43.7.0#egg=notifications-utils==43.7.0
+git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha
+
+# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains
+prometheus-client==0.9.0
+gds-metrics==0.2.4
diff --git a/requirements_for_test.in b/requirements_for_test.in
new file mode 100644
index 000000000..afa944e1c
--- /dev/null
+++ b/requirements_for_test.in
@@ -0,0 +1,15 @@
+-r requirements.txt
+pip-tools==5.4.0
+isort==4.3.21
+pytest==5.3.2
+pytest-env==0.6.2
+pytest-mock==1.11.2
+pytest-xdist==1.31.0
+beautifulsoup4==4.8.1
+freezegun==0.3.12
+flake8==3.7.9
+flake8-bugbear==19.8.0
+flake8-print==3.1.4
+requests-mock==1.7.0
+# used for creating manifest file locally
+jinja2-cli[yaml]==0.7.0

From 0f68d252659d597c17e1cf9eb4bccef9de0a2a3b Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:21:13 +0000
Subject: [PATCH 2/7] Replace custom `pip freeze` script with `pip-tools`

In the past we've avoided using out-of-the-box solutions for Python
dependency resolution because a) they haven't been very mature and b)
we've had lots of issues with version conflicts. See [[1]], [[2]] for
details. Instead, we've been using a custom Python script that
under-the-hood runs `pip freeze` and saves the output to
`requirements.txt`.

This script works well for us, but it doesn't integrate well with other
tools. On the other hand [`pip-tools`](https://github.com/jazzband/pip-tools)
as of 2020 seems to be well-supported by its maintainers and other
tools; for instance, GitHub's automated update service
[Dependabot](https://dependabot.com) supports `requirements.in` files.

This commit replaces our `freeze-requirements` make command with
`pip-compile`.

The Digital Marketplace team have made this change and seem happy with
the results.
---
 Makefile             | 46 +++++++++++++++++++++++---------------------
 scripts/run_tests.sh |  3 ---
 2 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 35daf9f5c..8aee851af 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,8 @@ $(eval export CF_HOME)
 
 NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
+VIRTUALENV_ROOT := $(shell [ -z $$VIRTUAL_ENV ] && echo $$(pwd)/venv || echo $$VIRTUAL_ENV)
+
 
 ## DEVELOPMENT
 
@@ -34,21 +36,35 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: dependencies
-dependencies: ## Install build dependencies
+.PHONY: virtualenv
+virtualenv:
+	[ -z $$VIRTUAL_ENV ] && [ ! -d venv ] && python3 -m venv venv || true
+
+.PHONY: upgrade-pip
+upgrade-pip: virtualenv
+	${VIRTUALENV_ROOT}/bin/pip install --upgrade pip
+
+.PHONY: requirements
+requirements: upgrade-pip requirements.txt
+	${VIRTUALENV_ROOT}/bin/pip install -r requirements.txt
+
+.PHONY: requirements-for-test
+requirements-for-test: upgrade-pip requirements_for_test.txt
+	${VIRTUALENV_ROOT}/bin/pip install -r requirements_for_test.txt
+
+.PHONY: frontend
+frontend:
 	npm set progress=false
 	npm install
 	npm rebuild node-sass
-	pip install -r requirements_for_test.txt
 
 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: build
-build: dependencies generate-version-file ## Build project
+build: frontend requirements-for-test generate-version-file
 	npm run build
-	pip install -r requirements.txt
 
 .PHONY: test
 test: ## Run tests
@@ -59,23 +75,9 @@ fix-imports:
 	isort -rc ./app ./tests
 
 .PHONY: freeze-requirements
-freeze-requirements:
-	rm -rf venv-freeze
-	virtualenv -p python3 venv-freeze
-	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
-	echo '# pyup: ignore file' > requirements.txt
-	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
-	cat requirements-app.txt >> requirements.txt
-	echo '' >> requirements.txt
-	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
-	rm -rf venv-freeze
-
-.PHONY: test-requirements
-test-requirements:
-	@diff requirements-app.txt requirements.txt | grep '<' \
-	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
-	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
-|| { echo "requirements.txt is up to date"; exit 0; }
+freeze-requirements: requirements-for-test requirements.in requirements_for_test.in
+	${VIRTUALENV_ROOT}/bin/pip-compile requirements.in
+	${VIRTUALENV_ROOT}/bin/pip-compile requirements_for_test.in
 
 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
index c676b6052..dc3d949ee 100755
--- a/scripts/run_tests.sh
+++ b/scripts/run_tests.sh
@@ -24,9 +24,6 @@ if [[ -z "$VIRTUAL_ENV" ]] && [[ -d venv ]]; then
   source ./venv/bin/activate
 fi
 
-make test-requirements
-display_result $? 1 "Requirements check"
-
 flake8 .
 display_result $? 1 "Code style check"
 

From f97517c6b0af11f5bb64af3c7d735223815d658b Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:33:42 +0000
Subject: [PATCH 3/7] Tell PyUp to look at `.in` files

We only want PyUp to be checking for the dependencies we specify
directly, not any sub-dependencies.

By telling it to now look only at `.in` files we preserve this existing
behaviour.
---
 .pyup.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pyup.yml b/.pyup.yml
index 993ab58ec..733d5f032 100644
--- a/.pyup.yml
+++ b/.pyup.yml
@@ -4,5 +4,5 @@ schedule: "every week on monday"
 
 search: False
 requirements:
-  - requirements-app.txt
-  - requirements-dev.txt
+  - requirements.in
+  - requirements_for_test.in

From fdb659e2bd8389d1b2edf21dab887bec8e82bc1f Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:10:08 +0000
Subject: [PATCH 4/7] Run `pip-compile` command and commit output

This command freezes all our requirements into the `.txt` files.

We want these files version controlled so that our builds are exactly
reproducible from environment to environment.
---
 requirements.txt          | 170 +++++++++++++++++++-------------------
 requirements_for_test.txt | 128 ++++++++++++++++++++++++----
 2 files changed, 197 insertions(+), 101 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 3d12659b4..8a0d1f833 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,88 +1,84 @@
-# pyup: ignore file
-# This file is autogenerated. Do not edit it manually.
-# Run `make freeze-requirements` to update requirements.txt
-# with package version changes made in requirements-app.txt
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements.in
+#
+ago==0.0.93               # via -r requirements.in
+awscli-cwlogs==1.4.6      # via -r requirements.in
+awscli==1.18.211          # via awscli-cwlogs, notifications-utils
+bleach==3.2.1             # via notifications-utils
+blinker==1.4              # via -r requirements.in, gds-metrics
+boto3==1.16.51            # via notifications-utils
+botocore==1.19.51         # via awscli, boto3, s3transfer
+cachetools==4.2.0         # via notifications-utils
+certifi==2020.12.5        # via requests
+chardet==4.0.0            # via requests
+click==7.1.2              # via flask
+colorama==0.4.3           # via awscli
+dnspython==1.16.0         # via eventlet
+docopt==0.6.2             # via notifications-python-client
+docutils==0.15.2          # via awscli
+et-xmlfile==1.0.1         # via openpyxl
+eventlet==0.30.0          # via -r requirements.in
+flask-login==0.5.0        # via -r requirements.in
+flask-redis==0.4.0        # via notifications-utils
+flask-wtf==0.14.3         # via -r requirements.in
+flask==1.1.2              # via -r requirements.in, flask-login, flask-redis, flask-wtf, gds-metrics, notifications-utils
+future==0.18.2            # via notifications-python-client
+gds-metrics==0.2.4        # via -r requirements.in
+geojson==2.5.0            # via notifications-utils
+govuk-bank-holidays==0.8  # via -r requirements.in, notifications-utils
+git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha  # via -r requirements.in
+greenlet==0.4.17          # via eventlet
+gunicorn==20.0.4          # via -r requirements.in
+humanize==3.2.0           # via -r requirements.in
+idna==2.10                # via requests
+itsdangerous==1.1.0       # via -r requirements.in, flask, flask-wtf, notifications-utils
+jdcal==1.4.1              # via openpyxl
+jinja2==2.11.2            # via flask, govuk-frontend-jinja, notifications-utils
+jmespath==0.10.0          # via boto3, botocore
+lml==0.1.0                # via pyexcel, pyexcel-io
+lxml==4.6.2               # via pyexcel-ezodf, pyexcel-ods3
+markupsafe==1.1.1         # via jinja2, wtforms
+mistune==0.8.4            # via notifications-utils
+monotonic==1.5            # via notifications-python-client, notifications-utils
+notifications-python-client==5.7.1  # via -r requirements.in
+git+https://github.com/alphagov/notifications-utils.git@43.7.0#egg=notifications-utils==43.7.0  # via -r requirements.in
+openpyxl==3.0.5           # via pyexcel-xlsx
+orderedset==2.0.3         # via notifications-utils
+packaging==20.8           # via bleach
+phonenumbers==8.12.15     # via notifications-utils
+prometheus-client==0.9.0  # via -r requirements.in, gds-metrics
+pyasn1==0.4.8             # via rsa
+pyexcel-ezodf==0.3.4      # via pyexcel-ods3
+pyexcel-io==0.6.4         # via -r requirements.in, pyexcel, pyexcel-ods3, pyexcel-xls, pyexcel-xlsx
+pyexcel-ods3==0.6.0       # via -r requirements.in
+pyexcel-xls==0.6.2        # via -r requirements.in
+pyexcel-xlsx==0.6.0       # via -r requirements.in
+pyexcel==0.6.6            # via -r requirements.in
+pyjwt==2.0.0              # via notifications-python-client
+pyparsing==2.4.7          # via packaging
+pypdf2==1.26.0            # via notifications-utils
+python-dateutil==2.8.1    # via awscli-cwlogs, botocore
+python-json-logger==2.0.1  # via notifications-utils
+pytz==2020.5              # via -r requirements.in, notifications-utils
+pyyaml==5.3.1             # via awscli, notifications-utils
+redis==3.5.3              # via flask-redis
+requests==2.25.1          # via awscli-cwlogs, govuk-bank-holidays, notifications-python-client, notifications-utils
+rsa==4.5                  # via awscli
+s3transfer==0.3.3         # via awscli, boto3
+shapely==1.7.1            # via -r requirements.in
+six==1.15.0               # via awscli-cwlogs, bleach, eventlet, govuk-bank-holidays, python-dateutil
+smartypants==2.0.1        # via notifications-utils
+statsd==3.3.0             # via notifications-utils
+texttable==1.6.3          # via pyexcel
+urllib3==1.26.2           # via botocore, requests
+webencodings==0.5.1       # via bleach
+werkzeug==1.0.1           # via flask
+wtforms==2.3.3            # via flask-wtf
+xlrd==1.2.0               # via pyexcel-xls
+xlwt==1.3.0               # via pyexcel-xls
 
-ago==0.0.93
-govuk-bank-holidays==0.8
-humanize==3.2.0
-Flask==1.1.2
-Flask-WTF==0.14.3
-Flask-Login==0.5.0
-
-blinker==1.4
-pyexcel==0.6.6
-pyexcel-io==0.6.4
-pyexcel-xls==0.6.2
-pyexcel-xlsx==0.6.0
-pyexcel-ods3==0.6.0
-pytz==2020.5
-gunicorn==20.0.4
-eventlet==0.30.0
-notifications-python-client==5.7.1
-Shapely==1.7.1
-
-# PaaS
-awscli-cwlogs>=1.4,<1.5
-itsdangerous==1.1.0
-
-git+https://github.com/alphagov/notifications-utils.git@43.7.0#egg=notifications-utils==43.7.0
-git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha
-
-# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains
-prometheus-client==0.9.0
-gds-metrics==0.2.4
-
-## The following requirements were added by pip freeze:
-awscli==1.18.210
-bleach==3.2.1
-boto3==1.16.50
-botocore==1.19.50
-cachetools==4.2.0
-certifi==2020.12.5
-chardet==4.0.0
-click==7.1.2
-colorama==0.4.3
-dnspython==1.16.0
-docopt==0.6.2
-docutils==0.15.2
-et-xmlfile==1.0.1
-flask-redis==0.4.0
-future==0.18.2
-geojson==2.5.0
-greenlet==0.4.17
-idna==2.10
-jdcal==1.4.1
-Jinja2==2.11.2
-jmespath==0.10.0
-lml==0.1.0
-lxml==4.6.2
-MarkupSafe==1.1.1
-mistune==0.8.4
-monotonic==1.5
-openpyxl==3.0.5
-orderedset==2.0.3
-packaging==20.8
-phonenumbers==8.12.15
-pyasn1==0.4.8
-pyexcel-ezodf==0.3.4
-PyJWT==2.0.0
-pyparsing==2.4.7
-PyPDF2==1.26.0
-python-dateutil==2.8.1
-python-json-logger==2.0.1
-PyYAML==5.3.1
-redis==3.5.3
-requests==2.25.1
-rsa==4.5
-s3transfer==0.3.3
-six==1.15.0
-smartypants==2.0.1
-statsd==3.3.0
-texttable==1.6.3
-urllib3==1.26.2
-webencodings==0.5.1
-Werkzeug==1.0.1
-WTForms==2.3.3
-xlrd==1.2.0
-xlwt==1.3.0
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools
diff --git a/requirements_for_test.txt b/requirements_for_test.txt
index bc6a0b2b9..409161489 100644
--- a/requirements_for_test.txt
+++ b/requirements_for_test.txt
@@ -1,14 +1,114 @@
--r requirements.txt
-isort==4.3.21
-pytest==5.3.2
-pytest-env==0.6.2
-pytest-mock==1.11.2
-pytest-xdist==1.31.0
-beautifulsoup4==4.8.1
-freezegun==0.3.12
-flake8==3.7.9
-flake8-bugbear==19.8.0
-flake8-print==3.1.4
-requests-mock==1.7.0
-# used for creating manifest file locally
-jinja2-cli[yaml]==0.7.0
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile requirements_for_test.in
+#
+ago==0.0.93               # via -r requirements.txt
+apipkg==1.5               # via execnet
+attrs==20.3.0             # via flake8-bugbear, pytest
+awscli-cwlogs==1.4.6      # via -r requirements.txt
+awscli==1.18.211          # via -r requirements.txt, awscli-cwlogs, notifications-utils
+beautifulsoup4==4.8.1     # via -r requirements_for_test.in
+bleach==3.2.1             # via -r requirements.txt, notifications-utils
+blinker==1.4              # via -r requirements.txt, gds-metrics
+boto3==1.16.51            # via -r requirements.txt, notifications-utils
+botocore==1.19.51         # via -r requirements.txt, awscli, boto3, s3transfer
+cachetools==4.2.0         # via -r requirements.txt, notifications-utils
+certifi==2020.12.5        # via -r requirements.txt, requests
+chardet==4.0.0            # via -r requirements.txt, requests
+click==7.1.2              # via -r requirements.txt, flask, pip-tools
+colorama==0.4.3           # via -r requirements.txt, awscli
+dnspython==1.16.0         # via -r requirements.txt, eventlet
+docopt==0.6.2             # via -r requirements.txt, notifications-python-client
+docutils==0.15.2          # via -r requirements.txt, awscli
+entrypoints==0.3          # via flake8
+et-xmlfile==1.0.1         # via -r requirements.txt, openpyxl
+eventlet==0.30.0          # via -r requirements.txt
+execnet==1.7.1            # via pytest-xdist
+flake8-bugbear==19.8.0    # via -r requirements_for_test.in
+flake8-print==3.1.4       # via -r requirements_for_test.in
+flake8==3.7.9             # via -r requirements_for_test.in, flake8-bugbear, flake8-print
+flask-login==0.5.0        # via -r requirements.txt
+flask-redis==0.4.0        # via -r requirements.txt, notifications-utils
+flask-wtf==0.14.3         # via -r requirements.txt
+flask==1.1.2              # via -r requirements.txt, flask-login, flask-redis, flask-wtf, gds-metrics, notifications-utils
+freezegun==0.3.12         # via -r requirements_for_test.in
+future==0.18.2            # via -r requirements.txt, notifications-python-client
+gds-metrics==0.2.4        # via -r requirements.txt
+geojson==2.5.0            # via -r requirements.txt, notifications-utils
+govuk-bank-holidays==0.8  # via -r requirements.txt, notifications-utils
+git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha  # via -r requirements.txt
+greenlet==0.4.17          # via -r requirements.txt, eventlet
+gunicorn==20.0.4          # via -r requirements.txt
+humanize==3.2.0           # via -r requirements.txt
+idna==2.10                # via -r requirements.txt, requests
+importlib-metadata==3.3.0  # via pluggy, pytest
+isort==4.3.21             # via -r requirements_for_test.in
+itsdangerous==1.1.0       # via -r requirements.txt, flask, flask-wtf, notifications-utils
+jdcal==1.4.1              # via -r requirements.txt, openpyxl
+jinja2-cli[yaml]==0.7.0   # via -r requirements_for_test.in
+jinja2==2.11.2            # via -r requirements.txt, flask, govuk-frontend-jinja, jinja2-cli, notifications-utils
+jmespath==0.10.0          # via -r requirements.txt, boto3, botocore
+lml==0.1.0                # via -r requirements.txt, pyexcel, pyexcel-io
+lxml==4.6.2               # via -r requirements.txt, pyexcel-ezodf, pyexcel-ods3
+markupsafe==1.1.1         # via -r requirements.txt, jinja2, wtforms
+mccabe==0.6.1             # via flake8
+mistune==0.8.4            # via -r requirements.txt, notifications-utils
+monotonic==1.5            # via -r requirements.txt, notifications-python-client, notifications-utils
+more-itertools==8.6.0     # via pytest
+notifications-python-client==5.7.1  # via -r requirements.txt
+git+https://github.com/alphagov/notifications-utils.git@43.7.0#egg=notifications-utils==43.7.0  # via -r requirements.txt
+openpyxl==3.0.5           # via -r requirements.txt, pyexcel-xlsx
+orderedset==2.0.3         # via -r requirements.txt, notifications-utils
+packaging==20.8           # via -r requirements.txt, bleach, pytest
+phonenumbers==8.12.15     # via -r requirements.txt, notifications-utils
+pip-tools==5.4.0          # via -r requirements_for_test.in
+pluggy==0.13.1            # via pytest
+prometheus-client==0.9.0  # via -r requirements.txt, gds-metrics
+py==1.10.0                # via pytest, pytest-forked
+pyasn1==0.4.8             # via -r requirements.txt, rsa
+pycodestyle==2.5.0        # via flake8, flake8-print
+pyexcel-ezodf==0.3.4      # via -r requirements.txt, pyexcel-ods3
+pyexcel-io==0.6.4         # via -r requirements.txt, pyexcel, pyexcel-ods3, pyexcel-xls, pyexcel-xlsx
+pyexcel-ods3==0.6.0       # via -r requirements.txt
+pyexcel-xls==0.6.2        # via -r requirements.txt
+pyexcel-xlsx==0.6.0       # via -r requirements.txt
+pyexcel==0.6.6            # via -r requirements.txt
+pyflakes==2.1.1           # via flake8
+pyjwt==2.0.0              # via -r requirements.txt, notifications-python-client
+pyparsing==2.4.7          # via -r requirements.txt, packaging
+pypdf2==1.26.0            # via -r requirements.txt, notifications-utils
+pytest-env==0.6.2         # via -r requirements_for_test.in
+pytest-forked==1.3.0      # via pytest-xdist
+pytest-mock==1.11.2       # via -r requirements_for_test.in
+pytest-xdist==1.31.0      # via -r requirements_for_test.in
+pytest==5.3.2             # via -r requirements_for_test.in, pytest-env, pytest-forked, pytest-mock, pytest-xdist
+python-dateutil==2.8.1    # via -r requirements.txt, awscli-cwlogs, botocore, freezegun
+python-json-logger==2.0.1  # via -r requirements.txt, notifications-utils
+pytz==2020.5              # via -r requirements.txt, notifications-utils
+pyyaml==5.3.1             # via -r requirements.txt, awscli, jinja2-cli, notifications-utils
+redis==3.5.3              # via -r requirements.txt, flask-redis
+requests-mock==1.7.0      # via -r requirements_for_test.in
+requests==2.25.1          # via -r requirements.txt, awscli-cwlogs, govuk-bank-holidays, notifications-python-client, notifications-utils, requests-mock
+rsa==4.5                  # via -r requirements.txt, awscli
+s3transfer==0.3.3         # via -r requirements.txt, awscli, boto3
+shapely==1.7.1            # via -r requirements.txt
+six==1.15.0               # via -r requirements.txt, awscli-cwlogs, bleach, eventlet, flake8-print, freezegun, govuk-bank-holidays, pip-tools, pytest-xdist, python-dateutil, requests-mock
+smartypants==2.0.1        # via -r requirements.txt, notifications-utils
+soupsieve==2.1            # via beautifulsoup4
+statsd==3.3.0             # via -r requirements.txt, notifications-utils
+texttable==1.6.3          # via -r requirements.txt, pyexcel
+typing-extensions==3.7.4.3  # via importlib-metadata
+urllib3==1.26.2           # via -r requirements.txt, botocore, requests
+wcwidth==0.2.5            # via pytest
+webencodings==0.5.1       # via -r requirements.txt, bleach
+werkzeug==1.0.1           # via -r requirements.txt, flask
+wtforms==2.3.3            # via -r requirements.txt, flask-wtf
+xlrd==1.2.0               # via -r requirements.txt, pyexcel-xls
+xlwt==1.3.0               # via -r requirements.txt, pyexcel-xls
+zipp==3.4.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# pip
+# setuptools

From 1cfd1ca7b98fb7cd71b9691f3d99b2cae02f740c Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:08:02 +0000
Subject: [PATCH 5/7] Delete requirements-app.txt

This is the file we used to use to specify direct dependencies. We use
`requirements.in` now instead.
---
 requirements-app.txt | 32 --------------------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 requirements-app.txt

diff --git a/requirements-app.txt b/requirements-app.txt
deleted file mode 100644
index 36413d59c..000000000
--- a/requirements-app.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-# Run `make freeze-requirements` to update requirements.txt
-# with package version changes made in requirements-app.txt
-
-ago==0.0.93
-govuk-bank-holidays==0.8
-humanize==3.2.0
-Flask==1.1.2
-Flask-WTF==0.14.3
-Flask-Login==0.5.0
-
-blinker==1.4
-pyexcel==0.6.6
-pyexcel-io==0.6.4
-pyexcel-xls==0.6.2
-pyexcel-xlsx==0.6.0
-pyexcel-ods3==0.6.0
-pytz==2020.5
-gunicorn==20.0.4
-eventlet==0.30.0
-notifications-python-client==5.7.1
-Shapely==1.7.1
-
-# PaaS
-awscli-cwlogs>=1.4,<1.5
-itsdangerous==1.1.0
-
-git+https://github.com/alphagov/notifications-utils.git@43.7.0#egg=notifications-utils==43.7.0
-git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.1-alpha#egg=govuk-frontend-jinja==0.5.1-alpha
-
-# gds-metrics requires prometheseus 0.2.0, override that requirement as later versions bring significant performance gains
-prometheus-client==0.9.0
-gds-metrics==0.2.4

From 8b1e4afd3b346070911c624c6fb1b839077196f0 Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Fri, 8 Jan 2021 16:40:51 +0000
Subject: [PATCH 6/7] Install pip tools as part of freeze-requirements

Makes the command faster and more self contained.
---
 Makefile                  | 3 ++-
 requirements_for_test.in  | 1 -
 requirements_for_test.txt | 6 ++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 8aee851af..fdc65cb57 100644
--- a/Makefile
+++ b/Makefile
@@ -75,7 +75,8 @@ fix-imports:
 	isort -rc ./app ./tests
 
 .PHONY: freeze-requirements
-freeze-requirements: requirements-for-test requirements.in requirements_for_test.in
+freeze-requirements: ## create static requirements.txt
+	${VIRTUALENV_ROOT}/bin/pip install pip-tools
 	${VIRTUALENV_ROOT}/bin/pip-compile requirements.in
 	${VIRTUALENV_ROOT}/bin/pip-compile requirements_for_test.in
 
diff --git a/requirements_for_test.in b/requirements_for_test.in
index afa944e1c..bc6a0b2b9 100644
--- a/requirements_for_test.in
+++ b/requirements_for_test.in
@@ -1,5 +1,4 @@
 -r requirements.txt
-pip-tools==5.4.0
 isort==4.3.21
 pytest==5.3.2
 pytest-env==0.6.2
diff --git a/requirements_for_test.txt b/requirements_for_test.txt
index 409161489..2e9a71209 100644
--- a/requirements_for_test.txt
+++ b/requirements_for_test.txt
@@ -17,7 +17,7 @@ botocore==1.19.51         # via -r requirements.txt, awscli, boto3, s3transfer
 cachetools==4.2.0         # via -r requirements.txt, notifications-utils
 certifi==2020.12.5        # via -r requirements.txt, requests
 chardet==4.0.0            # via -r requirements.txt, requests
-click==7.1.2              # via -r requirements.txt, flask, pip-tools
+click==7.1.2              # via -r requirements.txt, flask
 colorama==0.4.3           # via -r requirements.txt, awscli
 dnspython==1.16.0         # via -r requirements.txt, eventlet
 docopt==0.6.2             # via -r requirements.txt, notifications-python-client
@@ -63,7 +63,6 @@ openpyxl==3.0.5           # via -r requirements.txt, pyexcel-xlsx
 orderedset==2.0.3         # via -r requirements.txt, notifications-utils
 packaging==20.8           # via -r requirements.txt, bleach, pytest
 phonenumbers==8.12.15     # via -r requirements.txt, notifications-utils
-pip-tools==5.4.0          # via -r requirements_for_test.in
 pluggy==0.13.1            # via pytest
 prometheus-client==0.9.0  # via -r requirements.txt, gds-metrics
 py==1.10.0                # via pytest, pytest-forked
@@ -94,7 +93,7 @@ requests==2.25.1          # via -r requirements.txt, awscli-cwlogs, govuk-bank-h
 rsa==4.5                  # via -r requirements.txt, awscli
 s3transfer==0.3.3         # via -r requirements.txt, awscli, boto3
 shapely==1.7.1            # via -r requirements.txt
-six==1.15.0               # via -r requirements.txt, awscli-cwlogs, bleach, eventlet, flake8-print, freezegun, govuk-bank-holidays, pip-tools, pytest-xdist, python-dateutil, requests-mock
+six==1.15.0               # via -r requirements.txt, awscli-cwlogs, bleach, eventlet, flake8-print, freezegun, govuk-bank-holidays, pytest-xdist, python-dateutil, requests-mock
 smartypants==2.0.1        # via -r requirements.txt, notifications-utils
 soupsieve==2.1            # via beautifulsoup4
 statsd==3.3.0             # via -r requirements.txt, notifications-utils
@@ -110,5 +109,4 @@ xlwt==1.3.0               # via -r requirements.txt, pyexcel-xls
 zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
-# pip
 # setuptools

From 54d3d0421bdbd762bf03f93b67a68be7b695d079 Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Fri, 8 Jan 2021 16:44:24 +0000
Subject: [PATCH 7/7] Restore and add comments

---
 Makefile | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/Makefile b/Makefile
index fdc65cb57..e9906c3b9 100644
--- a/Makefile
+++ b/Makefile
@@ -45,11 +45,11 @@ upgrade-pip: virtualenv
 	${VIRTUALENV_ROOT}/bin/pip install --upgrade pip
 
 .PHONY: requirements
-requirements: upgrade-pip requirements.txt
+requirements: upgrade-pip requirements.txt ## Install dependencies for running the app
 	${VIRTUALENV_ROOT}/bin/pip install -r requirements.txt
 
 .PHONY: requirements-for-test
-requirements-for-test: upgrade-pip requirements_for_test.txt
+requirements-for-test: upgrade-pip requirements_for_test.txt ## Install all dependencies for running the app, development and testing
 	${VIRTUALENV_ROOT}/bin/pip install -r requirements_for_test.txt
 
 .PHONY: frontend
@@ -63,7 +63,7 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: build
-build: frontend requirements-for-test generate-version-file
+build: frontend requirements-for-test generate-version-file ## Build project
 	npm run build
 
 .PHONY: test