From 0f68d252659d597c17e1cf9eb4bccef9de0a2a3b Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Tue, 29 Dec 2020 13:21:13 +0000
Subject: [PATCH] Replace custom `pip freeze` script with `pip-tools`

In the past we've avoided using out-of-the-box solutions for Python
dependency resolution because a) they haven't been very mature and b)
we've had lots of issues with version conflicts. See [[1]], [[2]] for
details. Instead, we've been using a custom Python script that
under-the-hood runs `pip freeze` and saves the output to
`requirements.txt`.

This script works well for us, but it doesn't integrate well with other
tools. On the other hand [`pip-tools`](https://github.com/jazzband/pip-tools)
as of 2020 seems to be well-supported by its maintainers and other
tools; for instance, GitHub's automated update service
[Dependabot](https://dependabot.com) supports `requirements.in` files.

This commit replaces our `freeze-requirements` make command with
`pip-compile`.

The Digital Marketplace team have made this change and seem happy with
the results.
---
 Makefile             | 46 +++++++++++++++++++++++---------------------
 scripts/run_tests.sh |  3 ---
 2 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/Makefile b/Makefile
index 35daf9f5c..8aee851af 100644
--- a/Makefile
+++ b/Makefile
@@ -27,6 +27,8 @@ $(eval export CF_HOME)
 
 NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 
+VIRTUALENV_ROOT := $(shell [ -z $$VIRTUAL_ENV ] && echo $$(pwd)/venv || echo $$VIRTUAL_ENV)
+
 
 ## DEVELOPMENT
 
@@ -34,21 +36,35 @@ NOTIFY_CREDENTIALS ?= ~/.notify-credentials
 help:
 	@cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-.PHONY: dependencies
-dependencies: ## Install build dependencies
+.PHONY: virtualenv
+virtualenv:
+	[ -z $$VIRTUAL_ENV ] && [ ! -d venv ] && python3 -m venv venv || true
+
+.PHONY: upgrade-pip
+upgrade-pip: virtualenv
+	${VIRTUALENV_ROOT}/bin/pip install --upgrade pip
+
+.PHONY: requirements
+requirements: upgrade-pip requirements.txt
+	${VIRTUALENV_ROOT}/bin/pip install -r requirements.txt
+
+.PHONY: requirements-for-test
+requirements-for-test: upgrade-pip requirements_for_test.txt
+	${VIRTUALENV_ROOT}/bin/pip install -r requirements_for_test.txt
+
+.PHONY: frontend
+frontend:
 	npm set progress=false
 	npm install
 	npm rebuild node-sass
-	pip install -r requirements_for_test.txt
 
 .PHONY: generate-version-file
 generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: build
-build: dependencies generate-version-file ## Build project
+build: frontend requirements-for-test generate-version-file
 	npm run build
-	pip install -r requirements.txt
 
 .PHONY: test
 test: ## Run tests
@@ -59,23 +75,9 @@ fix-imports:
 	isort -rc ./app ./tests
 
 .PHONY: freeze-requirements
-freeze-requirements:
-	rm -rf venv-freeze
-	virtualenv -p python3 venv-freeze
-	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
-	echo '# pyup: ignore file' > requirements.txt
-	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
-	cat requirements-app.txt >> requirements.txt
-	echo '' >> requirements.txt
-	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
-	rm -rf venv-freeze
-
-.PHONY: test-requirements
-test-requirements:
-	@diff requirements-app.txt requirements.txt | grep '<' \
-	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
-	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
-|| { echo "requirements.txt is up to date"; exit 0; }
+freeze-requirements: requirements-for-test requirements.in requirements_for_test.in
+	${VIRTUALENV_ROOT}/bin/pip-compile requirements.in
+	${VIRTUALENV_ROOT}/bin/pip-compile requirements_for_test.in
 
 .PHONY: prepare-docker-build-image
 prepare-docker-build-image: ## Prepare the Docker builder image
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
index c676b6052..dc3d949ee 100755
--- a/scripts/run_tests.sh
+++ b/scripts/run_tests.sh
@@ -24,9 +24,6 @@ if [[ -z "$VIRTUAL_ENV" ]] && [[ -d venv ]]; then
   source ./venv/bin/activate
 fi
 
-make test-requirements
-display_result $? 1 "Requirements check"
-
 flake8 .
 display_result $? 1 "Code style check"