switch from freeze reqs script to pip-tools

instead of alexey's home-grown script, pip-tools offers a quicker, more
efficient and better supported way to freeze requirements.

see prior art here:

https://github.com/alphagov/notifications-admin/pull/3753
https://github.com/alphagov/notifications-ftp/pull/333

Makefile

@@ -54,29 +54,15 @@ generate-version-file: ## Generates the app version file
 	@echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE}
 
 .PHONY: test
-test: test-requirements ## Run tests
+test: ## Run tests
 	flake8 .
 	isort --check-only ./app ./tests
 	pytest -n4 --maxfail=10
 
 .PHONY: freeze-requirements
 freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt
-	rm -rf venv-freeze
+	pip install --upgrade pip-tools
-	virtualenv -p python3 venv-freeze
+	pip-compile requirements.in
-	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
-	echo '# pyup: ignore file' > requirements.txt
-	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
-	cat requirements-app.txt >> requirements.txt
-	echo '' >> requirements.txt
-	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
-	rm -rf venv-freeze
-
-.PHONY: test-requirements
-test-requirements:
-	@diff requirements-app.txt requirements.txt | grep '<' \
-	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
-	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
-|| { echo "requirements.txt is up to date"; exit 0; }
 
 .PHONY: clean
 clean:

README.md

@@ -88,17 +88,9 @@ make test
 
 ## To update application dependencies
 
-`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+## To update application dependencies
-versions of all nested dependencies. If `requirements-app.txt` has been changed (or
-we want to update the unpinned nested dependencies) `requirements.txt` should be
-regenerated with
-
-```
-make freeze-requirements
-```
-
-`requirements.txt` should be committed alongside `requirements-app.txt` changes.
 
+`requirements.txt` is generated from the `requirements.in` in order to pin versions of all nested dependencies. If `requirements.in` has been changed, run `make freeze-requirements` to regenerate it.
 
 ## To run one off tasks
 

requirements.txt

@@ -1,110 +1,263 @@
-# pyup: ignore file
+#
-# This file is autogenerated. Do not edit it manually.
+# This file is autogenerated by pip-compile with python 3.9
-# Run `make freeze-requirements` to update requirements.txt
+# To update, run:
-# with package version changes made in requirements-app.txt
+#
-
+#    pip-compile requirements.in
-cffi==1.14.5
+#
-celery[sqs]==5.1.2
-docopt==0.6.2
-Flask-Bcrypt==0.7.1
-flask-marshmallow==0.14.0
-Flask-Migrate==2.7.0
-git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
-Flask==1.1.2
-click-datetime==0.2
-eventlet==0.30.2 # pyup: ignore # 0.31 breaks Gunicorn
-gunicorn==20.1.0
-iso8601==0.1.14
-itsdangerous==1.1.0
-jsonschema==3.2.0
-marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors
-marshmallow==2.21.0 # pyup: <3 # v3 throws errors
-psycopg2-binary==2.8.6
-PyJWT==2.0.1
-SQLAlchemy==1.4.10
-strict-rfc3339==0.7
-rfc3987==1.3.8
-cachetools==4.2.1
-beautifulsoup4==4.9.3
-lxml==4.6.3
-Werkzeug==2.0.2
-
-# higher version causes build to fail on PaaS due to lack of Rust
-# see https://github.com/pyca/cryptography/issues/5810
-cryptography<3.4 # pyup: <3.4
-
-notifications-python-client==6.0.2
-
-# PaaS
-awscli-cwlogs==1.4.6
-
-git+https://github.com/alphagov/notifications-utils.git@48.0.0#egg=notifications-utils==48.0.0
-
-# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
-prometheus-client==0.10.1
-gds-metrics==0.2.4
-
-## The following requirements were added by pip freeze:
 alembic==1.7.4
+    # via flask-migrate
 amqp==5.0.6
+    # via kombu
 attrs==21.2.0
+    # via jsonschema
 awscli==1.21.4
+    # via
+    #   awscli-cwlogs
+    #   notifications-utils
+awscli-cwlogs==1.4.6
+    # via -r requirements.in
 bcrypt==3.2.0
+    # via flask-bcrypt
+beautifulsoup4==4.9.3
+    # via -r requirements.in
 billiard==3.6.4.0
+    # via celery
 bleach==4.1.0
+    # via notifications-utils
 blinker==1.4
+    # via gds-metrics
 boto3==1.19.4
+    # via
+    #   celery
+    #   notifications-utils
 botocore==1.22.4
-cached-property==1.5.2
+    # via
+    #   awscli
+    #   boto3
+    #   s3transfer
+cachetools==4.2.1
+    # via
+    #   -r requirements.in
+    #   notifications-utils
+celery[sqs]==5.1.2
+    # via -r requirements.in
 certifi==2021.10.8
+    # via requests
+cffi==1.14.5
+    # via
+    #   -r requirements.in
+    #   bcrypt
+    #   cryptography
 charset-normalizer==2.0.7
+    # via requests
 click==7.1.2
+    # via
+    #   celery
+    #   click-datetime
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+    #   flask
+click-datetime==0.2
+    # via -r requirements.in
 click-didyoumean==0.3.0
+    # via celery
 click-plugins==1.1.1
+    # via celery
 click-repl==0.2.0
+    # via celery
 colorama==0.4.3
-dataclasses==0.8
+    # via awscli
+cryptography==3.3.2
+    # via -r requirements.in
 dnspython==1.16.0
+    # via eventlet
+docopt==0.6.2
+    # via
+    #   -r requirements.in
+    #   notifications-python-client
 docutils==0.15.2
+    # via awscli
+eventlet==0.30.2
+    # via -r requirements.in
+flask==1.1.2
+    # via
+    #   -r requirements.in
+    #   flask-bcrypt
+    #   flask-marshmallow
+    #   flask-migrate
+    #   flask-redis
+    #   gds-metrics
+    #   notifications-utils
+flask-bcrypt==0.7.1
+    # via -r requirements.in
+flask-marshmallow==0.14.0
+    # via -r requirements.in
+flask-migrate==2.7.0
+    # via -r requirements.in
 flask-redis==0.4.0
+    # via notifications-utils
+flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262
+    # via
+    #   -r requirements.in
+    #   flask-migrate
+gds-metrics==0.2.4
+    # via -r requirements.in
 geojson==2.5.0
+    # via notifications-utils
 govuk-bank-holidays==0.10
+    # via notifications-utils
 greenlet==1.1.2
+    # via
+    #   eventlet
+    #   sqlalchemy
+gunicorn==20.1.0
+    # via -r requirements.in
 idna==3.3
-importlib-metadata==4.8.1
+    # via requests
-importlib-resources==5.3.0
+iso8601==0.1.14
-Jinja2==3.0.2
+    # via -r requirements.in
+itsdangerous==1.1.0
+    # via
+    #   -r requirements.in
+    #   flask
+    #   notifications-utils
+jinja2==3.0.2
+    # via
+    #   flask
+    #   notifications-utils
 jmespath==0.10.0
+    # via
+    #   boto3
+    #   botocore
+jsonschema==3.2.0
+    # via -r requirements.in
 kombu==5.1.0
-Mako==1.1.5
+    # via celery
-MarkupSafe==2.0.1
+lxml==4.6.3
+    # via -r requirements.in
+mako==1.1.5
+    # via alembic
+markupsafe==2.0.1
+    # via
+    #   jinja2
+    #   mako
+marshmallow==2.21.0
+    # via
+    #   -r requirements.in
+    #   flask-marshmallow
+    #   marshmallow-sqlalchemy
+marshmallow-sqlalchemy==0.23.1
+    # via -r requirements.in
 mistune==0.8.4
+    # via notifications-utils
+notifications-python-client==6.0.2
+    # via -r requirements.in
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@48.0.0
+    # via -r requirements.in
 orderedset==2.0.3
+    # via notifications-utils
 packaging==21.0
+    # via bleach
 phonenumbers==8.12.36
+    # via notifications-utils
+prometheus-client==0.10.1
+    # via
+    #   -r requirements.in
+    #   gds-metrics
 prompt-toolkit==3.0.21
+    # via click-repl
+psycopg2-binary==2.8.6
+    # via -r requirements.in
 pyasn1==0.4.8
+    # via rsa
 pycparser==2.20
+    # via cffi
 pycurl==7.43.0.5
+    # via celery
+pyjwt==2.0.1
+    # via
+    #   -r requirements.in
+    #   notifications-python-client
 pyparsing==3.0.1
-PyPDF2==1.26.0
+    # via packaging
+pypdf2==1.26.0
+    # via notifications-utils
 pyrsistent==0.18.0
+    # via jsonschema
 python-dateutil==2.8.2
+    # via
+    #   awscli-cwlogs
+    #   botocore
 python-json-logger==2.0.2
+    # via notifications-utils
 pytz==2021.3
-PyYAML==5.4.1
+    # via
+    #   celery
+    #   notifications-utils
+pyyaml==5.4.1
+    # via
+    #   awscli
+    #   notifications-utils
 redis==3.5.3
+    # via flask-redis
 requests==2.26.0
+    # via
+    #   awscli-cwlogs
+    #   govuk-bank-holidays
+    #   notifications-python-client
+    #   notifications-utils
+rfc3987==1.3.8
+    # via -r requirements.in
 rsa==4.7.2
+    # via awscli
 s3transfer==0.5.0
-Shapely==1.8.0
+    # via
+    #   awscli
+    #   boto3
+shapely==1.8.0
+    # via notifications-utils
 six==1.16.0
+    # via
+    #   awscli-cwlogs
+    #   bcrypt
+    #   bleach
+    #   click-repl
+    #   cryptography
+    #   eventlet
+    #   flask-marshmallow
+    #   jsonschema
+    #   python-dateutil
 smartypants==2.0.1
+    # via notifications-utils
 soupsieve==2.2.1
+    # via beautifulsoup4
+sqlalchemy==1.4.10
+    # via
+    #   -r requirements.in
+    #   alembic
+    #   marshmallow-sqlalchemy
 statsd==3.3.0
-typing-extensions==3.10.0.2
+    # via notifications-utils
+strict-rfc3339==0.7
+    # via -r requirements.in
 urllib3==1.26.7
+    # via
+    #   botocore
+    #   requests
 vine==5.0.0
+    # via
+    #   amqp
+    #   celery
+    #   kombu
 wcwidth==0.2.5
+    # via prompt-toolkit
 webencodings==0.5.1
-zipp==3.6.0
+    # via bleach
+werkzeug==2.0.2
+    # via
+    #   -r requirements.in
+    #   flask
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools