Pin all requirements in requirements.txt

The list of top-level dependencies is moved to requirements-app.txt,
which is used by `make freeze-requirements` to generate the full
list of requirements in requirements.txt.

(See alphagov/notifications-api#1938 for details.)

Makefile

@@ -101,6 +101,24 @@ upload-paas-artifact:
 test: venv ## Run tests
 	./scripts/run_tests.sh
 
+.PHONY: freeze-requirements
+freeze-requirements:
+	rm -rf venv-freeze
+	virtualenv -p python3 venv-freeze
+	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
+	echo '# This file is autogenerated. Do not edit it manually.' > requirements.txt
+	cat requirements-app.txt >> requirements.txt
+	echo '' >> requirements.txt
+	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
+	rm -rf venv-freeze
+
+.PHONY: test-requirements
+test-requirements:
+	@diff requirements-app.txt requirements.txt | grep '<' \
+	    && { echo "requirements.txt doesn't match requirements-app.txt."; \
+	         echo "Run 'make freeze-requirements' to update."; exit 1; } \
+|| { echo "requirements.txt is up to date"; exit 0; }
+
 .PHONY: coverage
 coverage: venv ## Create coverage report
 	. venv/bin/activate && coveralls

README.md

@@ -82,3 +82,17 @@ Your aws credentials should be stored in a folder located at `~/.aws`. Follow [A
 ```
 
 Then visit [localhost:6012](http://localhost:6012)
+
+
+## Updating application dependencies
+
+`requirements.txt` file is generated from the `requirements-app.txt` in order to pin
+versions of all nested dependencies. If `requirements-app.txt` has been changed (or
+we want to update the unpinned nested dependencies) `requirements.txt` should be
+regenerated with
+
+```
+make freeze-requirements
+```
+
+`requirements.txt` should be committed alongside `requirements-app.txt` changes.

requirements-app.txt

@@ -0,0 +1,24 @@
+# Run `make freeze-requirements` to update requirements.txt
+# with package version changes made in requirements-app.txt
+
+ago==0.0.92
+Flask==1.0.2
+Flask-WTF==0.14.2
+Flask-Login==0.4.1
+
+blinker==1.4
+pyexcel==0.5.8
+pyexcel-io==0.5.7
+pyexcel-xls==0.5.7
+pyexcel-xlsx==0.5.6
+pyexcel-ods3==0.5.2
+pytz==2018.5
+gunicorn==19.8.1
+whitenoise==3.3.1  #manages static assets
+eventlet==0.22.1
+notifications-python-client==4.10.0
+
+# PaaS
+awscli-cwlogs>=1.4,<1.5
+
+git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2

requirements.txt

@@ -1,8 +1,11 @@
+# Run `make freeze-requirements` to update requirements.txt
+# with package version changes made in requirements-app.txt
+# This file is autogenerated. Do not edit it manually.
+
 ago==0.0.92
 Flask==1.0.2
 Flask-WTF==0.14.2
 Flask-Login==0.4.1
-wtforms==2.1  # pyup: ignore
 
 blinker==1.4
 pyexcel==0.5.8
@@ -20,3 +23,54 @@ notifications-python-client==4.10.0
 awscli-cwlogs>=1.4,<1.5
 
 git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2
+
+## The following requirements were added by pip freeze:
+awscli==1.15.55
+bleach==2.1.3
+boto3==1.6.16
+botocore==1.10.54
+certifi==2018.4.16
+chardet==3.0.4
+click==6.7
+colorama==0.3.9
+docopt==0.6.2
+docutils==0.14
+et-xmlfile==1.0.1
+Flask-Redis==0.3.0
+future==0.16.0
+greenlet==0.4.13
+html5lib==1.0.1
+idna==2.7
+itsdangerous==0.24
+jdcal==1.4
+Jinja2==2.10
+jmespath==0.9.3
+lml==0.0.1
+lxml==4.2.3
+MarkupSafe==1.0
+mistune==0.8.3
+monotonic==1.5
+openpyxl==2.5.4
+orderedset==2.0.1
+phonenumbers==8.9.4
+pyasn1==0.4.3
+pyexcel-ezodf==0.3.4
+PyJWT==1.6.4
+PyPDF2==1.26.0
+python-dateutil==2.7.3
+python-json-logger==0.1.8
+PyYAML==3.12
+redis==2.10.6
+requests==2.19.1
+rsa==3.4.2
+s3transfer==0.1.13
+six==1.11.0
+smartypants==2.0.1
+statsd==3.2.2
+texttable==1.4.0
+urllib3==1.23
+webencodings==0.5.1
+Werkzeug==0.14.1
+WTForms==2.1
+xlrd==1.1.0
+xlwt==1.3.0

scripts/run_tests.sh

@@ -24,6 +24,9 @@ if [[ -z "$VIRTUAL_ENV" ]] && [[ -d venv ]]; then
   source ./venv/bin/activate
 fi
 
+make test-requirements
+display_result $? 1 "Requirements check"
+
 flake8 .
 display_result $? 1 "Code style check"