From 6b5d7ca639ee2379900e51245ecb9600958b619c Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Nov 2021 14:05:17 +0000 Subject: [PATCH 1/3] switch to python 3.9 --- README.md | 2 +- runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e463ac5ee..8fc7c993a 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Contains: ### Python version -At the moment we run Python 3.6 in production. You will run into problems if you try to use Python 3.5 or older, or Python 3.7 or newer. +We run python 3.9 both locally and in production. ### AWS credentials diff --git a/runtime.txt b/runtime.txt index 62ce9c02a..305091cae 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.6.x +python-3.9.x From 036bc92245c054039b1099d8ad356ef8026a0ac9 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Nov 2021 14:05:46 +0000 Subject: [PATCH 2/3] switch from freeze reqs script to pip-tools instead of alexey's home-grown script, pip-tools offers a quicker, more efficient and better supported way to freeze requirements. see prior art here: https://github.com/alphagov/notifications-admin/pull/3753 https://github.com/alphagov/notifications-ftp/pull/333 --- Makefile | 20 +- README.md | 12 +- requirements-app.txt => requirements.in | 0 requirements.txt | 271 ++++++++++++++++++------ 4 files changed, 217 insertions(+), 86 deletions(-) rename requirements-app.txt => requirements.in (100%) diff --git a/Makefile b/Makefile index 18a643dcf..0204d01d3 100644 --- a/Makefile +++ b/Makefile @@ -54,29 +54,15 @@ generate-version-file: ## Generates the app version file @echo -e "__git_commit__ = \"${GIT_COMMIT}\"\n__time__ = \"${DATE}\"" > ${APP_VERSION_FILE} .PHONY: test -test: test-requirements ## Run tests +test: ## Run tests flake8 . isort --check-only ./app ./tests pytest -n4 --maxfail=10 .PHONY: freeze-requirements freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt - rm -rf venv-freeze - virtualenv -p python3 venv-freeze - $$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt - echo '# pyup: ignore file' > requirements.txt - echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt - cat requirements-app.txt >> requirements.txt - echo '' >> requirements.txt - $$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt - rm -rf venv-freeze - -.PHONY: test-requirements -test-requirements: - @diff requirements-app.txt requirements.txt | grep '<' \ - && { echo "requirements.txt doesn't match requirements-app.txt."; \ - echo "Run 'make freeze-requirements' to update."; exit 1; } \ -|| { echo "requirements.txt is up to date"; exit 0; } + pip install --upgrade pip-tools + pip-compile requirements.in .PHONY: clean clean: diff --git a/README.md b/README.md index 8fc7c993a..40abcb1d6 100644 --- a/README.md +++ b/README.md @@ -88,17 +88,9 @@ make test ## To update application dependencies -`requirements.txt` file is generated from the `requirements-app.txt` in order to pin -versions of all nested dependencies. If `requirements-app.txt` has been changed (or -we want to update the unpinned nested dependencies) `requirements.txt` should be -regenerated with - -``` -make freeze-requirements -``` - -`requirements.txt` should be committed alongside `requirements-app.txt` changes. +## To update application dependencies +`requirements.txt` is generated from the `requirements.in` in order to pin versions of all nested dependencies. If `requirements.in` has been changed, run `make freeze-requirements` to regenerate it. ## To run one off tasks diff --git a/requirements-app.txt b/requirements.in similarity index 100% rename from requirements-app.txt rename to requirements.in diff --git a/requirements.txt b/requirements.txt index e93ee962f..6e2d9446f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,110 +1,263 @@ -# pyup: ignore file -# This file is autogenerated. Do not edit it manually. -# Run `make freeze-requirements` to update requirements.txt -# with package version changes made in requirements-app.txt - -cffi==1.14.5 -celery[sqs]==5.1.2 -docopt==0.6.2 -Flask-Bcrypt==0.7.1 -flask-marshmallow==0.14.0 -Flask-Migrate==2.7.0 -git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 -Flask==1.1.2 -click-datetime==0.2 -eventlet==0.30.2 # pyup: ignore # 0.31 breaks Gunicorn -gunicorn==20.1.0 -iso8601==0.1.14 -itsdangerous==1.1.0 -jsonschema==3.2.0 -marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors -marshmallow==2.21.0 # pyup: <3 # v3 throws errors -psycopg2-binary==2.8.6 -PyJWT==2.0.1 -SQLAlchemy==1.4.10 -strict-rfc3339==0.7 -rfc3987==1.3.8 -cachetools==4.2.1 -beautifulsoup4==4.9.3 -lxml==4.6.3 -Werkzeug==2.0.2 - -# higher version causes build to fail on PaaS due to lack of Rust -# see https://github.com/pyca/cryptography/issues/5810 -cryptography<3.4 # pyup: <3.4 - -notifications-python-client==6.0.2 - -# PaaS -awscli-cwlogs==1.4.6 - -git+https://github.com/alphagov/notifications-utils.git@48.0.0#egg=notifications-utils==48.0.0 - -# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains -prometheus-client==0.10.1 -gds-metrics==0.2.4 - -## The following requirements were added by pip freeze: +# +# This file is autogenerated by pip-compile with python 3.9 +# To update, run: +# +# pip-compile requirements.in +# alembic==1.7.4 + # via flask-migrate amqp==5.0.6 + # via kombu attrs==21.2.0 + # via jsonschema awscli==1.21.4 + # via + # awscli-cwlogs + # notifications-utils +awscli-cwlogs==1.4.6 + # via -r requirements.in bcrypt==3.2.0 + # via flask-bcrypt +beautifulsoup4==4.9.3 + # via -r requirements.in billiard==3.6.4.0 + # via celery bleach==4.1.0 + # via notifications-utils blinker==1.4 + # via gds-metrics boto3==1.19.4 + # via + # celery + # notifications-utils botocore==1.22.4 -cached-property==1.5.2 + # via + # awscli + # boto3 + # s3transfer +cachetools==4.2.1 + # via + # -r requirements.in + # notifications-utils +celery[sqs]==5.1.2 + # via -r requirements.in certifi==2021.10.8 + # via requests +cffi==1.14.5 + # via + # -r requirements.in + # bcrypt + # cryptography charset-normalizer==2.0.7 + # via requests click==7.1.2 + # via + # celery + # click-datetime + # click-didyoumean + # click-plugins + # click-repl + # flask +click-datetime==0.2 + # via -r requirements.in click-didyoumean==0.3.0 + # via celery click-plugins==1.1.1 + # via celery click-repl==0.2.0 + # via celery colorama==0.4.3 -dataclasses==0.8 + # via awscli +cryptography==3.3.2 + # via -r requirements.in dnspython==1.16.0 + # via eventlet +docopt==0.6.2 + # via + # -r requirements.in + # notifications-python-client docutils==0.15.2 + # via awscli +eventlet==0.30.2 + # via -r requirements.in +flask==1.1.2 + # via + # -r requirements.in + # flask-bcrypt + # flask-marshmallow + # flask-migrate + # flask-redis + # gds-metrics + # notifications-utils +flask-bcrypt==0.7.1 + # via -r requirements.in +flask-marshmallow==0.14.0 + # via -r requirements.in +flask-migrate==2.7.0 + # via -r requirements.in flask-redis==0.4.0 + # via notifications-utils +flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262 + # via + # -r requirements.in + # flask-migrate +gds-metrics==0.2.4 + # via -r requirements.in geojson==2.5.0 + # via notifications-utils govuk-bank-holidays==0.10 + # via notifications-utils greenlet==1.1.2 + # via + # eventlet + # sqlalchemy +gunicorn==20.1.0 + # via -r requirements.in idna==3.3 -importlib-metadata==4.8.1 -importlib-resources==5.3.0 -Jinja2==3.0.2 + # via requests +iso8601==0.1.14 + # via -r requirements.in +itsdangerous==1.1.0 + # via + # -r requirements.in + # flask + # notifications-utils +jinja2==3.0.2 + # via + # flask + # notifications-utils jmespath==0.10.0 + # via + # boto3 + # botocore +jsonschema==3.2.0 + # via -r requirements.in kombu==5.1.0 -Mako==1.1.5 -MarkupSafe==2.0.1 + # via celery +lxml==4.6.3 + # via -r requirements.in +mako==1.1.5 + # via alembic +markupsafe==2.0.1 + # via + # jinja2 + # mako +marshmallow==2.21.0 + # via + # -r requirements.in + # flask-marshmallow + # marshmallow-sqlalchemy +marshmallow-sqlalchemy==0.23.1 + # via -r requirements.in mistune==0.8.4 + # via notifications-utils +notifications-python-client==6.0.2 + # via -r requirements.in +notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@48.0.0 + # via -r requirements.in orderedset==2.0.3 + # via notifications-utils packaging==21.0 + # via bleach phonenumbers==8.12.36 + # via notifications-utils +prometheus-client==0.10.1 + # via + # -r requirements.in + # gds-metrics prompt-toolkit==3.0.21 + # via click-repl +psycopg2-binary==2.8.6 + # via -r requirements.in pyasn1==0.4.8 + # via rsa pycparser==2.20 + # via cffi pycurl==7.43.0.5 + # via celery +pyjwt==2.0.1 + # via + # -r requirements.in + # notifications-python-client pyparsing==3.0.1 -PyPDF2==1.26.0 + # via packaging +pypdf2==1.26.0 + # via notifications-utils pyrsistent==0.18.0 + # via jsonschema python-dateutil==2.8.2 + # via + # awscli-cwlogs + # botocore python-json-logger==2.0.2 + # via notifications-utils pytz==2021.3 -PyYAML==5.4.1 + # via + # celery + # notifications-utils +pyyaml==5.4.1 + # via + # awscli + # notifications-utils redis==3.5.3 + # via flask-redis requests==2.26.0 + # via + # awscli-cwlogs + # govuk-bank-holidays + # notifications-python-client + # notifications-utils +rfc3987==1.3.8 + # via -r requirements.in rsa==4.7.2 + # via awscli s3transfer==0.5.0 -Shapely==1.8.0 + # via + # awscli + # boto3 +shapely==1.8.0 + # via notifications-utils six==1.16.0 + # via + # awscli-cwlogs + # bcrypt + # bleach + # click-repl + # cryptography + # eventlet + # flask-marshmallow + # jsonschema + # python-dateutil smartypants==2.0.1 + # via notifications-utils soupsieve==2.2.1 + # via beautifulsoup4 +sqlalchemy==1.4.10 + # via + # -r requirements.in + # alembic + # marshmallow-sqlalchemy statsd==3.3.0 -typing-extensions==3.10.0.2 + # via notifications-utils +strict-rfc3339==0.7 + # via -r requirements.in urllib3==1.26.7 + # via + # botocore + # requests vine==5.0.0 + # via + # amqp + # celery + # kombu wcwidth==0.2.5 + # via prompt-toolkit webencodings==0.5.1 -zipp==3.6.0 + # via bleach +werkzeug==2.0.2 + # via + # -r requirements.in + # flask + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From ccdba8a0d842bca9467293882f83425c88b56659 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 10 Nov 2021 15:47:12 +0000 Subject: [PATCH 3/3] make pyup point at new files '(requirements-dev hasn'\''t existed for ages anyway)' --- .pyup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pyup.yml b/.pyup.yml index 0d3d9e533..2dec6592b 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -4,5 +4,5 @@ schedule: "every week on wednesday" search: False requirements: - - requirements-app.txt - - requirements-dev.txt + - requirements.in + - requirements_for_test.txt