From 14ef2b95cd91349675a77b2de58a060d63a67bb1 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 16 Feb 2021 16:16:17 +0000 Subject: [PATCH 1/4] Revert "Revert "Scheduled weekly dependency update for week 07"" This reverts commit e23daa205cb9dd07693408bfb2d745f344075970. --- requirements.in | 6 +++--- requirements.txt | 42 ++++++++++++++++++++-------------------- requirements_for_test.in | 20 +++++++++---------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/requirements.in b/requirements.in index a1b35faf7..9afa61e71 100644 --- a/requirements.in +++ b/requirements.in @@ -14,10 +14,10 @@ pyexcel-io==0.6.4 pyexcel-xls==0.6.2 pyexcel-xlsx==0.6.0 pyexcel-ods3==0.6.0 -pytz==2020.5 +pytz==2021.1 gunicorn==20.0.4 -eventlet==0.30.0 -notifications-python-client==5.7.1 +eventlet==0.30.1 +notifications-python-client==6.0.2 Shapely==1.7.1 # PaaS diff --git a/requirements.txt b/requirements.txt index c90199f42..e38be2cf0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,7 +8,7 @@ ago==0.0.93 # via -r requirements.in awscli-cwlogs==1.4.6 # via -r requirements.in -awscli==1.18.211 +awscli==1.19.7 # via # awscli-cwlogs # notifications-utils @@ -18,14 +18,14 @@ blinker==1.4 # via # -r requirements.in # gds-metrics -boto3==1.16.51 +boto3==1.17.7 # via notifications-utils -botocore==1.19.51 +botocore==1.20.7 # via # awscli # boto3 # s3transfer -cachetools==4.2.0 +cachetools==4.2.1 # via notifications-utils certifi==2020.12.5 # via requests @@ -33,17 +33,17 @@ chardet==4.0.0 # via requests click==7.1.2 # via flask -colorama==0.4.3 +colorama==0.4.4 # via awscli -dnspython==1.16.0 +dnspython==2.1.0 # via eventlet docopt==0.6.2 # via notifications-python-client -docutils==0.15.2 +docutils==0.16 # via awscli et-xmlfile==1.0.1 # via openpyxl -eventlet==0.30.0 +eventlet==0.30.1 # via -r requirements.in flask-login==0.5.0 # via -r requirements.in @@ -71,13 +71,13 @@ govuk-bank-holidays==0.8 # notifications-utils git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.8-alpha#egg=govuk-frontend-jinja==0.5.8-alpha # via -r requirements.in -greenlet==0.4.17 +greenlet==1.0.0 # via eventlet gunicorn==20.0.4 # via -r requirements.in humanize==3.2.0 # via -r requirements.in -idna==2.10 +idna==3.1 # via requests itsdangerous==1.1.0 # via @@ -112,17 +112,17 @@ mistune==0.8.4 # via notifications-utils monotonic==1.5 # via notifications-python-client -notifications-python-client==5.7.1 +notifications-python-client==6.0.2 # via -r requirements.in git+https://github.com/alphagov/notifications-utils.git@43.8.3#egg=notifications-utils==43.8.3 # via -r requirements.in -openpyxl==3.0.5 +openpyxl==3.0.6 # via pyexcel-xlsx orderedset==2.0.3 # via notifications-utils -packaging==20.8 +packaging==20.9 # via bleach -phonenumbers==8.12.15 +phonenumbers==8.12.18 # via notifications-utils prometheus-client==0.9.0 # via @@ -147,7 +147,7 @@ pyexcel-xlsx==0.6.0 # via -r requirements.in pyexcel==0.6.6 # via -r requirements.in -pyjwt==2.0.0 +pyjwt==2.0.1 # via notifications-python-client pyparsing==2.4.7 # via packaging @@ -159,11 +159,11 @@ python-dateutil==2.8.1 # botocore python-json-logger==2.0.1 # via notifications-utils -pytz==2020.5 +pytz==2021.1 # via # -r requirements.in # notifications-utils -pyyaml==5.4.0 +pyyaml==5.4.1 # via # awscli # notifications-utils @@ -175,9 +175,9 @@ requests==2.25.1 # govuk-bank-holidays # notifications-python-client # notifications-utils -rsa==4.5 +rsa==4.7 # via awscli -s3transfer==0.3.3 +s3transfer==0.3.4 # via # awscli # boto3 @@ -198,7 +198,7 @@ statsd==3.3.0 # via notifications-utils texttable==1.6.3 # via pyexcel -urllib3==1.26.2 +urllib3==1.26.3 # via # botocore # requests @@ -208,7 +208,7 @@ werkzeug==1.0.1 # via flask wtforms==2.3.3 # via flask-wtf -xlrd==1.2.0 +xlrd==2.0.1 # via pyexcel-xls xlwt==1.3.0 # via pyexcel-xls diff --git a/requirements_for_test.in b/requirements_for_test.in index bc6a0b2b9..ba557520d 100644 --- a/requirements_for_test.in +++ b/requirements_for_test.in @@ -1,14 +1,14 @@ -r requirements.txt -isort==4.3.21 -pytest==5.3.2 +isort==5.7.0 +pytest==6.2.2 pytest-env==0.6.2 -pytest-mock==1.11.2 -pytest-xdist==1.31.0 -beautifulsoup4==4.8.1 -freezegun==0.3.12 -flake8==3.7.9 -flake8-bugbear==19.8.0 -flake8-print==3.1.4 -requests-mock==1.7.0 +pytest-mock==3.5.1 +pytest-xdist==2.2.1 +beautifulsoup4==4.9.3 +freezegun==1.1.0 +flake8==3.8.4 +flake8-bugbear==20.11.1 +flake8-print==4.0.0 +requests-mock==1.8.0 # used for creating manifest file locally jinja2-cli[yaml]==0.7.0 From 627e4e41ab1bd9c8660f8cc5fa19932e506f2727 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 16 Feb 2021 17:08:03 +0000 Subject: [PATCH 2/4] Stop locking non-test dependencies twice Currently we have a situation where we're not running tests against new versions of dependencies, as requirements_for_test.txt is not being kept in-sync with requirements.txt by pyup. Deploys are only working because Concourse silently ignores version issues. From a deployment log: awscli 1.18.211 has requirement PyYAML<5.4,>=3.10; python_version != "3.4", but you'll have pyyaml 5.4 which is incompatible. This switches to a single requirements file for test dependencies, in order to keep it in-sync with requirements.txt i.e. we run our tests against the same versions of dependencies that we deploy with, and the build fails if we try to use package versions that are not mutually compatible, as this example PR shows [1]. ERROR: Cannot install -r requirements_for_test.txt (line 17), -r requirements_for_test.txt (line 198) and pyyaml==5.4.1 because these package versions have conflicting dependencies. We shouldn't need to have fine-grained locking on test dependencies, beyond those we want to list manually in the file. [1]: https://github.com/alphagov/notifications-admin/pull/3804 --- .pyup.yml | 2 +- Makefile | 1 - requirements_for_test.in | 14 -- requirements_for_test.txt | 401 ++------------------------------------ 4 files changed, 14 insertions(+), 404 deletions(-) delete mode 100644 requirements_for_test.in diff --git a/.pyup.yml b/.pyup.yml index 733d5f032..b5d4f46c3 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -5,4 +5,4 @@ schedule: "every week on monday" search: False requirements: - requirements.in - - requirements_for_test.in + - requirements_for_test.txt diff --git a/Makefile b/Makefile index e9906c3b9..da06fad25 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,6 @@ fix-imports: freeze-requirements: ## create static requirements.txt ${VIRTUALENV_ROOT}/bin/pip install pip-tools ${VIRTUALENV_ROOT}/bin/pip-compile requirements.in - ${VIRTUALENV_ROOT}/bin/pip-compile requirements_for_test.in .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image diff --git a/requirements_for_test.in b/requirements_for_test.in deleted file mode 100644 index ba557520d..000000000 --- a/requirements_for_test.in +++ /dev/null @@ -1,14 +0,0 @@ --r requirements.txt -isort==5.7.0 -pytest==6.2.2 -pytest-env==0.6.2 -pytest-mock==3.5.1 -pytest-xdist==2.2.1 -beautifulsoup4==4.9.3 -freezegun==1.1.0 -flake8==3.8.4 -flake8-bugbear==20.11.1 -flake8-print==4.0.0 -requests-mock==1.8.0 -# used for creating manifest file locally -jinja2-cli[yaml]==0.7.0 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 9ecfa5a90..ba557520d 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,389 +1,14 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile requirements_for_test.in -# -ago==0.0.93 - # via -r requirements.txt -apipkg==1.5 - # via execnet -attrs==20.3.0 - # via - # flake8-bugbear - # pytest -awscli-cwlogs==1.4.6 - # via -r requirements.txt -awscli==1.18.211 - # via - # -r requirements.txt - # awscli-cwlogs - # notifications-utils -beautifulsoup4==4.8.1 - # via -r requirements_for_test.in -bleach==3.3.0 - # via - # -r requirements.txt - # notifications-utils -blinker==1.4 - # via - # -r requirements.txt - # gds-metrics -boto3==1.16.51 - # via - # -r requirements.txt - # notifications-utils -botocore==1.19.51 - # via - # -r requirements.txt - # awscli - # boto3 - # s3transfer -cachetools==4.2.0 - # via - # -r requirements.txt - # notifications-utils -certifi==2020.12.5 - # via - # -r requirements.txt - # requests -chardet==4.0.0 - # via - # -r requirements.txt - # requests -click==7.1.2 - # via - # -r requirements.txt - # flask -colorama==0.4.3 - # via - # -r requirements.txt - # awscli -dnspython==1.16.0 - # via - # -r requirements.txt - # eventlet -docopt==0.6.2 - # via - # -r requirements.txt - # notifications-python-client -docutils==0.15.2 - # via - # -r requirements.txt - # awscli -entrypoints==0.3 - # via flake8 -et-xmlfile==1.0.1 - # via - # -r requirements.txt - # openpyxl -eventlet==0.30.0 - # via -r requirements.txt -execnet==1.7.1 - # via pytest-xdist -flake8-bugbear==19.8.0 - # via -r requirements_for_test.in -flake8-print==3.1.4 - # via -r requirements_for_test.in -flake8==3.7.9 - # via - # -r requirements_for_test.in - # flake8-bugbear - # flake8-print -flask-login==0.5.0 - # via -r requirements.txt -flask-redis==0.4.0 - # via - # -r requirements.txt - # notifications-utils -flask-wtf==0.14.3 - # via -r requirements.txt -flask==1.1.2 - # via - # -r requirements.txt - # flask-login - # flask-redis - # flask-wtf - # gds-metrics - # notifications-utils -freezegun==0.3.12 - # via -r requirements_for_test.in -future==0.18.2 - # via - # -r requirements.txt - # notifications-python-client -gds-metrics==0.2.4 - # via -r requirements.txt -geojson==2.5.0 - # via - # -r requirements.txt - # notifications-utils -govuk-bank-holidays==0.8 - # via - # -r requirements.txt - # notifications-utils -git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.8-alpha#egg=govuk-frontend-jinja==0.5.8-alpha - # via -r requirements.txt -greenlet==0.4.17 - # via - # -r requirements.txt - # eventlet -gunicorn==20.0.4 - # via -r requirements.txt -humanize==3.2.0 - # via -r requirements.txt -idna==2.10 - # via - # -r requirements.txt - # requests -importlib-metadata==3.3.0 - # via - # pluggy - # pytest -isort==4.3.21 - # via -r requirements_for_test.in -itsdangerous==1.1.0 - # via - # -r requirements.txt - # flask - # flask-wtf - # notifications-utils -jdcal==1.4.1 - # via - # -r requirements.txt - # openpyxl -jinja2-cli[yaml]==0.7.0 - # via -r requirements_for_test.in -jinja2==2.11.3 - # via - # -r requirements.txt - # flask - # govuk-frontend-jinja - # jinja2-cli - # notifications-utils -jmespath==0.10.0 - # via - # -r requirements.txt - # boto3 - # botocore -lml==0.1.0 - # via - # -r requirements.txt - # pyexcel - # pyexcel-io -lxml==4.6.2 - # via - # -r requirements.txt - # pyexcel-ezodf - # pyexcel-ods3 -markupsafe==1.1.1 - # via - # -r requirements.txt - # jinja2 - # wtforms -mccabe==0.6.1 - # via flake8 -mistune==0.8.4 - # via - # -r requirements.txt - # notifications-utils -monotonic==1.5 - # via - # -r requirements.txt - # notifications-python-client -more-itertools==8.6.0 - # via pytest -notifications-python-client==5.7.1 - # via -r requirements.txt -git+https://github.com/alphagov/notifications-utils.git@43.8.3#egg=notifications-utils==43.8.3 - # via -r requirements.txt -openpyxl==3.0.5 - # via - # -r requirements.txt - # pyexcel-xlsx -orderedset==2.0.3 - # via - # -r requirements.txt - # notifications-utils -packaging==20.8 - # via - # -r requirements.txt - # bleach - # pytest -phonenumbers==8.12.15 - # via - # -r requirements.txt - # notifications-utils -pluggy==0.13.1 - # via pytest -prometheus-client==0.9.0 - # via - # -r requirements.txt - # gds-metrics -py==1.10.0 - # via - # pytest - # pytest-forked -pyasn1==0.4.8 - # via - # -r requirements.txt - # rsa -pycodestyle==2.5.0 - # via - # flake8 - # flake8-print -pyexcel-ezodf==0.3.4 - # via - # -r requirements.txt - # pyexcel-ods3 -pyexcel-io==0.6.4 - # via - # -r requirements.txt - # pyexcel - # pyexcel-ods3 - # pyexcel-xls - # pyexcel-xlsx -pyexcel-ods3==0.6.0 - # via -r requirements.txt -pyexcel-xls==0.6.2 - # via -r requirements.txt -pyexcel-xlsx==0.6.0 - # via -r requirements.txt -pyexcel==0.6.6 - # via -r requirements.txt -pyflakes==2.1.1 - # via flake8 -pyjwt==2.0.0 - # via - # -r requirements.txt - # notifications-python-client -pyparsing==2.4.7 - # via - # -r requirements.txt - # packaging -pypdf2==1.26.0 - # via - # -r requirements.txt - # notifications-utils +-r requirements.txt +isort==5.7.0 +pytest==6.2.2 pytest-env==0.6.2 - # via -r requirements_for_test.in -pytest-forked==1.3.0 - # via pytest-xdist -pytest-mock==1.11.2 - # via -r requirements_for_test.in -pytest-xdist==1.31.0 - # via -r requirements_for_test.in -pytest==5.3.2 - # via - # -r requirements_for_test.in - # pytest-env - # pytest-forked - # pytest-mock - # pytest-xdist -python-dateutil==2.8.1 - # via - # -r requirements.txt - # awscli-cwlogs - # botocore - # freezegun -python-json-logger==2.0.1 - # via - # -r requirements.txt - # notifications-utils -pytz==2020.5 - # via - # -r requirements.txt - # notifications-utils -pyyaml==5.3.1 - # via - # -r requirements.txt - # awscli - # jinja2-cli - # notifications-utils -redis==3.5.3 - # via - # -r requirements.txt - # flask-redis -requests-mock==1.7.0 - # via -r requirements_for_test.in -requests==2.25.1 - # via - # -r requirements.txt - # awscli-cwlogs - # govuk-bank-holidays - # notifications-python-client - # notifications-utils - # requests-mock -rsa==4.5 - # via - # -r requirements.txt - # awscli -s3transfer==0.3.3 - # via - # -r requirements.txt - # awscli - # boto3 -shapely==1.7.1 - # via - # -r requirements.txt - # notifications-utils -six==1.15.0 - # via - # -r requirements.txt - # awscli-cwlogs - # bleach - # eventlet - # flake8-print - # freezegun - # govuk-bank-holidays - # pytest-xdist - # python-dateutil - # requests-mock -smartypants==2.0.1 - # via - # -r requirements.txt - # notifications-utils -soupsieve==2.1 - # via beautifulsoup4 -statsd==3.3.0 - # via - # -r requirements.txt - # notifications-utils -texttable==1.6.3 - # via - # -r requirements.txt - # pyexcel -typing-extensions==3.7.4.3 - # via importlib-metadata -urllib3==1.26.2 - # via - # -r requirements.txt - # botocore - # requests -wcwidth==0.2.5 - # via pytest -webencodings==0.5.1 - # via - # -r requirements.txt - # bleach -werkzeug==1.0.1 - # via - # -r requirements.txt - # flask -wtforms==2.3.3 - # via - # -r requirements.txt - # flask-wtf -xlrd==1.2.0 - # via - # -r requirements.txt - # pyexcel-xls -xlwt==1.3.0 - # via - # -r requirements.txt - # pyexcel-xls -zipp==3.4.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +pytest-mock==3.5.1 +pytest-xdist==2.2.1 +beautifulsoup4==4.9.3 +freezegun==1.1.0 +flake8==3.8.4 +flake8-bugbear==20.11.1 +flake8-print==4.0.0 +requests-mock==1.8.0 +# used for creating manifest file locally +jinja2-cli[yaml]==0.7.0 From 5d946c9d0b6c02b3ad9a0ddcf0c5ab53daa96a29 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 16 Feb 2021 16:35:45 +0000 Subject: [PATCH 3/4] Run 'make freeze-requirements' to fix install errors This downgrades various packages so they are mutually compatible and "pip install -r requirements.txt" succeeds. This downgrades to pyyaml 5.3.1, despite it having a security issue, in order to fix the build for the time being. This also downgrades dnspython, due to a suspected issue with eventlet [1], which caused the admin app to start failing with errors like this: File "/home/vcap/deps/0/python/lib/python3.6/site-packages/urllib3/connectionpool.py", line 1010, in _validate_conn conn.connect() File "/home/vcap/deps/0/python/lib/python3.6/site-packages/urllib3/connection.py", line 394, in connect cert_reqs=resolve_cert_reqs(self.cert_reqs), File "/home/vcap/deps/0/python/lib/python3.6/site-packages/urllib3/util/ssl_.py", line 303, in create_urllib3_context context.options |= options File "/home/vcap/deps/0/python/lib/python3.6/ssl.py", line 465, in options super(SSLContext, SSLContext).options.__set__(self, value) File "/home/vcap/deps/0/python/lib/python3.6/ssl.py", line 465, in options super(SSLContext, SSLContext).options.__set__(self, value) File "/home/vcap/deps/0/python/lib/python3.6/ssl.py", line 465, in options super(SSLContext, SSLContext).options.__set__(self, value) [Previous line repeated 280 more times] RecursionError: maximum recursion depth exceeded while calling a Python object [1]: https://github.com/eventlet/eventlet/pull/684 --- requirements.txt | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/requirements.txt b/requirements.txt index e38be2cf0..8a25960bf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,13 +33,13 @@ chardet==4.0.0 # via requests click==7.1.2 # via flask -colorama==0.4.4 +colorama==0.4.3 # via awscli -dnspython==2.1.0 +dnspython==1.16.0 # via eventlet docopt==0.6.2 # via notifications-python-client -docutils==0.16 +docutils==0.15.2 # via awscli et-xmlfile==1.0.1 # via openpyxl @@ -59,8 +59,6 @@ flask==1.1.2 # flask-wtf # gds-metrics # notifications-utils -future==0.18.2 - # via notifications-python-client gds-metrics==0.2.4 # via -r requirements.in geojson==2.5.0 @@ -77,7 +75,7 @@ gunicorn==20.0.4 # via -r requirements.in humanize==3.2.0 # via -r requirements.in -idna==3.1 +idna==2.10 # via requests itsdangerous==1.1.0 # via @@ -110,8 +108,6 @@ markupsafe==1.1.1 # wtforms mistune==0.8.4 # via notifications-utils -monotonic==1.5 - # via notifications-python-client notifications-python-client==6.0.2 # via -r requirements.in git+https://github.com/alphagov/notifications-utils.git@43.8.3#egg=notifications-utils==43.8.3 @@ -163,7 +159,7 @@ pytz==2021.1 # via # -r requirements.in # notifications-utils -pyyaml==5.4.1 +pyyaml==5.3.1 # via # awscli # notifications-utils @@ -175,7 +171,7 @@ requests==2.25.1 # govuk-bank-holidays # notifications-python-client # notifications-utils -rsa==4.7 +rsa==4.5 # via awscli s3transfer==0.3.4 # via @@ -208,7 +204,7 @@ werkzeug==1.0.1 # via flask wtforms==2.3.3 # via flask-wtf -xlrd==2.0.1 +xlrd==1.2.0 # via pyexcel-xls xlwt==1.3.0 # via pyexcel-xls From 765c8ddbe2ec8602d74d334750ac62446ebabc67 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 16 Feb 2021 17:33:05 +0000 Subject: [PATCH 4/4] Keep test dependencies as-is for now These are leading to multiple failures: - flake8 fails with various issues - isort fails with various issues - pytest fails on a couple of 2FA tests While we can and should upgrade these dependencies, the priority is fixing the build so that we can do this reliably. --- requirements_for_test.txt | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements_for_test.txt b/requirements_for_test.txt index ba557520d..bc6a0b2b9 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,14 +1,14 @@ -r requirements.txt -isort==5.7.0 -pytest==6.2.2 +isort==4.3.21 +pytest==5.3.2 pytest-env==0.6.2 -pytest-mock==3.5.1 -pytest-xdist==2.2.1 -beautifulsoup4==4.9.3 -freezegun==1.1.0 -flake8==3.8.4 -flake8-bugbear==20.11.1 -flake8-print==4.0.0 -requests-mock==1.8.0 +pytest-mock==1.11.2 +pytest-xdist==1.31.0 +beautifulsoup4==4.8.1 +freezegun==0.3.12 +flake8==3.7.9 +flake8-bugbear==19.8.0 +flake8-print==3.1.4 +requests-mock==1.7.0 # used for creating manifest file locally jinja2-cli[yaml]==0.7.0