From 627e4e41ab1bd9c8660f8cc5fa19932e506f2727 Mon Sep 17 00:00:00 2001 From: Ben Thorner Date: Tue, 16 Feb 2021 17:08:03 +0000 Subject: [PATCH] Stop locking non-test dependencies twice Currently we have a situation where we're not running tests against new versions of dependencies, as requirements_for_test.txt is not being kept in-sync with requirements.txt by pyup. Deploys are only working because Concourse silently ignores version issues. From a deployment log: awscli 1.18.211 has requirement PyYAML<5.4,>=3.10; python_version != "3.4", but you'll have pyyaml 5.4 which is incompatible. This switches to a single requirements file for test dependencies, in order to keep it in-sync with requirements.txt i.e. we run our tests against the same versions of dependencies that we deploy with, and the build fails if we try to use package versions that are not mutually compatible, as this example PR shows [1]. ERROR: Cannot install -r requirements_for_test.txt (line 17), -r requirements_for_test.txt (line 198) and pyyaml==5.4.1 because these package versions have conflicting dependencies. We shouldn't need to have fine-grained locking on test dependencies, beyond those we want to list manually in the file. [1]: https://github.com/alphagov/notifications-admin/pull/3804 --- .pyup.yml | 2 +- Makefile | 1 - requirements_for_test.in | 14 -- requirements_for_test.txt | 401 ++------------------------------------ 4 files changed, 14 insertions(+), 404 deletions(-) delete mode 100644 requirements_for_test.in diff --git a/.pyup.yml b/.pyup.yml index 733d5f032..b5d4f46c3 100644 --- a/.pyup.yml +++ b/.pyup.yml @@ -5,4 +5,4 @@ schedule: "every week on monday" search: False requirements: - requirements.in - - requirements_for_test.in + - requirements_for_test.txt diff --git a/Makefile b/Makefile index e9906c3b9..da06fad25 100644 --- a/Makefile +++ b/Makefile @@ -78,7 +78,6 @@ fix-imports: freeze-requirements: ## create static requirements.txt ${VIRTUALENV_ROOT}/bin/pip install pip-tools ${VIRTUALENV_ROOT}/bin/pip-compile requirements.in - ${VIRTUALENV_ROOT}/bin/pip-compile requirements_for_test.in .PHONY: prepare-docker-build-image prepare-docker-build-image: ## Prepare the Docker builder image diff --git a/requirements_for_test.in b/requirements_for_test.in deleted file mode 100644 index ba557520d..000000000 --- a/requirements_for_test.in +++ /dev/null @@ -1,14 +0,0 @@ --r requirements.txt -isort==5.7.0 -pytest==6.2.2 -pytest-env==0.6.2 -pytest-mock==3.5.1 -pytest-xdist==2.2.1 -beautifulsoup4==4.9.3 -freezegun==1.1.0 -flake8==3.8.4 -flake8-bugbear==20.11.1 -flake8-print==4.0.0 -requests-mock==1.8.0 -# used for creating manifest file locally -jinja2-cli[yaml]==0.7.0 diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 9ecfa5a90..ba557520d 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,389 +1,14 @@ -# -# This file is autogenerated by pip-compile -# To update, run: -# -# pip-compile requirements_for_test.in -# -ago==0.0.93 - # via -r requirements.txt -apipkg==1.5 - # via execnet -attrs==20.3.0 - # via - # flake8-bugbear - # pytest -awscli-cwlogs==1.4.6 - # via -r requirements.txt -awscli==1.18.211 - # via - # -r requirements.txt - # awscli-cwlogs - # notifications-utils -beautifulsoup4==4.8.1 - # via -r requirements_for_test.in -bleach==3.3.0 - # via - # -r requirements.txt - # notifications-utils -blinker==1.4 - # via - # -r requirements.txt - # gds-metrics -boto3==1.16.51 - # via - # -r requirements.txt - # notifications-utils -botocore==1.19.51 - # via - # -r requirements.txt - # awscli - # boto3 - # s3transfer -cachetools==4.2.0 - # via - # -r requirements.txt - # notifications-utils -certifi==2020.12.5 - # via - # -r requirements.txt - # requests -chardet==4.0.0 - # via - # -r requirements.txt - # requests -click==7.1.2 - # via - # -r requirements.txt - # flask -colorama==0.4.3 - # via - # -r requirements.txt - # awscli -dnspython==1.16.0 - # via - # -r requirements.txt - # eventlet -docopt==0.6.2 - # via - # -r requirements.txt - # notifications-python-client -docutils==0.15.2 - # via - # -r requirements.txt - # awscli -entrypoints==0.3 - # via flake8 -et-xmlfile==1.0.1 - # via - # -r requirements.txt - # openpyxl -eventlet==0.30.0 - # via -r requirements.txt -execnet==1.7.1 - # via pytest-xdist -flake8-bugbear==19.8.0 - # via -r requirements_for_test.in -flake8-print==3.1.4 - # via -r requirements_for_test.in -flake8==3.7.9 - # via - # -r requirements_for_test.in - # flake8-bugbear - # flake8-print -flask-login==0.5.0 - # via -r requirements.txt -flask-redis==0.4.0 - # via - # -r requirements.txt - # notifications-utils -flask-wtf==0.14.3 - # via -r requirements.txt -flask==1.1.2 - # via - # -r requirements.txt - # flask-login - # flask-redis - # flask-wtf - # gds-metrics - # notifications-utils -freezegun==0.3.12 - # via -r requirements_for_test.in -future==0.18.2 - # via - # -r requirements.txt - # notifications-python-client -gds-metrics==0.2.4 - # via -r requirements.txt -geojson==2.5.0 - # via - # -r requirements.txt - # notifications-utils -govuk-bank-holidays==0.8 - # via - # -r requirements.txt - # notifications-utils -git+https://github.com/alphagov/govuk-frontend-jinja.git@v0.5.8-alpha#egg=govuk-frontend-jinja==0.5.8-alpha - # via -r requirements.txt -greenlet==0.4.17 - # via - # -r requirements.txt - # eventlet -gunicorn==20.0.4 - # via -r requirements.txt -humanize==3.2.0 - # via -r requirements.txt -idna==2.10 - # via - # -r requirements.txt - # requests -importlib-metadata==3.3.0 - # via - # pluggy - # pytest -isort==4.3.21 - # via -r requirements_for_test.in -itsdangerous==1.1.0 - # via - # -r requirements.txt - # flask - # flask-wtf - # notifications-utils -jdcal==1.4.1 - # via - # -r requirements.txt - # openpyxl -jinja2-cli[yaml]==0.7.0 - # via -r requirements_for_test.in -jinja2==2.11.3 - # via - # -r requirements.txt - # flask - # govuk-frontend-jinja - # jinja2-cli - # notifications-utils -jmespath==0.10.0 - # via - # -r requirements.txt - # boto3 - # botocore -lml==0.1.0 - # via - # -r requirements.txt - # pyexcel - # pyexcel-io -lxml==4.6.2 - # via - # -r requirements.txt - # pyexcel-ezodf - # pyexcel-ods3 -markupsafe==1.1.1 - # via - # -r requirements.txt - # jinja2 - # wtforms -mccabe==0.6.1 - # via flake8 -mistune==0.8.4 - # via - # -r requirements.txt - # notifications-utils -monotonic==1.5 - # via - # -r requirements.txt - # notifications-python-client -more-itertools==8.6.0 - # via pytest -notifications-python-client==5.7.1 - # via -r requirements.txt -git+https://github.com/alphagov/notifications-utils.git@43.8.3#egg=notifications-utils==43.8.3 - # via -r requirements.txt -openpyxl==3.0.5 - # via - # -r requirements.txt - # pyexcel-xlsx -orderedset==2.0.3 - # via - # -r requirements.txt - # notifications-utils -packaging==20.8 - # via - # -r requirements.txt - # bleach - # pytest -phonenumbers==8.12.15 - # via - # -r requirements.txt - # notifications-utils -pluggy==0.13.1 - # via pytest -prometheus-client==0.9.0 - # via - # -r requirements.txt - # gds-metrics -py==1.10.0 - # via - # pytest - # pytest-forked -pyasn1==0.4.8 - # via - # -r requirements.txt - # rsa -pycodestyle==2.5.0 - # via - # flake8 - # flake8-print -pyexcel-ezodf==0.3.4 - # via - # -r requirements.txt - # pyexcel-ods3 -pyexcel-io==0.6.4 - # via - # -r requirements.txt - # pyexcel - # pyexcel-ods3 - # pyexcel-xls - # pyexcel-xlsx -pyexcel-ods3==0.6.0 - # via -r requirements.txt -pyexcel-xls==0.6.2 - # via -r requirements.txt -pyexcel-xlsx==0.6.0 - # via -r requirements.txt -pyexcel==0.6.6 - # via -r requirements.txt -pyflakes==2.1.1 - # via flake8 -pyjwt==2.0.0 - # via - # -r requirements.txt - # notifications-python-client -pyparsing==2.4.7 - # via - # -r requirements.txt - # packaging -pypdf2==1.26.0 - # via - # -r requirements.txt - # notifications-utils +-r requirements.txt +isort==5.7.0 +pytest==6.2.2 pytest-env==0.6.2 - # via -r requirements_for_test.in -pytest-forked==1.3.0 - # via pytest-xdist -pytest-mock==1.11.2 - # via -r requirements_for_test.in -pytest-xdist==1.31.0 - # via -r requirements_for_test.in -pytest==5.3.2 - # via - # -r requirements_for_test.in - # pytest-env - # pytest-forked - # pytest-mock - # pytest-xdist -python-dateutil==2.8.1 - # via - # -r requirements.txt - # awscli-cwlogs - # botocore - # freezegun -python-json-logger==2.0.1 - # via - # -r requirements.txt - # notifications-utils -pytz==2020.5 - # via - # -r requirements.txt - # notifications-utils -pyyaml==5.3.1 - # via - # -r requirements.txt - # awscli - # jinja2-cli - # notifications-utils -redis==3.5.3 - # via - # -r requirements.txt - # flask-redis -requests-mock==1.7.0 - # via -r requirements_for_test.in -requests==2.25.1 - # via - # -r requirements.txt - # awscli-cwlogs - # govuk-bank-holidays - # notifications-python-client - # notifications-utils - # requests-mock -rsa==4.5 - # via - # -r requirements.txt - # awscli -s3transfer==0.3.3 - # via - # -r requirements.txt - # awscli - # boto3 -shapely==1.7.1 - # via - # -r requirements.txt - # notifications-utils -six==1.15.0 - # via - # -r requirements.txt - # awscli-cwlogs - # bleach - # eventlet - # flake8-print - # freezegun - # govuk-bank-holidays - # pytest-xdist - # python-dateutil - # requests-mock -smartypants==2.0.1 - # via - # -r requirements.txt - # notifications-utils -soupsieve==2.1 - # via beautifulsoup4 -statsd==3.3.0 - # via - # -r requirements.txt - # notifications-utils -texttable==1.6.3 - # via - # -r requirements.txt - # pyexcel -typing-extensions==3.7.4.3 - # via importlib-metadata -urllib3==1.26.2 - # via - # -r requirements.txt - # botocore - # requests -wcwidth==0.2.5 - # via pytest -webencodings==0.5.1 - # via - # -r requirements.txt - # bleach -werkzeug==1.0.1 - # via - # -r requirements.txt - # flask -wtforms==2.3.3 - # via - # -r requirements.txt - # flask-wtf -xlrd==1.2.0 - # via - # -r requirements.txt - # pyexcel-xls -xlwt==1.3.0 - # via - # -r requirements.txt - # pyexcel-xls -zipp==3.4.0 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +pytest-mock==3.5.1 +pytest-xdist==2.2.1 +beautifulsoup4==4.9.3 +freezegun==1.1.0 +flake8==3.8.4 +flake8-bugbear==20.11.1 +flake8-print==4.0.0 +requests-mock==1.8.0 +# used for creating manifest file locally +jinja2-cli[yaml]==0.7.0