From 9e8682ac29cdc01af3994614fa7088e42c5a2641 Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Wed, 7 Jul 2021 17:35:15 +0100 Subject: [PATCH 1/2] Bump utils to bring in fix for optional placeholder bug See https://github.com/alphagov/notifications-utils/pull/878 for details. Changes we had to make for our app and tests to work correctly after the dependency updates: 1. Update emergency alerts polygons test because we changed how exact we are with locations of the points on the map. 2. Use Flask's g object to set additional request attributes So far we have been storing them in _request_ctx_stack which is an innard for Flask's request context. Because of major update to Werkzeug dependency, which Flask relies on, the way we were using it stopped working, so we had a new way to set those values. The way we set those values now, by using g object, seems to also be favoured in Flask documentation: https://flask.palletsprojects.com/en/1.1.x/reqcontext/#how-the-context-works --- app/__init__.py | 5 ++-- app/authentication/auth.py | 6 ++-- requirements-app.txt | 2 +- requirements.txt | 28 +++++++++---------- tests/app/v2/broadcast/test_post_broadcast.py | 4 +-- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index c5a63bcf1..09e0d195d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -7,7 +7,6 @@ from time import monotonic from celery import current_task from flask import ( - _request_ctx_stack, current_app, g, has_request_context, @@ -69,8 +68,8 @@ metrics = GDSMetrics() notification_provider_clients = NotificationProviderClients() -api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user) -authenticated_service = LocalProxy(lambda: _request_ctx_stack.top.authenticated_service) +api_user = LocalProxy(lambda: g.api_user) +authenticated_service = LocalProxy(lambda: g.authenticated_service) CONCURRENT_REQUESTS = Gauge( 'concurrent_web_request_count', diff --git a/app/authentication/auth.py b/app/authentication/auth.py index b215d6b90..9db851e47 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -1,6 +1,6 @@ import uuid -from flask import _request_ctx_stack, current_app, g, request +from flask import current_app, g, request from gds_metrics import Histogram from notifications_python_client.authentication import ( decode_jwt_token, @@ -140,8 +140,8 @@ def requires_auth(): raise AuthError("Invalid token: API key revoked", 403, service_id=service.id, api_key_id=api_key.id) g.service_id = service.id - _request_ctx_stack.top.authenticated_service = service - _request_ctx_stack.top.api_user = api_key + g.api_user = api_key + g.authenticated_service = service current_app.logger.info('API authorised for service {} with api key {}, using issuer {} for URL: {}'.format( service.id, diff --git a/requirements-app.txt b/requirements-app.txt index ab701b28d..e0aa52214 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -34,7 +34,7 @@ notifications-python-client==6.0.2 # PaaS awscli-cwlogs==1.4.6 -git+https://github.com/alphagov/notifications-utils.git@44.1.0#egg=notifications-utils==44.1.0 +git+https://github.com/alphagov/notifications-utils.git@44.4.2#egg=notifications-utils==44.4.2 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.10.1 diff --git a/requirements.txt b/requirements.txt index 0b5a868d6..aaaac3d43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,28 +36,28 @@ notifications-python-client==6.0.2 # PaaS awscli-cwlogs==1.4.6 -git+https://github.com/alphagov/notifications-utils.git@44.1.0#egg=notifications-utils==44.1.0 +git+https://github.com/alphagov/notifications-utils.git@44.4.2#egg=notifications-utils==44.4.2 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.10.1 gds-metrics==0.2.4 ## The following requirements were added by pip freeze: -alembic==1.6.2 +alembic==1.6.5 amqp==1.4.9 anyjson==0.3.3 attrs==21.2.0 -awscli==1.19.69 +awscli==1.19.106 bcrypt==3.2.0 billiard==3.3.0.23 bleach==3.3.0 blinker==1.4 boto==2.49.0 -boto3==1.17.69 -botocore==1.20.69 -certifi==2020.12.5 +boto3==1.17.106 +botocore==1.20.106 +certifi==2021.5.30 chardet==4.0.0 -click==7.1.2 +click==8.0.1 colorama==0.4.3 dnspython==1.16.0 docutils==0.15.2 @@ -66,20 +66,20 @@ geojson==2.5.0 govuk-bank-holidays==0.8 greenlet==1.1.0 idna==2.10 -Jinja2==2.11.3 +Jinja2==3.0.1 jmespath==0.10.0 kombu==3.0.37 Mako==1.1.4 -MarkupSafe==1.1.1 +MarkupSafe==2.0.1 mistune==0.8.4 orderedset==2.0.3 -packaging==20.9 -phonenumbers==8.12.22 +packaging==21.0 +phonenumbers==8.12.26 pyasn1==0.4.8 pycparser==2.20 pyparsing==2.4.7 PyPDF2==1.26.0 -pyrsistent==0.17.3 +pyrsistent==0.18.0 python-dateutil==2.8.1 python-editor==1.0.4 python-json-logger==2.0.1 @@ -94,6 +94,6 @@ six==1.16.0 smartypants==2.0.1 soupsieve==2.2.1 statsd==3.3.0 -urllib3==1.26.4 +urllib3==1.26.6 webencodings==0.5.1 -Werkzeug==1.0.1 +Werkzeug==2.0.1 diff --git a/tests/app/v2/broadcast/test_post_broadcast.py b/tests/app/v2/broadcast/test_post_broadcast.py index 9edd65e69..fc824f946 100644 --- a/tests/app/v2/broadcast/test_post_broadcast.py +++ b/tests/app/v2/broadcast/test_post_broadcast.py @@ -104,8 +104,8 @@ def test_valid_post_cap_xml_broadcast_returns_201( assert response_json['service_id'] == str(sample_broadcast_service.id) assert len(response_json['simple_polygons']) == 1 assert len(response_json['simple_polygons'][0]) == 23 - assert response_json['simple_polygons'][0][0] == [53.10561946699971, 0.2441253049430708] - assert response_json['simple_polygons'][0][-1] == [53.10561946699971, 0.2441253049430708] + assert response_json['simple_polygons'][0][0] == [53.10562, 0.244127] + assert response_json['simple_polygons'][0][-1] == [53.10562, 0.244127] assert response_json['starts_at'] is None assert response_json['status'] == 'pending-approval' assert response_json['template_id'] is None From c28e9451d47db8bdb028f03b9767f7dfde5ac39c Mon Sep 17 00:00:00 2001 From: Pea Tyczynska Date: Thu, 8 Jul 2021 14:10:58 +0100 Subject: [PATCH 2/2] Bump moto version to try solve dependencies version conflict Also update mock import statements in some test files as they stopped working with this dependency update. --- requirements_for_test.txt | 2 +- tests/app/celery/test_scheduled_tasks.py | 2 +- tests/app/dao/test_fact_notification_status_dao.py | 2 +- tests/app/user/test_rest.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements_for_test.txt b/requirements_for_test.txt index c9862ea0a..36464b7b7 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -2,7 +2,7 @@ flake8==3.8.4 flake8-bugbear==20.11.1 isort==5.7.0 -moto==1.3.16 +moto==2.0.11 pytest==6.1.2 pytest-env==0.6.2 pytest-mock==3.3.1 diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 1d55d9e62..d72e6d544 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -1,10 +1,10 @@ from collections import namedtuple from datetime import datetime, timedelta +from unittest import mock from unittest.mock import call import pytest from freezegun import freeze_time -from mock import mock from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( diff --git a/tests/app/dao/test_fact_notification_status_dao.py b/tests/app/dao/test_fact_notification_status_dao.py index f807541a0..6fb2ae5e8 100644 --- a/tests/app/dao/test_fact_notification_status_dao.py +++ b/tests/app/dao/test_fact_notification_status_dao.py @@ -1,7 +1,7 @@ from datetime import date, datetime, time, timedelta +from unittest import mock from uuid import UUID -import mock import pytest from freezegun import freeze_time diff --git a/tests/app/user/test_rest.py b/tests/app/user/test_rest.py index 290a55782..96f6401a8 100644 --- a/tests/app/user/test_rest.py +++ b/tests/app/user/test_rest.py @@ -1,8 +1,8 @@ import json import uuid from datetime import datetime +from unittest import mock -import mock import pytest from flask import current_app, url_for from freezegun import freeze_time