From 20994c2d5d53af61415dac1e850ac2729f61e107 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 9 Dec 2020 13:00:03 +0000 Subject: [PATCH 01/13] Update cffi from 1.14.3 to 1.14.4 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 38a4f9cbf..231747809 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -1,7 +1,7 @@ # Run `make freeze-requirements` to update requirements.txt # with package version changes made in requirements-app.txt -cffi==1.14.3 +cffi==1.14.4 celery[sqs]==3.1.26.post2 # pyup: <4 docopt==0.6.2 Flask-Bcrypt==0.7.1 From e560b4a972c68359291c1f32f0c4fad0a8e1d4b0 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 9 Dec 2020 13:00:04 +0000 Subject: [PATCH 02/13] Update flask-marshmallow from 0.11.0 to 0.14.0 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 231747809..7e3fc969d 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -5,7 +5,7 @@ cffi==1.14.4 celery[sqs]==3.1.26.post2 # pyup: <4 docopt==0.6.2 Flask-Bcrypt==0.7.1 -flask-marshmallow==0.11.0 +flask-marshmallow==0.14.0 Flask-Migrate==2.5.3 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.1.2 From 39877e1e405cd148b3e4eb29c1b6c20d36b6ba59 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 9 Dec 2020 13:00:05 +0000 Subject: [PATCH 03/13] Update marshmallow-sqlalchemy from 0.23.1 to 0.24.1 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 7e3fc969d..fec9a1d68 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -15,7 +15,7 @@ gunicorn==20.0.4 iso8601==0.1.13 itsdangerous==1.1.0 jsonschema==3.2.0 -marshmallow-sqlalchemy==0.23.1 +marshmallow-sqlalchemy==0.24.1 marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 PyJWT==1.7.1 From 0c0821b9f9fd402c04dd90af5564ddc714a9722a Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 9 Dec 2020 13:00:06 +0000 Subject: [PATCH 04/13] Update prometheus-client from 0.8.0 to 0.9.0 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index fec9a1d68..4a7a7e5c7 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -32,5 +32,5 @@ awscli-cwlogs==1.4.6 git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains -prometheus-client==0.8.0 +prometheus-client==0.9.0 gds-metrics==0.2.4 From e4c56331503574f0b98bec8903aa1f2d64a07b73 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 16 Dec 2020 13:00:05 +0000 Subject: [PATCH 05/13] Update eventlet from 0.29.1 to 0.30.0 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 4a7a7e5c7..9f8029c42 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -10,7 +10,7 @@ Flask-Migrate==2.5.3 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.1.2 click-datetime==0.2 -eventlet==0.29.1 +eventlet==0.30.0 gunicorn==20.0.4 iso8601==0.1.13 itsdangerous==1.1.0 From 659a43e435ead9a56dfb92e0ad70f6398ef94f3d Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 16 Dec 2020 13:00:06 +0000 Subject: [PATCH 06/13] Update cachetools from 4.1.1 to 4.2.0 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 9f8029c42..59c2ebcc2 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -22,7 +22,7 @@ PyJWT==1.7.1 SQLAlchemy==1.3.20 strict-rfc3339==0.7 rfc3987==1.3.8 -cachetools==4.1.1 +cachetools==4.2.0 notifications-python-client==5.7.0 From 97d35b86b551609594a7632546146ca8e7eec7cc Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 23 Dec 2020 13:00:05 +0000 Subject: [PATCH 07/13] Update pyjwt from 1.7.1 to 2.0.0 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 59c2ebcc2..6739b5b79 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -18,7 +18,7 @@ jsonschema==3.2.0 marshmallow-sqlalchemy==0.24.1 marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 -PyJWT==1.7.1 +PyJWT==2.0.0 SQLAlchemy==1.3.20 strict-rfc3339==0.7 rfc3987==1.3.8 From b298440f002f8f5375e7395b755891e6f4fee6fa Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Wed, 23 Dec 2020 13:00:07 +0000 Subject: [PATCH 08/13] Update sqlalchemy from 1.3.20 to 1.3.22 --- requirements-app.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-app.txt b/requirements-app.txt index 6739b5b79..cbf3d1f30 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -19,7 +19,7 @@ marshmallow-sqlalchemy==0.24.1 marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 PyJWT==2.0.0 -SQLAlchemy==1.3.20 +SQLAlchemy==1.3.22 strict-rfc3339==0.7 rfc3987==1.3.8 cachetools==4.2.0 From 1da16eda23b026126236f8ea146884619a557553 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Wed, 23 Dec 2020 13:32:21 +0000 Subject: [PATCH 09/13] freeze reqs --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 794dc2d75..768b56386 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ Flask-Migrate==2.5.3 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.1.2 click-datetime==0.2 -eventlet==0.29.1 +eventlet==0.30.0 gunicorn==20.0.4 iso8601==0.1.13 itsdangerous==1.1.0 @@ -20,11 +20,11 @@ jsonschema==3.2.0 marshmallow-sqlalchemy==0.23.1 marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 -PyJWT==1.7.1 -SQLAlchemy==1.3.20 +PyJWT==2.0.0 +SQLAlchemy==1.3.22 strict-rfc3339==0.7 rfc3987==1.3.8 -cachetools==4.1.1 +cachetools==4.2.0 notifications-python-client==5.7.0 From 156c7aa32a59bdc5441ff07cfd875a7ed7ce44e3 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 31 Dec 2020 12:47:03 +0000 Subject: [PATCH 10/13] bump python client brings in jwt2.0 compat --- requirements-app.txt | 2 +- requirements.txt | 10 +++++----- tests/app/authentication/test_authentication.py | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements-app.txt b/requirements-app.txt index cbf3d1f30..e07fa9d0a 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -24,7 +24,7 @@ strict-rfc3339==0.7 rfc3987==1.3.8 cachetools==4.2.0 -notifications-python-client==5.7.0 +notifications-python-client==5.7.1 # PaaS awscli-cwlogs==1.4.6 diff --git a/requirements.txt b/requirements.txt index 768b56386..d28aea34c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,7 +26,7 @@ strict-rfc3339==0.7 rfc3987==1.3.8 cachetools==4.2.0 -notifications-python-client==5.7.0 +notifications-python-client==5.7.1 # PaaS awscli-cwlogs==1.4.6 @@ -42,14 +42,14 @@ alembic==1.4.3 amqp==1.4.9 anyjson==0.3.3 attrs==20.3.0 -awscli==1.18.199 +awscli==1.18.206 bcrypt==3.2.0 billiard==3.3.0.23 bleach==3.2.1 blinker==1.4 boto==2.49.0 -boto3==1.16.39 -botocore==1.19.39 +boto3==1.16.46 +botocore==1.19.46 certifi==2020.12.5 chardet==4.0.0 click==7.1.2 @@ -81,7 +81,7 @@ pyrsistent==0.17.3 python-dateutil==2.8.1 python-editor==1.0.4 python-json-logger==2.0.1 -pytz==2020.4 +pytz==2020.5 PyYAML==5.3.1 redis==3.5.3 requests==2.25.1 diff --git a/tests/app/authentication/test_authentication.py b/tests/app/authentication/test_authentication.py index e799bb619..b40f8b616 100644 --- a/tests/app/authentication/test_authentication.py +++ b/tests/app/authentication/test_authentication.py @@ -63,7 +63,7 @@ def test_should_not_allow_request_with_no_iss(client, auth_fn): 'iat': int(time.time()) } - token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers).decode() + token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: @@ -84,7 +84,7 @@ def test_auth_should_not_allow_request_with_no_iat(client, sample_api_key): # 'iat': not provided } - token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers).decode() + token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: @@ -105,7 +105,7 @@ def test_auth_should_not_allow_request_with_non_hs256_algorithm(client, sample_a 'iat': int(time.time()) } - token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers).decode() + token = jwt.encode(payload=claims, key=str(uuid.uuid4()), headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: @@ -128,7 +128,7 @@ def test_admin_auth_should_not_allow_request_with_no_iat(client): # 'iat': not provided } - token = jwt.encode(payload=claims, key=secret, headers=headers).decode() + token = jwt.encode(payload=claims, key=secret, headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: @@ -151,7 +151,7 @@ def test_admin_auth_should_not_allow_request_with_old_iat(client): 'iat': int(time.time()) - 60 } - token = jwt.encode(payload=claims, key=secret, headers=headers).decode() + token = jwt.encode(payload=claims, key=secret, headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: @@ -174,7 +174,7 @@ def test_auth_should_not_allow_request_with_extra_claims(client, sample_api_key) 'aud': 'notifications.service.gov.uk' # extra claim that we don't support } - token = jwt.encode(payload=claims, key=key, headers=headers).decode() + token = jwt.encode(payload=claims, key=key, headers=headers) request.headers = {'Authorization': 'Bearer {}'.format(token)} with pytest.raises(AuthError) as exc: From ee2bec2f72788b71cfb9ef6bb9befc84326a25ad Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 14 Dec 2020 11:40:07 +0000 Subject: [PATCH 11/13] pin marshmallow-sqlalchemy to keep marshmallow <=3.0 dep --- requirements-app.txt | 2 +- requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements-app.txt b/requirements-app.txt index e07fa9d0a..a71754683 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -15,7 +15,7 @@ gunicorn==20.0.4 iso8601==0.1.13 itsdangerous==1.1.0 jsonschema==3.2.0 -marshmallow-sqlalchemy==0.24.1 +marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.1 # marshmallow v3 throws errors marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 PyJWT==2.0.0 diff --git a/requirements.txt b/requirements.txt index d28aea34c..3e1cefe8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,11 +3,11 @@ # Run `make freeze-requirements` to update requirements.txt # with package version changes made in requirements-app.txt -cffi==1.14.3 +cffi==1.14.4 celery[sqs]==3.1.26.post2 # pyup: <4 docopt==0.6.2 Flask-Bcrypt==0.7.1 -flask-marshmallow==0.11.0 +flask-marshmallow==0.14.0 Flask-Migrate==2.5.3 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108 Flask==1.1.2 @@ -17,7 +17,7 @@ gunicorn==20.0.4 iso8601==0.1.13 itsdangerous==1.1.0 jsonschema==3.2.0 -marshmallow-sqlalchemy==0.23.1 +marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.1 # marshmallow v3 throws errors marshmallow==2.21.0 # pyup: <3 # v3 throws errors psycopg2-binary==2.8.6 PyJWT==2.0.0 @@ -34,7 +34,7 @@ awscli-cwlogs==1.4.6 git+https://github.com/alphagov/notifications-utils.git@43.5.9#egg=notifications-utils==43.5.9 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains -prometheus-client==0.8.0 +prometheus-client==0.9.0 gds-metrics==0.2.4 ## The following requirements were added by pip freeze: From a33ec5c7f1296ad95797a29d6f159d8d095a1f8f Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Mon, 14 Dec 2020 15:19:36 +0000 Subject: [PATCH 12/13] remove deprecated ModelSchema class --- app/schemas.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/schemas.py b/app/schemas.py index 192428f4e..6a0f29e57 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -74,7 +74,7 @@ class UUIDsAsStringsMixin: ] -class BaseSchema(ma.ModelSchema): +class BaseSchema(ma.SQLAlchemyAutoSchema): def __init__(self, load_json=False, *args, **kwargs): self.load_json = load_json From 4814c66c1d96a13570d44d363aa7003bdbc6b425 Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Thu, 31 Dec 2020 14:04:38 +0000 Subject: [PATCH 13/13] fix schema metaclasses marshmallow v0.22.0 added load_instance and include_relationship options, which we need to keep old ModelSchema code working --- app/schemas.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/app/schemas.py b/app/schemas.py index 6a0f29e57..1cebafbbc 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -76,6 +76,10 @@ class UUIDsAsStringsMixin: class BaseSchema(ma.SQLAlchemyAutoSchema): + class Meta: + load_instance = True + include_relationships = True + def __init__(self, load_json=False, *args, **kwargs): self.load_json = load_json super(BaseSchema, self).__init__(*args, **kwargs) @@ -108,7 +112,7 @@ class UserSchema(BaseSchema): retval[service_id].append(x.permission) return retval - class Meta: + class Meta(BaseSchema.Meta): model = models.User exclude = ( "updated_at", @@ -145,7 +149,7 @@ class UserSchema(BaseSchema): class UserUpdateAttributeSchema(BaseSchema): auth_type = field_for(models.User, 'auth_type') - class Meta: + class Meta(BaseSchema.Meta): model = models.User exclude = ( 'id', 'updated_at', 'created_at', 'user_to_service', @@ -182,7 +186,7 @@ class UserUpdateAttributeSchema(BaseSchema): class UserUpdatePasswordSchema(BaseSchema): - class Meta: + class Meta(BaseSchema.Meta): model = models.User only = ('password') strict = True @@ -197,7 +201,7 @@ class UserUpdatePasswordSchema(BaseSchema): class ProviderDetailsSchema(BaseSchema): created_by = fields.Nested(UserSchema, only=['id', 'name', 'email_address'], dump_only=True) - class Meta: + class Meta(BaseSchema.Meta): model = models.ProviderDetails exclude = ("provider_rates", "provider_stats") strict = True @@ -206,7 +210,7 @@ class ProviderDetailsSchema(BaseSchema): class ProviderDetailsHistorySchema(BaseSchema): created_by = fields.Nested(UserSchema, only=['id', 'name', 'email_address'], dump_only=True) - class Meta: + class Meta(BaseSchema.Meta): model = models.ProviderDetailsHistory exclude = ("provider_rates", "provider_stats") strict = True @@ -236,7 +240,7 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin): def get_letter_contact(self, service): return service.get_default_letter_contact() - class Meta: + class Meta(BaseSchema.Meta): model = models.Service exclude = ( 'updated_at', @@ -296,7 +300,7 @@ class DetailedServiceSchema(BaseSchema): statistics = fields.Dict() organisation_type = field_for(models.Service, 'organisation_type') - class Meta: + class Meta(BaseSchema.Meta): model = models.Service exclude = ( 'api_keys', @@ -337,7 +341,7 @@ class DetailedServiceSchema(BaseSchema): class NotificationModelSchema(BaseSchema): - class Meta: + class Meta(BaseSchema.Meta): model = models.Notification strict = True exclude = ('_personalisation', 'job', 'service', 'template', 'api_key',) @@ -355,7 +359,7 @@ class BaseTemplateSchema(BaseSchema): def get_reply_to_text(self, template): return template.get_reply_to_text() - class Meta: + class Meta(BaseSchema.Meta): model = models.Template exclude = ("service_id", "jobs", "service_letter_contact_id", "broadcast_messages") strict = True @@ -421,7 +425,7 @@ class TemplateHistorySchema(BaseSchema): def get_reply_to_text(self, template): return template.get_reply_to_text() - class Meta: + class Meta(BaseSchema.Meta): model = models.TemplateHistory @@ -430,7 +434,7 @@ class ApiKeySchema(BaseSchema): created_by = field_for(models.ApiKey, 'created_by', required=True) key_type = field_for(models.ApiKey, 'key_type', required=True) - class Meta: + class Meta(BaseSchema.Meta): model = models.ApiKey exclude = ("service", "_secret") strict = True @@ -462,7 +466,7 @@ class JobSchema(BaseSchema): _validate_datetime_not_in_past(value) _validate_datetime_not_more_than_96_hours_in_future(value) - class Meta: + class Meta(BaseSchema.Meta): model = models.Job exclude = ( 'notifications', @@ -515,7 +519,7 @@ class SmsTemplateNotificationSchema(SmsNotificationSchema): class NotificationWithTemplateSchema(BaseSchema): - class Meta: + class Meta(BaseSchema.Meta): model = models.Notification strict = True exclude = ('_personalisation', 'scheduled_notification') @@ -594,7 +598,7 @@ class NotificationWithPersonalisationSchema(NotificationWithTemplateSchema): class InvitedUserSchema(BaseSchema): auth_type = field_for(models.InvitedUser, 'auth_type') - class Meta: + class Meta(BaseSchema.Meta): model = models.InvitedUser strict = True @@ -697,7 +701,7 @@ class ApiKeyHistorySchema(ma.Schema): class EventSchema(BaseSchema): - class Meta: + class Meta(BaseSchema.Meta): model = models.Event strict = True