From b0f819dbd9e6e8390b7cd572417dff71b81769e3 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Thu, 15 Sep 2022 14:59:13 -0700 Subject: [PATCH 01/65] canada UK ses callbacks monster mash --- app/__init__.py | 5 + app/celery/process_ses_receipts_tasks.py | 222 +++++++++++++++--- app/dao/notifications_dao.py | 27 ++- app/delivery/send_to_providers.py | 5 +- app/models.py | 3 + app/notifications/callbacks.py | 52 ++++ .../notifications_ses_callback.py | 113 ++++++--- .../versions/0376_add_provider_response.py | 30 +++ requirements.in | 1 + requirements.txt | 7 + .../celery/test_process_ses_receipts_tasks.py | 210 ++++++++++++----- tests/app/conftest.py | 91 +++++++ .../notifications/test_get_notifications.py | 2 + 13 files changed, 638 insertions(+), 130 deletions(-) create mode 100644 app/notifications/callbacks.py create mode 100644 migrations/versions/0376_add_provider_response.py diff --git a/app/__init__.py b/app/__init__.py index 2fffe2673..035ce112a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -139,6 +139,7 @@ def register_blueprint(application): ) from app.billing.rest import billing_blueprint from app.broadcast_message.rest import broadcast_message_blueprint + from app.celery.process_ses_receipts_tasks import ses_callback_blueprint from app.complaint.complaint_rest import complaint_blueprint from app.email_branding.rest import email_branding_blueprint from app.events.rest import events as events_blueprint @@ -196,6 +197,10 @@ def register_blueprint(application): status_blueprint.before_request(requires_no_auth) application.register_blueprint(status_blueprint) + + # delivery receipts + ses_callback_blueprint.before_request(requires_no_auth) + application.register_blueprint(ses_callback_blueprint) # delivery receipts # TODO: make sure research mode can still trigger sms callbacks, then re-enable this diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 04a8d14f1..bfa21ad89 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -1,80 +1,159 @@ +import enum from datetime import datetime, timedelta +from json import decoder import iso8601 +import requests +import validatesns from celery.exceptions import Retry -from flask import current_app, json +from flask import Blueprint, current_app, json, jsonify, request +from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, statsd_client -from app.clients.email.aws_ses import get_aws_responses +from app import notify_celery, redis_store, statsd_client from app.config import QueueNames from app.dao import notifications_dao +from app.errors import InvalidRequest, register_errors from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING from app.notifications.notifications_ses_callback import ( _check_and_queue_complaint_callback_task, + _determine_notification_bounce_type, check_and_queue_callback_task, - determine_notification_bounce_type, + get_aws_responses, handle_complaint, ) +ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) + +register_errors(ses_callback_blueprint) + +class SNSMessageType(enum.Enum): + SubscriptionConfirmation = 'SubscriptionConfirmation' + Notification = 'Notification' + UnsubscribeConfirmation = 'UnsubscribeConfirmation' + + +class InvalidMessageTypeException(Exception): + pass + + +def verify_message_type(message_type: str): + try: + SNSMessageType(message_type) + except ValueError: + raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') + + +def get_certificate(url): + res = redis_store.get(url) + if res is not None: + return res + res = requests.get(url).content + redis_store.set(url, res, ex=60 * 60) # 60 minutes + return res + +# 400 counts as a permanent failure so SNS will not retry. +# 500 counts as a failed delivery attempt so SNS will retry. +# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies +# This should not be here, it used to be in notifications/notifications_ses_callback. It then +# got refactored into a task, which is fine, but it created a circular dependency. Will need +# to investigate why GDS extracted this into a lambda +@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) +def sns_callback_handler(): + message_type = request.headers.get('x-amz-sns-message-type') + try: + verify_message_type(message_type) + except InvalidMessageTypeException: + raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + + try: + message = json.loads(request.data) + except decoder.JSONDecodeError: + raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) + + try: + validatesns.validate(message, get_certificate=get_certificate) + except validatesns.ValidationError: + raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + + if message.get('Type') == 'SubscriptionConfirmation': + url = message.get('SubscribeURL') + response = requests.get(url) + try: + response.raise_for_status() + except Exception as e: + current_app.logger.warning("Response: {}".format(response.text)) + raise e + + return jsonify( + result="success", message="SES-SNS auto-confirm callback succeeded" + ), 200 + + process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY) + + return jsonify( + result="success", message="SES-SNS callback succeeded" + ), 200 + @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) +@statsd(namespace="tasks") def process_ses_results(self, response): try: - ses_message = json.loads(response['Message']) - notification_type = ses_message['notificationType'] - bounce_message = None - - if notification_type == 'Bounce': - notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message) - elif notification_type == 'Complaint': + ses_message = json.loads(response["Message"]) + notification_type = ses_message["notificationType"] + print(f"ses_message is: {ses_message}") + if notification_type == "Complaint": _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True - aws_response_dict = get_aws_responses(notification_type) + aws_response_dict = get_aws_responses(ses_message) - notification_status = aws_response_dict['notification_status'] - reference = ses_message['mail']['messageId'] + notification_status = aws_response_dict["notification_status"] + reference = ses_message["mail"]["messageId"] + + print(f"notification_status is: {notification_status}") try: - notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference) + notification = notifications_dao.dao_get_notification_by_reference(reference) except NoResultFound: - message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) + message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): - current_app.logger.info( - f"notification not found for reference: {reference} (update to {notification_status}). " - f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue" - ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - f"notification not found for reference: {reference} (update to {notification_status})" + "notification not found for reference: {} (update to {})".format(reference, notification_status) ) return - if bounce_message: - current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}") - - if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]: + if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: notifications_dao._duplicate_update_warning( - notification=notification, - status=notification_status + notification, + notification_status ) return + + notifications_dao._update_notification_status( + notification=notification, + status=notification_status, + provider_response=aws_response_dict["provider_response"], + ) + + if not aws_response_dict["success"]: + current_app.logger.info( + "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( + notification.id, reference, aws_response_dict["message"] + ) + ) else: - notifications_dao.dao_update_notifications_by_reference( - references=[reference], - update_dict={'status': notification_status} + current_app.logger.info( + "SES callback return status of {} for notification: {}".format(notification_status, notification.id) ) - statsd_client.incr('callback.ses.{}'.format(notification_status)) + statsd_client.incr("callback.ses.{}".format(notification_status)) if notification.sent_at: - statsd_client.timing_with_dates( - f'callback.ses.{notification_status}.elapsed-time', - datetime.utcnow(), - notification.sent_at - ) + statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at) check_and_queue_callback_task(notification) @@ -84,5 +163,74 @@ def process_ses_results(self, response): raise except Exception as e: - current_app.logger.exception('Error processing SES results: {}'.format(type(e))) + current_app.logger.exception("Error processing SES results: {}".format(type(e))) self.retry(queue=QueueNames.RETRY) + +# def process_ses_results(self, response): +# try: +# ses_message = json.loads(response['Message']) +# print(f"ses_message is {ses_message}") +# notification_type = ses_message['notificationType'] +# print(f"notification_type is {notification_type}") +# if notification_type == 'Bounce': +# notification_type = _determine_notification_bounce_type(ses_message) +# elif notification_type == 'Complaint': +# _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) +# return True +# aws_response_dict = get_aws_responses(notification_type) +# print(f"aws_response_dict is {aws_response_dict}") +# notification_status = aws_response_dict['notification_status'] +# print(f"notification_status is {notification_status}") +# reference = ses_message['mail']['messageId'] +# try: +# notification = notifications_dao.dao_get_notification_by_reference(reference) +# print(f"notification is {notification}") +# except NoResultFound: +# print(f"notification not found") +# message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) +# if datetime.utcnow() - message_time < timedelta(minutes=5): +# self.retry(queue=QueueNames.RETRY) +# else: +# current_app.logger.warning( +# "notification not found for reference: {} (update to {})".format(reference, notification_status) +# ) +# return +# print(f"notification.status is {notification.status}") +# if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: +# print(f"notification.status is not in [{NOTIFICATION_SENDING}, {NOTIFICATION_PENDING}]") +# notifications_dao._duplicate_update_warning(notification, notification_status) +# return +# notifications_dao._update_notification_status( +# notification=notification, +# status=notification_status, +# provider_response=None +# ) +# if not aws_response_dict['success']: +# current_app.logger.info( +# "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( +# notification.id, reference, aws_response_dict['message'] +# ) +# ) +# print( +# "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( +# notification.id, reference, aws_response_dict['message'] +# ) +# ) +# else: +# current_app.logger.info('SES callback return status of {} for notification: {}'.format( +# notification_status, notification.id +# )) +# print('SES callback return status of {} for notification: {}'.format( +# notification_status, notification.id +# )) +# statsd_client.incr('callback.ses.{}'.format(notification_status)) +# if notification.sent_at: +# statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at) +# check_and_queue_callback_task(notification) +# return True +# except Retry: +# raise +# except Exception as e: +# current_app.logger.exception('Error processing SES results: {}'.format(type(e))) +# self.retry(queue=QueueNames.RETRY) + \ No newline at end of file diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 4942de527..e5698a3b5 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -87,16 +87,21 @@ def country_records_delivery(phone_prefix): dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr'] return dlr and dlr.lower() == 'yes' +def _decide_permanent_temporary_failure(current_status, status): + # If we go from pending to delivered we need to set failure type as temporary-failure + if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE: + status = NOTIFICATION_TEMPORARY_FAILURE + return status -def _update_notification_status(notification, status, detailed_status_code=None): - # status = _decide_permanent_temporary_failure( - # status=status, notification=notification, detailed_status_code=detailed_status_code - # ) - # notification.status = status - # dao_update_notification(notification) + +def _update_notification_status(notification, status, provider_response=None): + status = _decide_permanent_temporary_failure(current_status=notification.status, status=status) + notification.status = status + if provider_response: + notification.provider_response = provider_response + dao_update_notification(notification) return notification - @autocommit def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None): notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first() @@ -599,6 +604,14 @@ def dao_get_notification_or_history_by_reference(reference): NotificationHistory.reference == reference ).one() +def dao_get_notification_history_by_reference(reference): + try: + # This try except is necessary because in test keys and research mode does not create notification history. + # Otherwise we could just search for the NotificationHistory object + return Notification.query.filter(Notification.reference == reference).one() + except NoResultFound: + return NotificationHistory.query.filter(NotificationHistory.reference == reference).one() + def dao_get_notifications_processing_time_stats(start_date, end_date): """ diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index d546cbc0c..48c634e76 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -163,7 +163,10 @@ def update_notification_to_sending(notification, provider): notification.sent_at = datetime.utcnow() notification.sent_by = provider.name if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED: - notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING + # We currently have no callback method for SNS + # TODO create celery task to request delivery receipts from cloudwatch api + notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING + dao_update_notification(notification) diff --git a/app/models.py b/app/models.py index ff6a8a8d9..a69188531 100644 --- a/app/models.py +++ b/app/models.py @@ -1505,6 +1505,8 @@ class Notification(db.Model): document_download_count = db.Column(db.Integer, nullable=True) postage = db.Column(db.String, nullable=True) + provider_response = db.Column(db.Text, nullable=True) + # queue_name = db.Column(db.Text, nullable=True) __table_args__ = ( db.ForeignKeyConstraint( @@ -1707,6 +1709,7 @@ class Notification(db.Model): "postcode": None, "type": self.notification_type, "status": self.get_letter_status() if self.notification_type == LETTER_TYPE else self.status, + "provider_response": self.provider_response, "template": template_dict, "body": self.content, "subject": self.subject, diff --git a/app/notifications/callbacks.py b/app/notifications/callbacks.py new file mode 100644 index 000000000..7c3f1eed6 --- /dev/null +++ b/app/notifications/callbacks.py @@ -0,0 +1,52 @@ +from app.celery.service_callback_tasks import send_delivery_status_to_service +from app.config import QueueNames +from app.dao.service_callback_api_dao import ( + get_service_delivery_status_callback_api_for_service, +) + + +def check_and_queue_callback_task(notification): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + notification_data = create_delivery_status_callback_data(notification, service_callback_api) + + send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) + + +def create_delivery_status_callback_data(notification, service_callback_api): + from app import encryption + from app.utils import DATETIME_FORMAT + + data = { + "notification_id": str(notification.id), + "notification_client_reference": notification.client_reference, + "notification_to": notification.to, + "notification_status": notification.status, + "notification_provider_response": notification.provider_response, + "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT), + "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None, + "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None, + "notification_type": notification.notification_type, + "service_callback_api_url": service_callback_api.url, + "service_callback_api_bearer_token": service_callback_api.bearer_token, + } + + return encryption.encrypt(data) + + +def create_complaint_callback_data(complaint, notification, service_callback_api, recipient): + from app import encryption + from app.utils import DATETIME_FORMAT + + data = { + "complaint_id": str(complaint.id), + "notification_id": str(notification.id), + "reference": notification.client_reference, + "to": recipient, + "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT), + "service_callback_api_url": service_callback_api.url, + "service_callback_api_bearer_token": service_callback_api.bearer_token, + } + + return encryption.encrypt(data) \ No newline at end of file diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index c1fdceaa0..1de6aeed6 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -1,4 +1,4 @@ -from flask import current_app +from flask import current_app, json from app.celery.service_callback_tasks import ( create_complaint_callback_data, @@ -8,65 +8,125 @@ from app.celery.service_callback_tasks import ( ) from app.config import QueueNames from app.dao.complaint_dao import save_complaint -from app.dao.notifications_dao import ( - dao_get_notification_or_history_by_reference, -) +from app.dao.notifications_dao import dao_get_notification_history_by_reference from app.dao.service_callback_api_dao import ( get_service_complaint_callback_api_for_service, get_service_delivery_status_callback_api_for_service, ) from app.models import Complaint +from app.notifications.callbacks import create_complaint_callback_data -def determine_notification_bounce_type(notification_type, ses_message): +def _determine_notification_bounce_type(ses_message): + notification_type = ses_message["notificationType"] + if notification_type in ["Delivery", "Complaint"]: + return notification_type + + if notification_type != "Bounce": + raise KeyError(f"Unhandled notification type {notification_type}") + remove_emails_from_bounce(ses_message) - if ses_message['bounce']['bounceType'] == 'Permanent': - notification_type = ses_message['bounce']['bounceType'] # permanent or not - else: - notification_type = 'Temporary' - return notification_type, ses_message + current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + if ses_message["bounce"]["bounceType"] == "Permanent": + return "Permanent" + return "Temporary" + + +def _determine_provider_response(ses_message): + if ses_message["notificationType"] != "Bounce": + return None + + bounce_type = ses_message["bounce"]["bounceType"] + bounce_subtype = ses_message["bounce"]["bounceSubType"] + + # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html + if bounce_type == "Permanent" and bounce_subtype == "Suppressed": + return "The email address is on our email provider suppression list" + elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList": + return "The email address is on the GC Notify suppression list" + elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected": + return "The email was rejected because of its attachments" + + return None + + +def get_aws_responses(ses_message): + status = _determine_notification_bounce_type(ses_message) + + base = { + "Permanent": { + "message": "Hard bounced", + "success": False, + "notification_status": "permanent-failure", + }, + "Temporary": { + "message": "Soft bounced", + "success": False, + "notification_status": "temporary-failure", + }, + "Delivery": { + "message": "Delivered", + "success": True, + "notification_status": "delivered", + }, + "Complaint": { + "message": "Complaint", + "success": True, + "notification_status": "delivered", + }, + }[status] + + base["provider_response"] = _determine_provider_response(ses_message) + + return base def handle_complaint(ses_message): recipient_email = remove_emails_from_complaint(ses_message)[0] - current_app.logger.info("Complaint from SES: \n{}".format(ses_message)) + current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) try: - reference = ses_message['mail']['messageId'] + reference = ses_message["mail"]["messageId"] except KeyError as e: current_app.logger.exception("Complaint from SES failed to get reference from message", e) return - notification = dao_get_notification_or_history_by_reference(reference) - ses_complaint = ses_message.get('complaint', None) + notification = dao_get_notification_history_by_reference(reference) + ses_complaint = ses_message.get("complaint", None) complaint = Complaint( notification_id=notification.id, service_id=notification.service_id, - ses_feedback_id=ses_complaint.get('feedbackId', None) if ses_complaint else None, - complaint_type=ses_complaint.get('complaintFeedbackType', None) if ses_complaint else None, - complaint_date=ses_complaint.get('timestamp', None) if ses_complaint else None + ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None, + complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None, + complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None, ) save_complaint(complaint) return complaint, notification, recipient_email def remove_mail_headers(dict_to_edit): - if dict_to_edit['mail'].get('headers'): - dict_to_edit['mail'].pop('headers') - if dict_to_edit['mail'].get('commonHeaders'): - dict_to_edit['mail'].pop('commonHeaders') + if dict_to_edit["mail"].get("headers"): + dict_to_edit["mail"].pop("headers") + if dict_to_edit["mail"].get("commonHeaders"): + dict_to_edit["mail"].pop("commonHeaders") def remove_emails_from_bounce(bounce_dict): remove_mail_headers(bounce_dict) - bounce_dict['mail'].pop('destination') - bounce_dict['bounce'].pop('bouncedRecipients') + bounce_dict["mail"].pop("destination") + bounce_dict["bounce"].pop("bouncedRecipients") def remove_emails_from_complaint(complaint_dict): remove_mail_headers(complaint_dict) - complaint_dict['complaint'].pop('complainedRecipients') - return complaint_dict['mail'].pop('destination') + complaint_dict["complaint"].pop("complainedRecipients") + return complaint_dict["mail"].pop("destination") +def check_and_queue_callback_task(notification): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + notification_data = create_delivery_status_callback_data(notification, service_callback_api) + send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) def check_and_queue_callback_task(notification): # queue callback task only if the service_callback_api exists @@ -76,10 +136,9 @@ def check_and_queue_callback_task(notification): send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) - def _check_and_queue_complaint_callback_task(complaint, notification, recipient): # queue callback task only if the service_callback_api exists service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) if service_callback_api: complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) - send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) + send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) \ No newline at end of file diff --git a/migrations/versions/0376_add_provider_response.py b/migrations/versions/0376_add_provider_response.py new file mode 100644 index 000000000..20dc83273 --- /dev/null +++ b/migrations/versions/0376_add_provider_response.py @@ -0,0 +1,30 @@ +"""empty message + +Revision ID: 0376_add_provider_response +Revises: 0375_fix_service_name +Create Date: 2022-09-14 11:04:15.888017 + +""" +# revision identifiers, used by Alembic. +from datetime import datetime + +revision = '0376_add_provider_response' +down_revision = '0375_fix_service_name' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column('notifications', sa.Column('provider_response', sa.Text(), nullable=True)) + op.add_column('notifications', sa.Column('queue_name', sa.Text(), nullable=True)) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_column('notifications', 'provider_response') + op.drop_column('notifications', 'queue_name') + ### end Alembic commands ### + \ No newline at end of file diff --git a/requirements.in b/requirements.in index de8a98254..fec2b41c6 100644 --- a/requirements.in +++ b/requirements.in @@ -19,6 +19,7 @@ marshmallow==3.15.0 psycopg2-binary==2.9.3 PyJWT==2.4.0 SQLAlchemy==1.4.40 +validatesns==0.1.1 cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.9.1 diff --git a/requirements.txt b/requirements.txt index 154c1c908..2cdc474ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration +asn1crypto==1.5.1 + # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -169,6 +171,8 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils +oscrypto==1.3.0 + # via validatesns packaging==21.3 # via # bleach @@ -247,6 +251,7 @@ six==1.16.0 # flask-marshmallow # python-dateutil # rfc3339-validator + # validatesns smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 @@ -267,6 +272,8 @@ urllib3==1.26.9 # via # botocore # requests +validatesns==0.1.1 + # via -r requirements.in vine==5.0.0 # via # amqp diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 4cb614ae8..51fda72aa 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -10,12 +10,16 @@ from app.celery.research_mode_tasks import ( ses_notification_callback, ses_soft_bounce_callback, ) +from app.celery.service_callback_tasks import ( + create_delivery_status_callback_data, +) from app.dao.notifications_dao import get_notification_by_id from app.models import Complaint, Notification from app.notifications.notifications_ses_callback import ( remove_emails_from_bounce, remove_emails_from_complaint, ) +from tests.app.conftest import create_sample_notification from tests.app.db import ( create_notification, create_service_callback_api, @@ -23,16 +27,87 @@ from tests.app.db import ( ) +def test_notifications_ses_400_with_invalid_header(client): + data = json.dumps({"foo": "bar"}) + response = client.post( + path='/notifications/email/ses', + data=data, + headers=[('Content-Type', 'application/json')] + ) + assert response.status_code == 400 + + +def test_notifications_ses_400_with_invalid_message_type(client): + data = json.dumps({"foo": "bar"}) + response = client.post( + path='/notifications/email/ses', + data=data, + headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'foo')] + ) + assert response.status_code == 400 + assert "SES-SNS callback failed: invalid message type" in response.get_data(as_text=True) + + +def test_notifications_ses_400_with_invalid_json(client): + data = "FOOO" + response = client.post( + path='/notifications/email/ses', + data=data, + headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')] + ) + assert response.status_code == 400 + assert "SES-SNS callback failed: invalid JSON given" in response.get_data(as_text=True) + + +def test_notifications_ses_400_with_certificate(client): + data = json.dumps({"foo": "bar"}) + response = client.post( + path='/notifications/email/ses', + data=data, + headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')] + ) + assert response.status_code == 400 + assert "SES-SNS callback failed: validation failed" in response.get_data(as_text=True) + + +def test_notifications_ses_200_autoconfirms_subscription(client, mocker): + mocker.patch("validatesns.validate") + requests_mock = mocker.patch("requests.get") + data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"}) + response = client.post( + path='/notifications/email/ses', + data=data, + headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'SubscriptionConfirmation')] + ) + + requests_mock.assert_called_once_with("https://foo") + assert response.status_code == 200 + + +def test_notifications_ses_200_call_process_task(client, mocker): + mocker.patch("validatesns.validate") + process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async") + data = {"Type": "Notification", "foo": "bar"} + json_data = json.dumps(data) + response = client.post( + path='/notifications/email/ses', + data=json_data, + headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')] + ) + + process_mock.assert_called_once_with([{'Message': None}], queue='notify-internal-tasks') + assert response.status_code == 200 + + def test_process_ses_results(sample_email_template): create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending') assert process_ses_results(response=ses_notification_callback(reference='ref1')) -def test_process_ses_results_retry_called(sample_email_template, mocker): +def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker): create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending') - - mocker.patch("app.dao.notifications_dao.dao_update_notifications_by_reference", side_effect=Exception("EXPECTED")) + mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED")) mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') process_ses_results(response=ses_notification_callback(reference='ref1')) assert mocked.call_count != 0 @@ -62,6 +137,7 @@ def test_remove_email_from_bounce(): def test_ses_callback_should_update_notification_status( client, + _notify_db, notify_db_session, sample_email_template, mocker): @@ -69,140 +145,159 @@ def test_ses_callback_should_update_notification_status( mocker.patch('app.statsd_client.incr') mocker.patch('app.statsd_client.timing_with_dates') send_mock = mocker.patch( - 'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task' + 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' ) - notification = create_notification( + notification = create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref', + status='sending', + sent_at=datetime.utcnow() ) + callback_api = create_service_callback_api(service=sample_email_template.service, url="https://original_url.com") assert get_notification_by_id(notification.id).status == 'sending' - assert process_ses_results(ses_notification_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'delivered' statsd_client.timing_with_dates.assert_any_call( - "callback.ses.delivered.elapsed-time", datetime.utcnow(), notification.sent_at + "callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at ) statsd_client.incr.assert_any_call("callback.ses.delivered") updated_notification = Notification.query.get(notification.id) - send_mock.assert_called_once_with(updated_notification) + encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api) + send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks") def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker): mock_dup = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._duplicate_update_warning') - mock_upd = mocker.patch( - 'app.celery.process_ses_receipts_tasks.notifications_dao.dao_update_notifications_by_reference' - ) + mock_upd = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._update_notification_status') notification = create_notification(template=sample_email_template, reference='ref', status='delivered') - assert process_ses_results(ses_notification_callback(reference='ref')) is None assert get_notification_by_id(notification.id).status == 'delivered' - - mock_dup.assert_called_once_with(notification=notification, status='delivered') + mock_dup.assert_called_once_with(notification, 'delivered') assert mock_upd.call_count == 0 -def test_ses_callback_should_retry_if_notification_is_new(client, notify_db_session, mocker): +def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') - with freeze_time('2017-11-17T12:14:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_logger.call_count == 0 assert mock_retry.call_count == 1 - - -def test_ses_callback_should_log_if_notification_is_missing(client, notify_db_session, mocker): +def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning') - with freeze_time('2017-11-17T12:34:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 mock_logger.assert_called_once_with('notification not found for reference: ref (update to delivered)') - - -def test_ses_callback_should_not_retry_if_notification_is_old(client, notify_db_session, mocker): +def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') - with freeze_time('2017-11-21T12:14:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_logger.call_count == 0 assert mock_retry.call_count == 0 - - -def test_ses_callback_should_update_multiple_notification_status_sent( +def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry( client, + _notify_db, + notify_db_session, + sample_email_template, + mocker): + with freeze_time('2001-01-01T12:00:00'): + send_mock = mocker.patch( + 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' + ) + notification = create_sample_notification( + _notify_db, + notify_db_session, + template=sample_email_template, + reference='ref', + status='sending', + sent_at=datetime.utcnow() + ) + assert get_notification_by_id(notification.id).status == 'sending' + assert process_ses_results(ses_notification_callback(reference='ref')) + assert get_notification_by_id(notification.id).status == 'delivered' + send_mock.assert_not_called() +def test_ses_callback_should_update_multiple_notification_status_sent( + client, + _notify_db, notify_db_session, sample_email_template, mocker): - send_mock = mocker.patch( - 'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task' + 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' ) - create_notification( + create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref1', - ) - create_notification( + sent_at=datetime.utcnow(), + status='sending') + create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref2', - ) - create_notification( + sent_at=datetime.utcnow(), + status='sending') + create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref3', - ) + sent_at=datetime.utcnow(), + status='sending') + create_service_callback_api(service=sample_email_template.service, url="https://original_url.com") assert process_ses_results(ses_notification_callback(reference='ref1')) assert process_ses_results(ses_notification_callback(reference='ref2')) assert process_ses_results(ses_notification_callback(reference='ref3')) assert send_mock.called - - def test_ses_callback_should_set_status_to_temporary_failure(client, + _notify_db, notify_db_session, sample_email_template, mocker): send_mock = mocker.patch( - 'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task' + 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' ) - mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info') - notification = create_notification( + notification = create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref', + status='sending', + sent_at=datetime.utcnow() ) + create_service_callback_api(service=notification.service, url="https://original_url.com") assert get_notification_by_id(notification.id).status == 'sending' assert process_ses_results(ses_soft_bounce_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'temporary-failure' assert send_mock.called - assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0] - - def test_ses_callback_should_set_status_to_permanent_failure(client, + _notify_db, notify_db_session, sample_email_template, mocker): send_mock = mocker.patch( - 'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task' + 'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async' ) - mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info') - notification = create_notification( + notification = create_sample_notification( + _notify_db, + notify_db_session, template=sample_email_template, - status='sending', reference='ref', + status='sending', + sent_at=datetime.utcnow() ) - + create_service_callback_api(service=sample_email_template.service, url="https://original_url.com") assert get_notification_by_id(notification.id).status == 'sending' assert process_ses_results(ses_hard_bounce_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'permanent-failure' assert send_mock.called - assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0] - - def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker): send_mock = mocker.patch( 'app.celery.service_callback_tasks.send_complaint_to_service.apply_async' @@ -210,13 +305,11 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email create_service_callback_api( service=sample_email_template.service, url="https://original_url.com", callback_type="complaint" ) - notification = create_notification( template=sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending' ) response = ses_complaint_callback() assert process_ses_results(response) - assert send_mock.call_count == 1 assert encryption.decrypt(send_mock.call_args[0][0][0]) == { 'complaint_date': '2018-06-05T13:59:58.000000Z', @@ -227,3 +320,4 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email 'service_callback_api_url': 'https://original_url.com', 'to': 'recipient1@example.com' } + \ No newline at end of file diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 61b87ca8b..206a5a2d5 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -31,6 +31,7 @@ from app.models import ( KEY_TYPE_TEAM, KEY_TYPE_TEST, LETTER_TYPE, + NOTIFICATION_STATUS_TYPES_COMPLETED, SERVICE_PERMISSION_TYPES, SMS_TYPE, ApiKey, @@ -69,6 +70,96 @@ def rmock(): yield rmock +def create_sample_notification( + notify_db, + notify_db_session, + service=None, + template=None, + job=None, + job_row_number=None, + to_field=None, + status="created", + provider_response=None, + reference=None, + created_at=None, + sent_at=None, + billable_units=1, + personalisation=None, + api_key=None, + key_type=KEY_TYPE_NORMAL, + sent_by=None, + international=False, + client_reference=None, + rate_multiplier=1.0, + scheduled_for=None, + normalised_to=None, + postage=None, +): + if created_at is None: + created_at = datetime.utcnow() + if service is None: + service = create_service(check_if_service_exists=True) + if template is None: + template = create_template(service=service) + + if job is None and api_key is None: + # we didn't specify in test - lets create it + api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first() + if not api_key: + api_key = create_api_key(template.service, key_type=key_type) + + notification_id = uuid.uuid4() + + if to_field: + to = to_field + else: + to = "+16502532222" + + data = { + "id": notification_id, + "to": to, + "job_id": job.id if job else None, + "job": job, + "service_id": service.id, + "service": service, + "template_id": template.id, + "template_version": template.version, + "status": status, + "provider_response": provider_response, + "reference": reference, + "created_at": created_at, + "sent_at": sent_at, + "billable_units": billable_units, + "personalisation": personalisation, + "notification_type": template.template_type, + "api_key": api_key, + "api_key_id": api_key and api_key.id, + "key_type": api_key.key_type if api_key else key_type, + "sent_by": sent_by, + "updated_at": created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None, + "client_reference": client_reference, + "rate_multiplier": rate_multiplier, + "normalised_to": normalised_to, + "postage": postage, + } + if job_row_number is not None: + data["job_row_number"] = job_row_number + notification = Notification(**data) + dao_create_notification(notification) + # if scheduled_for: + # scheduled_notification = ScheduledNotification( + # id=uuid.uuid4(), + # notification_id=notification.id, + # scheduled_for=datetime.strptime(scheduled_for, "%Y-%m-%d %H:%M"), + # ) + # if status != "created": + # scheduled_notification.pending = False + # db.session.add(scheduled_notification) + # db.session.commit() + + return notification + + @pytest.fixture(scope='function') def service_factory(sample_user): class ServiceFactory(object): diff --git a/tests/app/v2/notifications/test_get_notifications.py b/tests/app/v2/notifications/test_get_notifications.py index 299b1d6e2..7c93f9aad 100644 --- a/tests/app/v2/notifications/test_get_notifications.py +++ b/tests/app/v2/notifications/test_get_notifications.py @@ -68,6 +68,7 @@ def test_get_notification_by_id_returns_200( 'completed_at': sample_notification.completed_at(), 'scheduled_for': None, 'postage': None, + 'provider_response': None } assert json_response == expected_response @@ -120,6 +121,7 @@ def test_get_notification_by_id_with_placeholders_returns_200( 'completed_at': sample_notification.completed_at(), 'scheduled_for': None, 'postage': None, + 'provider_response': None } assert json_response == expected_response From f1aec5466532a4a30866968e2dae602aed48bf8f Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Thu, 15 Sep 2022 15:48:37 -0700 Subject: [PATCH 02/65] clean up comments and method dupes --- app/celery/process_ses_receipts_tasks.py | 87 +++---------------- app/celery/tasks.py | 4 +- app/dao/notifications_dao.py | 12 --- app/delivery/send_to_providers.py | 4 +- .../notifications_ses_callback.py | 15 ++-- .../celery/test_process_ses_receipts_tasks.py | 2 +- tests/app/conftest.py | 10 --- .../notification_dao/test_notification_dao.py | 14 +-- 8 files changed, 30 insertions(+), 118 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index bfa21ad89..8f743fa95 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -97,13 +97,15 @@ def sns_callback_handler(): @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) -@statsd(namespace="tasks") def process_ses_results(self, response): try: ses_message = json.loads(response["Message"]) notification_type = ses_message["notificationType"] - print(f"ses_message is: {ses_message}") - if notification_type == "Complaint": + bounce_message = None + + if notification_type == 'Bounce': + bounce_message = _determine_notification_bounce_type(ses_message) + elif notification_type == 'Complaint': _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True @@ -111,21 +113,26 @@ def process_ses_results(self, response): notification_status = aws_response_dict["notification_status"] reference = ses_message["mail"]["messageId"] - - print(f"notification_status is: {notification_status}") try: notification = notifications_dao.dao_get_notification_by_reference(reference) except NoResultFound: message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): + current_app.logger.info( + f"notification not found for reference: {reference} (while attempting update to {notification_status}). " + f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue" + ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - "notification not found for reference: {} (update to {})".format(reference, notification_status) + "notification not found for reference: {} (while attempting update to {})".format(reference, notification_status) ) return + if bounce_message: + current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}") + if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: notifications_dao._duplicate_update_warning( notification, @@ -166,71 +173,3 @@ def process_ses_results(self, response): current_app.logger.exception("Error processing SES results: {}".format(type(e))) self.retry(queue=QueueNames.RETRY) -# def process_ses_results(self, response): -# try: -# ses_message = json.loads(response['Message']) -# print(f"ses_message is {ses_message}") -# notification_type = ses_message['notificationType'] -# print(f"notification_type is {notification_type}") -# if notification_type == 'Bounce': -# notification_type = _determine_notification_bounce_type(ses_message) -# elif notification_type == 'Complaint': -# _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) -# return True -# aws_response_dict = get_aws_responses(notification_type) -# print(f"aws_response_dict is {aws_response_dict}") -# notification_status = aws_response_dict['notification_status'] -# print(f"notification_status is {notification_status}") -# reference = ses_message['mail']['messageId'] -# try: -# notification = notifications_dao.dao_get_notification_by_reference(reference) -# print(f"notification is {notification}") -# except NoResultFound: -# print(f"notification not found") -# message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None) -# if datetime.utcnow() - message_time < timedelta(minutes=5): -# self.retry(queue=QueueNames.RETRY) -# else: -# current_app.logger.warning( -# "notification not found for reference: {} (update to {})".format(reference, notification_status) -# ) -# return -# print(f"notification.status is {notification.status}") -# if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: -# print(f"notification.status is not in [{NOTIFICATION_SENDING}, {NOTIFICATION_PENDING}]") -# notifications_dao._duplicate_update_warning(notification, notification_status) -# return -# notifications_dao._update_notification_status( -# notification=notification, -# status=notification_status, -# provider_response=None -# ) -# if not aws_response_dict['success']: -# current_app.logger.info( -# "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( -# notification.id, reference, aws_response_dict['message'] -# ) -# ) -# print( -# "SES delivery failed: notification id {} and reference {} has error found. Status {}".format( -# notification.id, reference, aws_response_dict['message'] -# ) -# ) -# else: -# current_app.logger.info('SES callback return status of {} for notification: {}'.format( -# notification_status, notification.id -# )) -# print('SES callback return status of {} for notification: {}'.format( -# notification_status, notification.id -# )) -# statsd_client.incr('callback.ses.{}'.format(notification_status)) -# if notification.sent_at: -# statsd_client.timing_with_dates('callback.ses.elapsed-time', datetime.utcnow(), notification.sent_at) -# check_and_queue_callback_task(notification) -# return True -# except Retry: -# raise -# except Exception as e: -# current_app.logger.exception('Error processing SES results: {}'.format(type(e))) -# self.retry(queue=QueueNames.RETRY) - \ No newline at end of file diff --git a/app/celery/tasks.py b/app/celery/tasks.py index d83bf7cc2..48c6f3b66 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -21,7 +21,7 @@ from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job from app.dao.notifications_dao import ( dao_get_last_notification_added_for_job_id, - dao_get_notification_or_history_by_reference, + dao_get_notification_history_by_reference, dao_update_notifications_by_reference, get_notification_by_id, update_notification_status_by_reference, @@ -547,7 +547,7 @@ def update_letter_notification(filename, temporary_failures, update): def check_billable_units(notification_update): - notification = dao_get_notification_or_history_by_reference(notification_update.reference) + notification = dao_get_notification_history_by_reference(notification_update.reference) if int(notification_update.page_count) != notification.billable_units: msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format( diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index e5698a3b5..efdbfaefb 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -592,18 +592,6 @@ def dao_get_notification_by_reference(reference): ).one() -def dao_get_notification_or_history_by_reference(reference): - try: - # This try except is necessary because in test keys and research mode does not create notification history. - # Otherwise we could just search for the NotificationHistory object - return Notification.query.filter( - Notification.reference == reference - ).one() - except NoResultFound: - return NotificationHistory.query.filter( - NotificationHistory.reference == reference - ).one() - def dao_get_notification_history_by_reference(reference): try: # This try except is necessary because in test keys and research mode does not create notification history. diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index 48c634e76..e3e8aa6b4 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -163,8 +163,8 @@ def update_notification_to_sending(notification, provider): notification.sent_at = datetime.utcnow() notification.sent_by = provider.name if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED: - # We currently have no callback method for SNS - # TODO create celery task to request delivery receipts from cloudwatch api + # We currently have no callback method for SMS deliveries + # TODO create celery task to request SMS delivery receipts from cloudwatch api notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING dao_update_notification(notification) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index 1de6aeed6..a56de6f5a 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -112,8 +112,8 @@ def remove_mail_headers(dict_to_edit): def remove_emails_from_bounce(bounce_dict): remove_mail_headers(bounce_dict) - bounce_dict["mail"].pop("destination") - bounce_dict["bounce"].pop("bouncedRecipients") + bounce_dict["mail"].pop("destination", None) + bounce_dict["bounce"].pop("bouncedRecipients", None) def remove_emails_from_complaint(complaint_dict): @@ -121,6 +121,7 @@ def remove_emails_from_complaint(complaint_dict): complaint_dict["complaint"].pop("complainedRecipients") return complaint_dict["mail"].pop("destination") + def check_and_queue_callback_task(notification): # queue callback task only if the service_callback_api exists service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) @@ -128,17 +129,11 @@ def check_and_queue_callback_task(notification): notification_data = create_delivery_status_callback_data(notification, service_callback_api) send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) -def check_and_queue_callback_task(notification): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - notification_data = create_delivery_status_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), notification_data], - queue=QueueNames.CALLBACKS) def _check_and_queue_complaint_callback_task(complaint, notification, recipient): # queue callback task only if the service_callback_api exists service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) if service_callback_api: complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) - send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) \ No newline at end of file + send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) + \ No newline at end of file diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 51fda72aa..934f76c79 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -191,7 +191,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, with freeze_time('2017-11-17T12:34:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 - mock_logger.assert_called_once_with('notification not found for reference: ref (update to delivered)') + mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)') def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 206a5a2d5..f68afb94f 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -146,16 +146,6 @@ def create_sample_notification( data["job_row_number"] = job_row_number notification = Notification(**data) dao_create_notification(notification) - # if scheduled_for: - # scheduled_notification = ScheduledNotification( - # id=uuid.uuid4(), - # notification_id=notification.id, - # scheduled_for=datetime.strptime(scheduled_for, "%Y-%m-%d %H:%M"), - # ) - # if status != "created": - # scheduled_notification.pending = False - # db.session.add(scheduled_notification) - # db.session.commit() return notification diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index b7a4430d4..b05c7f649 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -15,7 +15,7 @@ from app.dao.notifications_dao import ( dao_get_letters_to_be_printed, dao_get_notification_by_reference, dao_get_notification_count_for_job_id, - dao_get_notification_or_history_by_reference, + dao_get_notification_history_by_reference, dao_get_notifications_by_recipient_or_reference, dao_timeout_notifications, dao_update_notification, @@ -1607,28 +1607,28 @@ def test_dao_get_notification_by_reference_with_no_matches_raises_error(notify_d dao_get_notification_by_reference('REF1') -def test_dao_get_notification_or_history_by_reference_with_one_match_returns_notification( +def test_dao_get_notification_history_by_reference_with_one_match_returns_notification( sample_letter_template ): create_notification(template=sample_letter_template, reference='REF1') - notification = dao_get_notification_or_history_by_reference('REF1') + notification = dao_get_notification_history_by_reference('REF1') assert notification.reference == 'REF1' -def test_dao_get_notification_or_history_by_reference_with_multiple_matches_raises_error( +def test_dao_get_notification_history_by_reference_with_multiple_matches_raises_error( sample_letter_template ): create_notification(template=sample_letter_template, reference='REF1') create_notification(template=sample_letter_template, reference='REF1') with pytest.raises(SQLAlchemyError): - dao_get_notification_or_history_by_reference('REF1') + dao_get_notification_history_by_reference('REF1') -def test_dao_get_notification_or_history_by_reference_with_no_matches_raises_error(notify_db_session): +def test_dao_get_notification_history_by_reference_with_no_matches_raises_error(notify_db_session): with pytest.raises(SQLAlchemyError): - dao_get_notification_or_history_by_reference('REF1') + dao_get_notification_history_by_reference('REF1') @pytest.mark.parametrize("notification_type", From a03de0dd5664a6a1a705aae23231d983a481b5b1 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 20 Sep 2022 20:11:09 -0700 Subject: [PATCH 03/65] remove outdated validatesns library and replace with maintainable code --- Makefile | 1 + app/celery/process_ses_receipts_tasks.py | 48 +++++--- app/celery/validate_sns.py | 113 ++++++++++++++++++ app/config.py | 3 + app/notifications/callbacks.py | 2 +- .../notifications_ses_callback.py | 4 +- devcontainer-api/Dockerfile | 1 + requirements.in | 2 +- requirements.txt | 9 +- .../celery/test_process_ses_receipts_tasks.py | 4 +- 10 files changed, 155 insertions(+), 32 deletions(-) create mode 100644 app/celery/validate_sns.py diff --git a/Makefile b/Makefile index 18caff76d..98fdef116 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in + pip3 install -r requirements.txt .PHONY: audit audit: diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 8f743fa95..0e9a4dabf 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -4,21 +4,20 @@ from json import decoder import iso8601 import requests -import validatesns from celery.exceptions import Retry from flask import Blueprint, current_app, json, jsonify, request -from notifications_utils.statsd_decorators import statsd from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, redis_store, statsd_client +from app import notify_celery, statsd_client +from app.celery.validate_sns import valid_sns_message from app.config import QueueNames from app.dao import notifications_dao from app.errors import InvalidRequest, register_errors from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING from app.notifications.notifications_ses_callback import ( _check_and_queue_complaint_callback_task, - _determine_notification_bounce_type, check_and_queue_callback_task, + determine_notification_bounce_type, get_aws_responses, handle_complaint, ) @@ -26,7 +25,6 @@ from app.notifications.notifications_ses_callback import ( ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) register_errors(ses_callback_blueprint) - class SNSMessageType(enum.Enum): SubscriptionConfirmation = 'SubscriptionConfirmation' Notification = 'Notification' @@ -44,14 +42,6 @@ def verify_message_type(message_type: str): raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') -def get_certificate(url): - res = redis_store.get(url) - if res is not None: - return res - res = requests.get(url).content - redis_store.set(url, res, ex=60 * 60) # 60 minutes - return res - # 400 counts as a permanent failure so SNS will not retry. # 500 counts as a failed delivery attempt so SNS will retry. # See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies @@ -62,35 +52,53 @@ def get_certificate(url): def sns_callback_handler(): message_type = request.headers.get('x-amz-sns-message-type') try: + print("validating message type") verify_message_type(message_type) except InvalidMessageTypeException: + current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) try: - message = json.loads(request.data) + print("loading message") + message = json.loads(request.data.decode('utf-8')) except decoder.JSONDecodeError: + current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) + current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}") + try: - validatesns.validate(message, get_certificate=get_certificate) - except validatesns.ValidationError: + print("attempting to validate sns") + if valid_sns_message(message) == False: + current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") + print("attempting to validate sns failed") + raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + except Exception as e: + current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': - url = message.get('SubscribeURL') + print("processing subscription") + url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) try: response.raise_for_status() except Exception as e: - current_app.logger.warning("Response: {}".format(response.text)) + current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}") raise e return jsonify( result="success", message="SES-SNS auto-confirm callback succeeded" ), 200 + print("info logging") + # TODO remove after smoke testing on prod is implemented + current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.") + + print("running process_ses_results") process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY) + print("returning success") return jsonify( result="success", message="SES-SNS callback succeeded" ), 200 @@ -101,10 +109,12 @@ def process_ses_results(self, response): try: ses_message = json.loads(response["Message"]) notification_type = ses_message["notificationType"] + # TODO remove after smoke testing on prod is implemented + current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}") bounce_message = None if notification_type == 'Bounce': - bounce_message = _determine_notification_bounce_type(ses_message) + bounce_message = determine_notification_bounce_type(ses_message) elif notification_type == 'Complaint': _check_and_queue_complaint_callback_task(*handle_complaint(ses_message)) return True diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns.py new file mode 100644 index 000000000..e81e8fd15 --- /dev/null +++ b/app/celery/validate_sns.py @@ -0,0 +1,113 @@ +import base64 +import re +from urllib.parse import urlparse + +import requests +from M2Crypto import X509 + +from app import redis_store +from app.config import Config + +USE_CACHE = True +VALIDATE_ARN = True + + +_signing_cert_cache = {} +_cert_url_re = re.compile( + r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com', +) + + +VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS +# VALID_SNS_TOPICS = ['my_bounce_topic_name', 'my_success_topic_name', 'my_complaint_topic_name'] + + +def get_certificate(url): + if USE_CACHE: + res = redis_store.get(url) + if res is not None: + return res + res = requests.get(url).text + redis_store.set(url, res, ex=60 * 60) # 60 minutes + return res + else: + return requests.get(url).text + + +def valid_sns_message(sns_payload): + """ + Adapted from the solution posted at + https://github.com/boto/boto3/issues/2508#issuecomment-992931814 + """ + if not isinstance(sns_payload, dict): + return False + + # Amazon SNS currently supports signature version 1. + if sns_payload.get('SignatureVersion') != '1': + return False + + if VALIDATE_ARN: + arn = sns_payload.get('TopicArn') + topic_name = arn.split(':')[5] + if topic_name not in VALID_SNS_TOPICS: + return False + + payload_type = sns_payload.get('Type') + if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']: + fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'] + elif payload_type == 'Notification': + fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type'] + else: + return False + + # Build the string to be signed. + string_to_sign = '' + for field in fields: + field_value = sns_payload.get(field) + if not isinstance(field_value, str): + return False + string_to_sign += field + '\n' + field_value + '\n' + + # Get the signature + try: + decoded_signature = base64.b64decode(sns_payload.get('Signature')) + except (TypeError, ValueError): + return False + + # Key signing cert url via Lambda and via webhook are slightly different + signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL') + if not isinstance(signing_cert_url, str): + return False + cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url) + if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc): + # The cert doesn't seem to be from AWS + return False + certificate = _signing_cert_cache.get(signing_cert_url) + if certificate is None: + certificate = X509.load_cert_string(get_certificate(signing_cert_url)) + _signing_cert_cache[signing_cert_url] = certificate + + if certificate.get_subject().as_text() != 'CN=sns.amazonaws.com': + return False + + # Extract the public key. + public_key = certificate.get_pubkey() + + # Amazon SNS uses SHA1withRSA. + # http://sns-public-resources.s3.amazonaws.com/SNS_Message_Signing_Release_Note_Jan_25_2011.pdf + public_key.reset_context(md='sha1') + public_key.verify_init() + + # Sign the string. + public_key.verify_update(string_to_sign.encode()) + + # Verify the signature matches. + verification_result = public_key.verify_final(decoded_signature) + + # M2Crypto uses EVP_VerifyFinal() from openssl as the underlying + # verification function. 1 indicates success, anything else is either + # a failure or an error. + if verification_result != 1: + return False + + return True \ No newline at end of file diff --git a/app/config.py b/app/config.py index f1415e294..7aaf81ae9 100644 --- a/app/config.py +++ b/app/config.py @@ -119,6 +119,9 @@ class Config(object): # Use notify.sandbox.10x sending domain unless overwritten by environment NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' + + # AWS SNS topics for delivery receipts + VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint'] # URL of redis instance REDIS_URL = os.environ.get('REDIS_URL') diff --git a/app/notifications/callbacks.py b/app/notifications/callbacks.py index 7c3f1eed6..253e83af1 100644 --- a/app/notifications/callbacks.py +++ b/app/notifications/callbacks.py @@ -49,4 +49,4 @@ def create_complaint_callback_data(complaint, notification, service_callback_api "service_callback_api_bearer_token": service_callback_api.bearer_token, } - return encryption.encrypt(data) \ No newline at end of file + return encryption.encrypt(data) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index a56de6f5a..472891418 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -17,7 +17,7 @@ from app.models import Complaint from app.notifications.callbacks import create_complaint_callback_data -def _determine_notification_bounce_type(ses_message): +def determine_notification_bounce_type(ses_message): notification_type = ses_message["notificationType"] if notification_type in ["Delivery", "Complaint"]: return notification_type @@ -51,7 +51,7 @@ def _determine_provider_response(ses_message): def get_aws_responses(ses_message): - status = _determine_notification_bounce_type(ses_message) + status = determine_notification_bounce_type(ses_message) base = { "Permanent": { diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile index cdedfcc59..fd394946c 100644 --- a/devcontainer-api/Dockerfile +++ b/devcontainer-api/Dockerfile @@ -21,6 +21,7 @@ RUN apt-get update \ openssh-client \ procps \ sudo \ + swig \ tldr \ unzip \ vim \ diff --git a/requirements.in b/requirements.in index fec2b41c6..b82832472 100644 --- a/requirements.in +++ b/requirements.in @@ -16,10 +16,10 @@ itsdangerous==2.1.2 jsonschema[format]==4.5.1 marshmallow-sqlalchemy==0.28.1 marshmallow==3.15.0 +M2Crypto==0.38.0 psycopg2-binary==2.9.3 PyJWT==2.4.0 SQLAlchemy==1.4.40 -validatesns==0.1.1 cachetools==5.1.0 beautifulsoup4==4.11.1 lxml==4.9.1 diff --git a/requirements.txt b/requirements.txt index 2cdc474ae..ecbb8c22d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,6 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration -asn1crypto==1.5.1 - # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -150,6 +148,8 @@ kombu==5.2.4 # via celery lxml==4.9.1 # via -r requirements.in +m2crypto==0.38.0 + # via -r requirements.in mako==1.2.0 # via alembic markupsafe==2.1.1 @@ -171,8 +171,6 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils -oscrypto==1.3.0 - # via validatesns packaging==21.3 # via # bleach @@ -251,7 +249,6 @@ six==1.16.0 # flask-marshmallow # python-dateutil # rfc3339-validator - # validatesns smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 @@ -272,8 +269,6 @@ urllib3==1.26.9 # via # botocore # requests -validatesns==0.1.1 - # via -r requirements.in vine==5.0.0 # via # amqp diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 934f76c79..ad244c4f8 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -71,7 +71,7 @@ def test_notifications_ses_400_with_certificate(client): def test_notifications_ses_200_autoconfirms_subscription(client, mocker): - mocker.patch("validatesns.validate") + mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) requests_mock = mocker.patch("requests.get") data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"}) response = client.post( @@ -85,7 +85,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker): def test_notifications_ses_200_call_process_task(client, mocker): - mocker.patch("validatesns.validate") + mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async") data = {"Type": "Notification", "foo": "bar"} json_data = json.dumps(data) From 4c86024f219c7a4eb329b6cc25251f722c6df789 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 20 Sep 2022 20:22:12 -0700 Subject: [PATCH 04/65] clean up comments --- app/celery/process_ses_receipts_tasks.py | 8 -------- app/celery/validate_sns.py | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 0e9a4dabf..44e000404 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -52,14 +52,12 @@ def verify_message_type(message_type: str): def sns_callback_handler(): message_type = request.headers.get('x-amz-sns-message-type') try: - print("validating message type") verify_message_type(message_type) except InvalidMessageTypeException: current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) try: - print("loading message") message = json.loads(request.data.decode('utf-8')) except decoder.JSONDecodeError: current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") @@ -68,17 +66,14 @@ def sns_callback_handler(): current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}") try: - print("attempting to validate sns") if valid_sns_message(message) == False: current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") - print("attempting to validate sns failed") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) except Exception as e: current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': - print("processing subscription") url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) try: @@ -91,14 +86,11 @@ def sns_callback_handler(): result="success", message="SES-SNS auto-confirm callback succeeded" ), 200 - print("info logging") # TODO remove after smoke testing on prod is implemented current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.") - print("running process_ses_results") process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY) - print("returning success") return jsonify( result="success", message="SES-SNS callback succeeded" ), 200 diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns.py index e81e8fd15..bd86d0da7 100644 --- a/app/celery/validate_sns.py +++ b/app/celery/validate_sns.py @@ -10,6 +10,7 @@ from app.config import Config USE_CACHE = True VALIDATE_ARN = True +VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS _signing_cert_cache = {} @@ -18,10 +19,6 @@ _cert_url_re = re.compile( ) -VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS -# VALID_SNS_TOPICS = ['my_bounce_topic_name', 'my_success_topic_name', 'my_complaint_topic_name'] - - def get_certificate(url): if USE_CACHE: res = redis_store.get(url) From ea3eefa81cb46051d320448475d611806caa6949 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 23 Sep 2022 11:56:39 -0700 Subject: [PATCH 05/65] test branch for notify-api-alt temporary deploy --- Makefile | 2 +- app/celery/process_ses_receipts_tasks.py | 37 +++++++++++++++++++----- app/cloudfoundry_config.py | 5 ++-- manifest.yml | 18 +++++++----- requirements.in | 5 ++-- requirements.txt | 11 +++++-- 6 files changed, 54 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 98fdef116..9e58f797c 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,7 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in - pip3 install -r requirements.txt + pip3 install -r requirements.txt --no-cache-dir .PHONY: audit audit: diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 44e000404..00cbb367c 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -4,12 +4,14 @@ from json import decoder import iso8601 import requests +import traceback from celery.exceptions import Retry from flask import Blueprint, current_app, json, jsonify, request from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, statsd_client -from app.celery.validate_sns import valid_sns_message +from app import notify_celery, statsd_client, redis_store +# from app.celery.validate_sns import valid_sns_message +import validatesns from app.config import QueueNames from app.dao import notifications_dao from app.errors import InvalidRequest, register_errors @@ -23,6 +25,7 @@ from app.notifications.notifications_ses_callback import ( ) ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) +DEFAULT_MAX_AGE = timedelta(days=10000) register_errors(ses_callback_blueprint) class SNSMessageType(enum.Enum): @@ -41,6 +44,13 @@ def verify_message_type(message_type: str): except ValueError: raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') +def get_certificate(url): + res = redis_store.get(url) + if res is not None: + return res + res = requests.get(url).content + redis_store.set(url, res, ex=60 * 60) # 60 minutes + return res # 400 counts as a permanent failure so SNS will not retry. # 500 counts as a failed delivery attempt so SNS will retry. @@ -64,15 +74,26 @@ def sns_callback_handler(): raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}") - + try: - if valid_sns_message(message) == False: - current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") - raise InvalidRequest("SES-SNS callback failed: validation failed", 400) - except Exception as e: - current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") + # AWS sends SigningCertURL if sending to a webhook, but SigningCertUrl if sending to a Lambda function + message["SigningCertURL"] = message["SigningCertURL"] if "SigningCertURL" in message else message["SigningCertUrl"] + # Some SNS messages now contain "Subject": null, which is not handled by the validatesns library + if "Subject" in message and message["Subject"] == None: + message.pop("Subject") + validatesns.validate(message, get_certificate=get_certificate, max_age=DEFAULT_MAX_AGE) + except Exception as err: + current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err} and traceback {traceback.format_exc()}") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + # try: + # if valid_sns_message(message) == False: + # current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") + # raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + # except Exception as e: + # current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") + # raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + if message.get('Type') == 'SubscriptionConfirmation': url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 7daad3216..7d0920b6a 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -6,7 +6,6 @@ def extract_cloudfoundry_config(): vcap_services = json.loads(os.environ['VCAP_SERVICES']) # Postgres config - os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', - 'postgresql') + os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql') # Redis config - os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'] + os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://') diff --git a/manifest.yml b/manifest.yml index ca6ea42eb..798603ac1 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,7 +1,7 @@ --- applications: - - name: notifications-api + - name: notify-api-alt buildpack: https://github.com/cloudfoundry/python-buildpack.git#v1.7.58 instances: 1 memory: 1G @@ -9,22 +9,24 @@ applications: health-check-type: process health-check-invocation-timeout: 1 routes: - - route: notifications-api.app.cloud.gov + - route: notify-api-alt.app.cloud.gov services: - - api-psql - - api-redis + - api-alt-psql + - api-alt-redis env: + BP_PIP_VERSION: latest NOTIFY_APP_NAME: api NOTIFY_LOG_PATH: /home/vcap/logs/app.log FLASK_APP: application.py FLASK_ENV: production NOTIFY_ENVIRONMENT: live - API_HOST_NAME: https://notifications-api.app.cloud.gov - ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov - NOTIFICATION_QUEUE_PREFIX: prototype_10x + API_HOST_NAME: https://notify-api-alt.app.cloud.gov + ADMIN_BASE_URL: https://notify-admin-alt.app.cloud.gov + NOTIFICATION_QUEUE_PREFIX: notify_alt_ + REDIS_ENABLED: true STATSD_HOST: localhost INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' @@ -33,6 +35,8 @@ applications: ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) + AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID)) + AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY)) AWS_REGION: us-west-2 AWS_PINPOINT_REGION: us-west-2 AWS_US_TOLL_FREE_NUMBER: +18446120782 diff --git a/requirements.in b/requirements.in index b82832472..10f58e908 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ celery[sqs]==5.2.6 Flask-Bcrypt==1.0.1 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1 +Flask-SQLAlchemy==2.5.1 Flask==2.1.2 click-datetime==0.2 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues) @@ -16,7 +16,6 @@ itsdangerous==2.1.2 jsonschema[format]==4.5.1 marshmallow-sqlalchemy==0.28.1 marshmallow==3.15.0 -M2Crypto==0.38.0 psycopg2-binary==2.9.3 PyJWT==2.4.0 SQLAlchemy==1.4.40 @@ -27,6 +26,8 @@ defusedxml==0.7.1 Werkzeug==2.1.1 python-dotenv==0.20.0 +validatesns==0.1.1 + notifications-python-client==6.3.0 # PaaS diff --git a/requirements.txt b/requirements.txt index ecbb8c22d..5270fbb29 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration +asn1crypto==1.5.1 + # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -99,7 +101,7 @@ flask-migrate==3.1.0 # via -r requirements.in flask-redis==0.4.0 # via notifications-utils -flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa +flask-sqlalchemy==2.5.1 # via # -r requirements.in # flask-migrate @@ -148,8 +150,6 @@ kombu==5.2.4 # via celery lxml==4.9.1 # via -r requirements.in -m2crypto==0.38.0 - # via -r requirements.in mako==1.2.0 # via alembic markupsafe==2.1.1 @@ -171,6 +171,8 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils +oscrypto==1.3.0 + # via validatesns packaging==21.3 # via # bleach @@ -249,6 +251,7 @@ six==1.16.0 # flask-marshmallow # python-dateutil # rfc3339-validator + # validatesns smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 @@ -269,6 +272,8 @@ urllib3==1.26.9 # via # botocore # requests +validatesns==0.1.1 + # via -r requirements.in vine==5.0.0 # via # amqp From c636eac96481ca8748dd286c466e291eb2589c72 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 23 Sep 2022 15:57:06 -0700 Subject: [PATCH 06/65] replace m2crypto with oscrypto --- app/celery/process_ses_receipts_tasks.py | 29 +------ app/celery/validate_sns.py | 105 ++++++++++++----------- 2 files changed, 57 insertions(+), 77 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 00cbb367c..0b7290593 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -10,8 +10,7 @@ from flask import Blueprint, current_app, json, jsonify, request from sqlalchemy.orm.exc import NoResultFound from app import notify_celery, statsd_client, redis_store -# from app.celery.validate_sns import valid_sns_message -import validatesns +from app.celery.validate_sns import validate_sns_message from app.config import QueueNames from app.dao import notifications_dao from app.errors import InvalidRequest, register_errors @@ -44,13 +43,6 @@ def verify_message_type(message_type: str): except ValueError: raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') -def get_certificate(url): - res = redis_store.get(url) - if res is not None: - return res - res = requests.get(url).content - redis_store.set(url, res, ex=60 * 60) # 60 minutes - return res # 400 counts as a permanent failure so SNS will not retry. # 500 counts as a failed delivery attempt so SNS will retry. @@ -73,27 +65,12 @@ def sns_callback_handler(): current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) - current_app.logger.info(f"Message type: {message_type}\nResponse data: {message}") - try: - # AWS sends SigningCertURL if sending to a webhook, but SigningCertUrl if sending to a Lambda function - message["SigningCertURL"] = message["SigningCertURL"] if "SigningCertURL" in message else message["SigningCertUrl"] - # Some SNS messages now contain "Subject": null, which is not handled by the validatesns library - if "Subject" in message and message["Subject"] == None: - message.pop("Subject") - validatesns.validate(message, get_certificate=get_certificate, max_age=DEFAULT_MAX_AGE) + validate_sns_message(message) except Exception as err: - current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err} and traceback {traceback.format_exc()}") + current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err}") raise InvalidRequest("SES-SNS callback failed: validation failed", 400) - # try: - # if valid_sns_message(message) == False: - # current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed.") - # raise InvalidRequest("SES-SNS callback failed: validation failed", 400) - # except Exception as e: - # current_app.logger.exception(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: {e}") - # raise InvalidRequest("SES-SNS callback failed: validation failed", 400) - if message.get('Type') == 'SubscriptionConfirmation': url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns.py index bd86d0da7..e639b17a1 100644 --- a/app/celery/validate_sns.py +++ b/app/celery/validate_sns.py @@ -3,11 +3,14 @@ import re from urllib.parse import urlparse import requests -from M2Crypto import X509 +import oscrypto.asymmetric +import oscrypto.errors from app import redis_store from app.config import Config +import six + USE_CACHE = True VALIDATE_ARN = True VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS @@ -18,6 +21,11 @@ _cert_url_re = re.compile( r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com', ) +class ValidationError(Exception): + """ + ValidationError. Raised when a message fails integrity checks. + """ + def get_certificate(url): if USE_CACHE: @@ -31,80 +39,75 @@ def get_certificate(url): return requests.get(url).text -def valid_sns_message(sns_payload): - """ - Adapted from the solution posted at - https://github.com/boto/boto3/issues/2508#issuecomment-992931814 - """ - if not isinstance(sns_payload, dict): - return False - - # Amazon SNS currently supports signature version 1. - if sns_payload.get('SignatureVersion') != '1': - return False - +def validate_arn(sns_payload): if VALIDATE_ARN: arn = sns_payload.get('TopicArn') topic_name = arn.split(':')[5] if topic_name not in VALID_SNS_TOPICS: - return False + raise ValidationError("Invalid Topic Name") + +def get_string_to_sign(sns_payload): payload_type = sns_payload.get('Type') if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']: fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'] elif payload_type == 'Notification': fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type'] else: - return False + raise ValidationError("Unexpected Message Type") - # Build the string to be signed. string_to_sign = '' for field in fields: field_value = sns_payload.get(field) if not isinstance(field_value, str): - return False + if field == 'Subject' and field_value == None: + continue + raise ValidationError(f"In {field}, found non-string value: {field_value}") string_to_sign += field + '\n' + field_value + '\n' + if isinstance(string_to_sign, six.text_type): + string_to_sign = string_to_sign.encode() + return string_to_sign - # Get the signature - try: - decoded_signature = base64.b64decode(sns_payload.get('Signature')) - except (TypeError, ValueError): - return False + +def validate_sns_message(sns_payload): + """ + Adapted from the solution posted at + https://github.com/boto/boto3/issues/2508#issuecomment-992931814 + """ + if not isinstance(sns_payload, dict): + raise ValidationError("Unexpected message type {!r}".format(type(sns_payload).__name__)) + + # Amazon SNS currently supports signature version 1. + if sns_payload.get('SignatureVersion') != '1': + raise ValidationError("Wrong Signature Version (expected 1)") + + validate_arn(sns_payload) + + string_to_sign = get_string_to_sign(sns_payload) # Key signing cert url via Lambda and via webhook are slightly different signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL') if not isinstance(signing_cert_url, str): - return False + raise ValidationError("Signing cert url must be a string") cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url) if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc): - # The cert doesn't seem to be from AWS - return False + raise ValidationError("Cert does not appear to be from AWS") + certificate = _signing_cert_cache.get(signing_cert_url) if certificate is None: - certificate = X509.load_cert_string(get_certificate(signing_cert_url)) - _signing_cert_cache[signing_cert_url] = certificate + certificate = get_certificate(signing_cert_url) + if isinstance(certificate, six.text_type): + certificate = certificate.encode() + + signature = base64.b64decode(sns_payload["Signature"]) - if certificate.get_subject().as_text() != 'CN=sns.amazonaws.com': - return False - - # Extract the public key. - public_key = certificate.get_pubkey() - - # Amazon SNS uses SHA1withRSA. - # http://sns-public-resources.s3.amazonaws.com/SNS_Message_Signing_Release_Note_Jan_25_2011.pdf - public_key.reset_context(md='sha1') - public_key.verify_init() - - # Sign the string. - public_key.verify_update(string_to_sign.encode()) - - # Verify the signature matches. - verification_result = public_key.verify_final(decoded_signature) - - # M2Crypto uses EVP_VerifyFinal() from openssl as the underlying - # verification function. 1 indicates success, anything else is either - # a failure or an error. - if verification_result != 1: - return False - - return True \ No newline at end of file + try: + oscrypto.asymmetric.rsa_pkcs1v15_verify( + oscrypto.asymmetric.load_certificate(certificate), + signature, + string_to_sign, + "sha1" + ) + return True + except oscrypto.errors.SignatureError: + raise ValidationError("Invalid signature") \ No newline at end of file From 06c2727e65cdb865f4976476b7d5c65366f692ab Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 23 Sep 2022 17:09:03 -0700 Subject: [PATCH 07/65] fix requirements --- Makefile | 1 - requirements.in | 2 -- requirements.txt | 7 ------- 3 files changed, 10 deletions(-) diff --git a/Makefile b/Makefile index 9e58f797c..18caff76d 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,6 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in - pip3 install -r requirements.txt --no-cache-dir .PHONY: audit audit: diff --git a/requirements.in b/requirements.in index 10f58e908..615065db4 100644 --- a/requirements.in +++ b/requirements.in @@ -26,8 +26,6 @@ defusedxml==0.7.1 Werkzeug==2.1.1 python-dotenv==0.20.0 -validatesns==0.1.1 - notifications-python-client==6.3.0 # PaaS diff --git a/requirements.txt b/requirements.txt index 5270fbb29..3982ed643 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,8 +10,6 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration -asn1crypto==1.5.1 - # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -171,8 +169,6 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils -oscrypto==1.3.0 - # via validatesns packaging==21.3 # via # bleach @@ -251,7 +247,6 @@ six==1.16.0 # flask-marshmallow # python-dateutil # rfc3339-validator - # validatesns smartypants==2.0.1 # via notifications-utils soupsieve==2.3.2.post1 @@ -272,8 +267,6 @@ urllib3==1.26.9 # via # botocore # requests -validatesns==0.1.1 - # via -r requirements.in vine==5.0.0 # via # amqp From 0e5ea849b90d02504366cbac1bdf3ff7f8f7c2e1 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Wed, 28 Sep 2022 13:26:21 -0400 Subject: [PATCH 08/65] latest --- .gitignore | 1 + Makefile | 5 +- app/notifications/receive_notifications.py | 56 ++++++++++++++++++++++ requirements.in | 3 +- requirements.txt | 2 +- 5 files changed, 63 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 220a2d3b2..4edc8e0d2 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ queues.csv __pycache__/ *.py[cod] +.venv/ venv/ venv-freeze/ diff --git a/Makefile b/Makefile index 18caff76d..9761abb2e 100644 --- a/Makefile +++ b/Makefile @@ -71,6 +71,7 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in + pip install -r requirements.txt .PHONY: audit audit: @@ -137,11 +138,11 @@ cf-deploy-api-db-migration: cf push notifications-api --no-route -f ${CF_MANIFEST_PATH} rm ${CF_MANIFEST_PATH} - cf run-task notifications-api --command="flask db upgrade" --name api_db_migration + cf run-task notify-api-alt --command="flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task cf-check-api-db-migration-task: ## Get the status for the last notifications-api task - @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" + @cf curl /v3/apps/`cf app --guid notify-api-alt`/tasks?order_by=-created_at | jq -r ".resources[0].state" .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 72103c20f..7b6987ac3 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -23,6 +23,62 @@ INBOUND_SMS_COUNTER = Counter( ['provider'] ) +@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST']) +def receive_sns_sms(): + """ + { + "originationNumber":"+14255550182", + "destinationNumber":"+12125550101", + "messageKeyword":"JOIN", # this is optional + "messageBody":"EXAMPLE", + "inboundMessageId":"cae173d2-66b9-564c-8309-21f858e9fb84", + "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + """ + + post_data = request.get_json() + + # validate sns from common module, WILL ALSO NEED TO AUTO-SUBSCRIBE... raise errors appropriately + + # TODO modify this for AWS SNS + inbound_number = strip_leading_forty_four(post_data['Number']) + + service = fetch_potential_service(inbound_number, 'sns') + if not service: + # since this is an issue with our service <-> number mapping, or no inbound_sms service permission + # we should still tell SNS that we received it successfully + current_app.logger.warning(f"Mapping between service id and inbound number is broken, or service does not have permission to receive inbound sms") + return jsonify({ + "status": "ok" + }), 200 + + INBOUND_SMS_COUNTER.labels("sns").inc() + + content = format_mmg_message(post_data["Message"]) + from_number = post_data['MSISDN'] + provider_ref = post_data["ID"] + date_received = post_data.get('DateRecieved') + provider_name = "sns" + + inbound_payload = {} + + # TODO fill inbound_payload and spread like create_inbound_sms_object(service, **inbound_payload) + inbound = create_inbound_sms_object(service, + content=format_mmg_message(post_data["Message"]), + from_number=from_number, + provider_ref=provider_ref, + date_received=date_received, + provider_name=provider_name) + + # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users + # tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) + + current_app.logger.debug( + '{} received inbound SMS with reference {} from SNS'.format(service.id, inbound.provider_reference)) + + return jsonify({ + "status": "ok" + }), 200 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) def receive_mmg_sms(): diff --git a/requirements.in b/requirements.in index 615065db4..5ef7876a0 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ celery[sqs]==5.2.6 Flask-Bcrypt==1.0.1 flask-marshmallow==0.14.0 Flask-Migrate==3.1.0 -Flask-SQLAlchemy==2.5.1 +git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1 Flask==2.1.2 click-datetime==0.2 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues) @@ -25,6 +25,7 @@ lxml==4.9.1 defusedxml==0.7.1 Werkzeug==2.1.1 python-dotenv==0.20.0 +oscrypto notifications-python-client==6.3.0 diff --git a/requirements.txt b/requirements.txt index 3982ed643..154c1c908 100644 --- a/requirements.txt +++ b/requirements.txt @@ -99,7 +99,7 @@ flask-migrate==3.1.0 # via -r requirements.in flask-redis==0.4.0 # via notifications-utils -flask-sqlalchemy==2.5.1 +flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa # via # -r requirements.in # flask-migrate From 1161e2c6cb40feb56974f7840cae531b22c032f1 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 30 Sep 2022 10:37:08 -0400 Subject: [PATCH 09/65] latest --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index 154c1c908..aa7708039 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,8 @@ amqp==5.1.1 # via kombu arrow==1.2.2 # via isoduration +asn1crypto==1.5.1 + # via oscrypto async-timeout==4.0.2 # via redis attrs==21.4.0 @@ -169,6 +171,8 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git # via -r requirements.in orderedset==2.0.3 # via notifications-utils +oscrypto==1.3.0 + # via -r requirements.in packaging==21.3 # via # bleach From 48af6f7c23db918f4ea189f81e2058dcfce999a6 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 30 Sep 2022 10:59:48 -0400 Subject: [PATCH 10/65] fix tests --- Makefile | 5 +- app/celery/process_ses_receipts_tasks.py | 4 +- app/celery/validate_sns.py | 5 +- app/notifications/receive_notifications.py | 56 ------------------- devcontainer-api/Dockerfile | 1 - manifest.yml | 23 ++++---- requirements.in | 2 +- .../celery/test_process_ses_receipts_tasks.py | 4 +- 8 files changed, 20 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 9761abb2e..18caff76d 100644 --- a/Makefile +++ b/Makefile @@ -71,7 +71,6 @@ test: ## Run tests freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt pip install --upgrade pip-tools pip-compile requirements.in - pip install -r requirements.txt .PHONY: audit audit: @@ -138,11 +137,11 @@ cf-deploy-api-db-migration: cf push notifications-api --no-route -f ${CF_MANIFEST_PATH} rm ${CF_MANIFEST_PATH} - cf run-task notify-api-alt --command="flask db upgrade" --name api_db_migration + cf run-task notifications-api --command="flask db upgrade" --name api_db_migration .PHONY: cf-check-api-db-migration-task cf-check-api-db-migration-task: ## Get the status for the last notifications-api task - @cf curl /v3/apps/`cf app --guid notify-api-alt`/tasks?order_by=-created_at | jq -r ".resources[0].state" + @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" .PHONY: cf-rollback cf-rollback: ## Rollbacks the app to the previous release diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 0b7290593..dd34b4ee0 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -1,15 +1,15 @@ import enum +import traceback from datetime import datetime, timedelta from json import decoder import iso8601 import requests -import traceback from celery.exceptions import Retry from flask import Blueprint, current_app, json, jsonify, request from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, statsd_client, redis_store +from app import notify_celery, redis_store, statsd_client from app.celery.validate_sns import validate_sns_message from app.config import QueueNames from app.dao import notifications_dao diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns.py index e639b17a1..d918d0590 100644 --- a/app/celery/validate_sns.py +++ b/app/celery/validate_sns.py @@ -2,15 +2,14 @@ import base64 import re from urllib.parse import urlparse -import requests import oscrypto.asymmetric import oscrypto.errors +import requests +import six from app import redis_store from app.config import Config -import six - USE_CACHE = True VALIDATE_ARN = True VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 7b6987ac3..72103c20f 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -23,62 +23,6 @@ INBOUND_SMS_COUNTER = Counter( ['provider'] ) -@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST']) -def receive_sns_sms(): - """ - { - "originationNumber":"+14255550182", - "destinationNumber":"+12125550101", - "messageKeyword":"JOIN", # this is optional - "messageBody":"EXAMPLE", - "inboundMessageId":"cae173d2-66b9-564c-8309-21f858e9fb84", - "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" - } - """ - - post_data = request.get_json() - - # validate sns from common module, WILL ALSO NEED TO AUTO-SUBSCRIBE... raise errors appropriately - - # TODO modify this for AWS SNS - inbound_number = strip_leading_forty_four(post_data['Number']) - - service = fetch_potential_service(inbound_number, 'sns') - if not service: - # since this is an issue with our service <-> number mapping, or no inbound_sms service permission - # we should still tell SNS that we received it successfully - current_app.logger.warning(f"Mapping between service id and inbound number is broken, or service does not have permission to receive inbound sms") - return jsonify({ - "status": "ok" - }), 200 - - INBOUND_SMS_COUNTER.labels("sns").inc() - - content = format_mmg_message(post_data["Message"]) - from_number = post_data['MSISDN'] - provider_ref = post_data["ID"] - date_received = post_data.get('DateRecieved') - provider_name = "sns" - - inbound_payload = {} - - # TODO fill inbound_payload and spread like create_inbound_sms_object(service, **inbound_payload) - inbound = create_inbound_sms_object(service, - content=format_mmg_message(post_data["Message"]), - from_number=from_number, - provider_ref=provider_ref, - date_received=date_received, - provider_name=provider_name) - - # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users - # tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) - - current_app.logger.debug( - '{} received inbound SMS with reference {} from SNS'.format(service.id, inbound.provider_reference)) - - return jsonify({ - "status": "ok" - }), 200 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) def receive_mmg_sms(): diff --git a/devcontainer-api/Dockerfile b/devcontainer-api/Dockerfile index fd394946c..cdedfcc59 100644 --- a/devcontainer-api/Dockerfile +++ b/devcontainer-api/Dockerfile @@ -21,7 +21,6 @@ RUN apt-get update \ openssh-client \ procps \ sudo \ - swig \ tldr \ unzip \ vim \ diff --git a/manifest.yml b/manifest.yml index 798603ac1..2d365c9dd 100644 --- a/manifest.yml +++ b/manifest.yml @@ -1,7 +1,6 @@ --- - applications: - - name: notify-api-alt + - name: notifications-api-((env)) buildpack: https://github.com/cloudfoundry/python-buildpack.git#v1.7.58 instances: 1 memory: 1G @@ -9,29 +8,29 @@ applications: health-check-type: process health-check-invocation-timeout: 1 routes: - - route: notify-api-alt.app.cloud.gov + - route: notifications-api.app.cloud.gov + - route: notifications-api-((env)).apps.internal services: - - api-alt-psql - - api-alt-redis + - notifications-api-rds-((env)) + - notifications-api-redis-((env)) + - notifications-api-csv-upload-bucket-((env)) + - notifications-api-contact-list-bucket-((env)) env: - BP_PIP_VERSION: latest NOTIFY_APP_NAME: api NOTIFY_LOG_PATH: /home/vcap/logs/app.log FLASK_APP: application.py FLASK_ENV: production + DEPLOY_ENV: ((env)) NOTIFY_ENVIRONMENT: live - API_HOST_NAME: https://notify-api-alt.app.cloud.gov - ADMIN_BASE_URL: https://notify-admin-alt.app.cloud.gov - NOTIFICATION_QUEUE_PREFIX: notify_alt_ - REDIS_ENABLED: true + API_HOST_NAME: https://notifications-api.app.cloud.gov + ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov STATSD_HOST: localhost - INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' - # Credentials variables + INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["((ADMIN_CLIENT_SECRET))"]}' ADMIN_CLIENT_SECRET: ((ADMIN_CLIENT_SECRET)) DANGEROUS_SALT: ((DANGEROUS_SALT)) SECRET_KEY: ((SECRET_KEY)) diff --git a/requirements.in b/requirements.in index 5ef7876a0..20da3c479 100644 --- a/requirements.in +++ b/requirements.in @@ -25,7 +25,7 @@ lxml==4.9.1 defusedxml==0.7.1 Werkzeug==2.1.1 python-dotenv==0.20.0 -oscrypto +oscrypto==1.3.0 notifications-python-client==6.3.0 diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index ad244c4f8..c6a9bbf4b 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -71,7 +71,7 @@ def test_notifications_ses_400_with_certificate(client): def test_notifications_ses_200_autoconfirms_subscription(client, mocker): - mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) + mocker.patch("app.celery.process_ses_receipts_tasks.validate_sns_message", return_value=True) requests_mock = mocker.patch("requests.get") data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"}) response = client.post( @@ -85,7 +85,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker): def test_notifications_ses_200_call_process_task(client, mocker): - mocker.patch("app.celery.process_ses_receipts_tasks.valid_sns_message", return_value=True) + mocker.patch("app.celery.process_ses_receipts_tasks.validate_sns_message", return_value=True) process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async") data = {"Type": "Notification", "foo": "bar"} json_data = json.dumps(data) From 8cb6f60f04311e58564a7b510314bc0f4462eeb4 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Mon, 3 Oct 2022 09:05:34 -0700 Subject: [PATCH 11/65] modify inbound notif processing --- .venv/bin/Activate.ps1 | 247 ++++++++++++++++++ .venv/bin/activate | 69 +++++ .venv/bin/activate.csh | 26 ++ .venv/bin/activate.fish | 66 +++++ .venv/bin/pip | 8 + .venv/bin/pip-compile | 8 + .venv/bin/pip-sync | 8 + .venv/bin/pip3 | 8 + .venv/bin/pip3.10 | 8 + .venv/bin/pyproject-build | 8 + .venv/bin/python | 1 + .venv/bin/python3 | 1 + .venv/bin/python3.10 | 1 + .venv/bin/wheel | 8 + .venv/pyvenv.cfg | 3 + app/celery/process_ses_receipts_tasks.py | 44 +--- .../{validate_sns.py => validate_sns_cert.py} | 3 +- app/celery/validate_sns_message.py | 66 +++++ app/config.py | 4 +- app/inbound_sms/rest.py | 7 +- app/notifications/receive_notifications.py | 63 ++++- .../versions/0377_add_inbound_sms_number.py | 52 ++++ 22 files changed, 665 insertions(+), 44 deletions(-) create mode 100644 .venv/bin/Activate.ps1 create mode 100644 .venv/bin/activate create mode 100644 .venv/bin/activate.csh create mode 100644 .venv/bin/activate.fish create mode 100755 .venv/bin/pip create mode 100755 .venv/bin/pip-compile create mode 100755 .venv/bin/pip-sync create mode 100755 .venv/bin/pip3 create mode 100755 .venv/bin/pip3.10 create mode 100755 .venv/bin/pyproject-build create mode 120000 .venv/bin/python create mode 120000 .venv/bin/python3 create mode 120000 .venv/bin/python3.10 create mode 100755 .venv/bin/wheel create mode 100644 .venv/pyvenv.cfg rename app/celery/{validate_sns.py => validate_sns_cert.py} (97%) create mode 100644 app/celery/validate_sns_message.py create mode 100644 migrations/versions/0377_add_inbound_sms_number.py diff --git a/.venv/bin/Activate.ps1 b/.venv/bin/Activate.ps1 new file mode 100644 index 000000000..b49d77ba4 --- /dev/null +++ b/.venv/bin/Activate.ps1 @@ -0,0 +1,247 @@ +<# +.Synopsis +Activate a Python virtual environment for the current PowerShell session. + +.Description +Pushes the python executable for a virtual environment to the front of the +$Env:PATH environment variable and sets the prompt to signify that you are +in a Python virtual environment. Makes use of the command line switches as +well as the `pyvenv.cfg` file values present in the virtual environment. + +.Parameter VenvDir +Path to the directory that contains the virtual environment to activate. The +default value for this is the parent of the directory that the Activate.ps1 +script is located within. + +.Parameter Prompt +The prompt prefix to display when this virtual environment is activated. By +default, this prompt is the name of the virtual environment folder (VenvDir) +surrounded by parentheses and followed by a single space (ie. '(.venv) '). + +.Example +Activate.ps1 +Activates the Python virtual environment that contains the Activate.ps1 script. + +.Example +Activate.ps1 -Verbose +Activates the Python virtual environment that contains the Activate.ps1 script, +and shows extra information about the activation as it executes. + +.Example +Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv +Activates the Python virtual environment located in the specified location. + +.Example +Activate.ps1 -Prompt "MyPython" +Activates the Python virtual environment that contains the Activate.ps1 script, +and prefixes the current prompt with the specified string (surrounded in +parentheses) while the virtual environment is active. + +.Notes +On Windows, it may be required to enable this Activate.ps1 script by setting the +execution policy for the user. You can do this by issuing the following PowerShell +command: + +PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + +For more information on Execution Policies: +https://go.microsoft.com/fwlink/?LinkID=135170 + +#> +Param( + [Parameter(Mandatory = $false)] + [String] + $VenvDir, + [Parameter(Mandatory = $false)] + [String] + $Prompt +) + +<# Function declarations --------------------------------------------------- #> + +<# +.Synopsis +Remove all shell session elements added by the Activate script, including the +addition of the virtual environment's Python executable from the beginning of +the PATH variable. + +.Parameter NonDestructive +If present, do not remove this function from the global namespace for the +session. + +#> +function global:deactivate ([switch]$NonDestructive) { + # Revert to original values + + # The prior prompt: + if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { + Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt + Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT + } + + # The prior PYTHONHOME: + if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { + Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME + Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME + } + + # The prior PATH: + if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { + Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH + Remove-Item -Path Env:_OLD_VIRTUAL_PATH + } + + # Just remove the VIRTUAL_ENV altogether: + if (Test-Path -Path Env:VIRTUAL_ENV) { + Remove-Item -Path env:VIRTUAL_ENV + } + + # Just remove VIRTUAL_ENV_PROMPT altogether. + if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { + Remove-Item -Path env:VIRTUAL_ENV_PROMPT + } + + # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: + if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { + Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force + } + + # Leave deactivate function in the global namespace if requested: + if (-not $NonDestructive) { + Remove-Item -Path function:deactivate + } +} + +<# +.Description +Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the +given folder, and returns them in a map. + +For each line in the pyvenv.cfg file, if that line can be parsed into exactly +two strings separated by `=` (with any amount of whitespace surrounding the =) +then it is considered a `key = value` line. The left hand string is the key, +the right hand is the value. + +If the value starts with a `'` or a `"` then the first and last character is +stripped from the value before being captured. + +.Parameter ConfigDir +Path to the directory that contains the `pyvenv.cfg` file. +#> +function Get-PyVenvConfig( + [String] + $ConfigDir +) { + Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" + + # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). + $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue + + # An empty map will be returned if no config file is found. + $pyvenvConfig = @{ } + + if ($pyvenvConfigPath) { + + Write-Verbose "File exists, parse `key = value` lines" + $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath + + $pyvenvConfigContent | ForEach-Object { + $keyval = $PSItem -split "\s*=\s*", 2 + if ($keyval[0] -and $keyval[1]) { + $val = $keyval[1] + + # Remove extraneous quotations around a string value. + if ("'""".Contains($val.Substring(0, 1))) { + $val = $val.Substring(1, $val.Length - 2) + } + + $pyvenvConfig[$keyval[0]] = $val + Write-Verbose "Adding Key: '$($keyval[0])'='$val'" + } + } + } + return $pyvenvConfig +} + + +<# Begin Activate script --------------------------------------------------- #> + +# Determine the containing directory of this script +$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition +$VenvExecDir = Get-Item -Path $VenvExecPath + +Write-Verbose "Activation script is located in path: '$VenvExecPath'" +Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" +Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" + +# Set values required in priority: CmdLine, ConfigFile, Default +# First, get the location of the virtual environment, it might not be +# VenvExecDir if specified on the command line. +if ($VenvDir) { + Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" +} +else { + Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." + $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") + Write-Verbose "VenvDir=$VenvDir" +} + +# Next, read the `pyvenv.cfg` file to determine any required value such +# as `prompt`. +$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir + +# Next, set the prompt from the command line, or the config file, or +# just use the name of the virtual environment folder. +if ($Prompt) { + Write-Verbose "Prompt specified as argument, using '$Prompt'" +} +else { + Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" + if ($pyvenvCfg -and $pyvenvCfg['prompt']) { + Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" + $Prompt = $pyvenvCfg['prompt']; + } + else { + Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" + Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" + $Prompt = Split-Path -Path $venvDir -Leaf + } +} + +Write-Verbose "Prompt = '$Prompt'" +Write-Verbose "VenvDir='$VenvDir'" + +# Deactivate any currently active virtual environment, but leave the +# deactivate function in place. +deactivate -nondestructive + +# Now set the environment variable VIRTUAL_ENV, used by many tools to determine +# that there is an activated venv. +$env:VIRTUAL_ENV = $VenvDir + +if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { + + Write-Verbose "Setting prompt to '$Prompt'" + + # Set the prompt to include the env name + # Make sure _OLD_VIRTUAL_PROMPT is global + function global:_OLD_VIRTUAL_PROMPT { "" } + Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT + New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt + + function global:prompt { + Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " + _OLD_VIRTUAL_PROMPT + } + $env:VIRTUAL_ENV_PROMPT = $Prompt +} + +# Clear PYTHONHOME +if (Test-Path -Path Env:PYTHONHOME) { + Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME + Remove-Item -Path Env:PYTHONHOME +} + +# Add the venv to the PATH +Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH +$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/.venv/bin/activate b/.venv/bin/activate new file mode 100644 index 000000000..d8aaaac7e --- /dev/null +++ b/.venv/bin/activate @@ -0,0 +1,69 @@ +# This file must be used with "source bin/activate" *from bash* +# you cannot run it directly + +deactivate () { + # reset old environment variables + if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then + PATH="${_OLD_VIRTUAL_PATH:-}" + export PATH + unset _OLD_VIRTUAL_PATH + fi + if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then + PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" + export PYTHONHOME + unset _OLD_VIRTUAL_PYTHONHOME + fi + + # This should detect bash and zsh, which have a hash command that must + # be called to get it to forget past commands. Without forgetting + # past commands the $PATH changes we made may not be respected + if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null + fi + + if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then + PS1="${_OLD_VIRTUAL_PS1:-}" + export PS1 + unset _OLD_VIRTUAL_PS1 + fi + + unset VIRTUAL_ENV + unset VIRTUAL_ENV_PROMPT + if [ ! "${1:-}" = "nondestructive" ] ; then + # Self destruct! + unset -f deactivate + fi +} + +# unset irrelevant variables +deactivate nondestructive + +VIRTUAL_ENV="/Users/jamesdmoffet/notifications-api/.venv" +export VIRTUAL_ENV + +_OLD_VIRTUAL_PATH="$PATH" +PATH="$VIRTUAL_ENV/bin:$PATH" +export PATH + +# unset PYTHONHOME if set +# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) +# could use `if (set -u; : $PYTHONHOME) ;` in bash +if [ -n "${PYTHONHOME:-}" ] ; then + _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" + unset PYTHONHOME +fi + +if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then + _OLD_VIRTUAL_PS1="${PS1:-}" + PS1="(.venv) ${PS1:-}" + export PS1 + VIRTUAL_ENV_PROMPT="(.venv) " + export VIRTUAL_ENV_PROMPT +fi + +# This should detect bash and zsh, which have a hash command that must +# be called to get it to forget past commands. Without forgetting +# past commands the $PATH changes we made may not be respected +if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then + hash -r 2> /dev/null +fi diff --git a/.venv/bin/activate.csh b/.venv/bin/activate.csh new file mode 100644 index 000000000..de77f1e28 --- /dev/null +++ b/.venv/bin/activate.csh @@ -0,0 +1,26 @@ +# This file must be used with "source bin/activate.csh" *from csh*. +# You cannot run it directly. +# Created by Davide Di Blasi . +# Ported to Python 3.3 venv by Andrew Svetlov + +alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' + +# Unset irrelevant variables. +deactivate nondestructive + +setenv VIRTUAL_ENV "/Users/jamesdmoffet/notifications-api/.venv" + +set _OLD_VIRTUAL_PATH="$PATH" +setenv PATH "$VIRTUAL_ENV/bin:$PATH" + + +set _OLD_VIRTUAL_PROMPT="$prompt" + +if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "(.venv) $prompt" + setenv VIRTUAL_ENV_PROMPT "(.venv) " +endif + +alias pydoc python -m pydoc + +rehash diff --git a/.venv/bin/activate.fish b/.venv/bin/activate.fish new file mode 100644 index 000000000..579f1fae1 --- /dev/null +++ b/.venv/bin/activate.fish @@ -0,0 +1,66 @@ +# This file must be used with "source /bin/activate.fish" *from fish* +# (https://fishshell.com/); you cannot run it directly. + +function deactivate -d "Exit virtual environment and return to normal shell environment" + # reset old environment variables + if test -n "$_OLD_VIRTUAL_PATH" + set -gx PATH $_OLD_VIRTUAL_PATH + set -e _OLD_VIRTUAL_PATH + end + if test -n "$_OLD_VIRTUAL_PYTHONHOME" + set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME + set -e _OLD_VIRTUAL_PYTHONHOME + end + + if test -n "$_OLD_FISH_PROMPT_OVERRIDE" + functions -e fish_prompt + set -e _OLD_FISH_PROMPT_OVERRIDE + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end + + set -e VIRTUAL_ENV + set -e VIRTUAL_ENV_PROMPT + if test "$argv[1]" != "nondestructive" + # Self-destruct! + functions -e deactivate + end +end + +# Unset irrelevant variables. +deactivate nondestructive + +set -gx VIRTUAL_ENV "/Users/jamesdmoffet/notifications-api/.venv" + +set -gx _OLD_VIRTUAL_PATH $PATH +set -gx PATH "$VIRTUAL_ENV/bin" $PATH + +# Unset PYTHONHOME if set. +if set -q PYTHONHOME + set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME + set -e PYTHONHOME +end + +if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" + # fish uses a function instead of an env var to generate the prompt. + + # Save the current fish_prompt function as the function _old_fish_prompt. + functions -c fish_prompt _old_fish_prompt + + # With the original prompt function renamed, we can override with our own. + function fish_prompt + # Save the return status of the last command. + set -l old_status $status + + # Output the venv prompt; color taken from the blue of the Python logo. + printf "%s%s%s" (set_color 4B8BBE) "(.venv) " (set_color normal) + + # Restore the return status of the previous command. + echo "exit $old_status" | . + # Output the original/"old" prompt. + _old_fish_prompt + end + + set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" + set -gx VIRTUAL_ENV_PROMPT "(.venv) " +end diff --git a/.venv/bin/pip b/.venv/bin/pip new file mode 100755 index 000000000..d9be3c77d --- /dev/null +++ b/.venv/bin/pip @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pip-compile b/.venv/bin/pip-compile new file mode 100755 index 000000000..a5e7ad8af --- /dev/null +++ b/.venv/bin/pip-compile @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from piptools.scripts.compile import cli +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli()) diff --git a/.venv/bin/pip-sync b/.venv/bin/pip-sync new file mode 100755 index 000000000..07dcdc6d6 --- /dev/null +++ b/.venv/bin/pip-sync @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from piptools.scripts.sync import cli +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(cli()) diff --git a/.venv/bin/pip3 b/.venv/bin/pip3 new file mode 100755 index 000000000..d9be3c77d --- /dev/null +++ b/.venv/bin/pip3 @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pip3.10 b/.venv/bin/pip3.10 new file mode 100755 index 000000000..d9be3c77d --- /dev/null +++ b/.venv/bin/pip3.10 @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from pip._internal.cli.main import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/bin/pyproject-build b/.venv/bin/pyproject-build new file mode 100755 index 000000000..d045f4ee0 --- /dev/null +++ b/.venv/bin/pyproject-build @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from build.__main__ import entrypoint +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(entrypoint()) diff --git a/.venv/bin/python b/.venv/bin/python new file mode 120000 index 000000000..15b49b769 --- /dev/null +++ b/.venv/bin/python @@ -0,0 +1 @@ +/Users/jamesdmoffet/.pyenv/versions/3.10.1/bin/python \ No newline at end of file diff --git a/.venv/bin/python3 b/.venv/bin/python3 new file mode 120000 index 000000000..d8654aa0e --- /dev/null +++ b/.venv/bin/python3 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/.venv/bin/python3.10 b/.venv/bin/python3.10 new file mode 120000 index 000000000..d8654aa0e --- /dev/null +++ b/.venv/bin/python3.10 @@ -0,0 +1 @@ +python \ No newline at end of file diff --git a/.venv/bin/wheel b/.venv/bin/wheel new file mode 100755 index 000000000..5dc0d7448 --- /dev/null +++ b/.venv/bin/wheel @@ -0,0 +1,8 @@ +#!/Users/jamesdmoffet/notifications-api/.venv/bin/python +# -*- coding: utf-8 -*- +import re +import sys +from wheel.cli import main +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/.venv/pyvenv.cfg b/.venv/pyvenv.cfg new file mode 100644 index 000000000..728fbf7d7 --- /dev/null +++ b/.venv/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Users/jamesdmoffet/.pyenv/versions/3.10.1/bin +include-system-site-packages = false +version = 3.10.1 diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 0b7290593..fa90ae141 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -9,8 +9,8 @@ from celery.exceptions import Retry from flask import Blueprint, current_app, json, jsonify, request from sqlalchemy.orm.exc import NoResultFound -from app import notify_celery, statsd_client, redis_store -from app.celery.validate_sns import validate_sns_message +from app import notify_celery, statsd_client +from app.celery.validate_sns_message import sns_notification_handler from app.config import QueueNames from app.dao import notifications_dao from app.errors import InvalidRequest, register_errors @@ -51,43 +51,15 @@ def verify_message_type(message_type: str): # got refactored into a task, which is fine, but it created a circular dependency. Will need # to investigate why GDS extracted this into a lambda @ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) -def sns_callback_handler(): - message_type = request.headers.get('x-amz-sns-message-type') +def email_ses_callback_handler(): try: - verify_message_type(message_type) - except InvalidMessageTypeException: - current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") + data = sns_notification_handler(request.data, request.headers) + except Exception as e: raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) - - try: - message = json.loads(request.data.decode('utf-8')) - except decoder.JSONDecodeError: - current_app.logger.exception(f"Response headers: {request.headers}\nResponse data: {request.data}") - raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) - - try: - validate_sns_message(message) - except Exception as err: - current_app.logger.error(f"SES-SNS callback failed: validation failed! Response headers: {request.headers}\nResponse data: {request.data}\nError: Signature validation failed with error {err}") - raise InvalidRequest("SES-SNS callback failed: validation failed", 400) - - if message.get('Type') == 'SubscriptionConfirmation': - url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') - response = requests.get(url) - try: - response.raise_for_status() - except Exception as e: - current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}") - raise e - - return jsonify( - result="success", message="SES-SNS auto-confirm callback succeeded" - ), 200 - - # TODO remove after smoke testing on prod is implemented - current_app.logger.info(f"SNS message: {message} is a valid delivery status message. Attempting to process it now.") - process_ses_results.apply_async([{"Message": message.get("Message")}], queue=QueueNames.NOTIFY) + message = data.get("Message") + if "mail" in message: + process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) return jsonify( result="success", message="SES-SNS callback succeeded" diff --git a/app/celery/validate_sns.py b/app/celery/validate_sns_cert.py similarity index 97% rename from app/celery/validate_sns.py rename to app/celery/validate_sns_cert.py index e639b17a1..28549d521 100644 --- a/app/celery/validate_sns.py +++ b/app/celery/validate_sns_cert.py @@ -69,10 +69,11 @@ def get_string_to_sign(sns_payload): return string_to_sign -def validate_sns_message(sns_payload): +def validate_sns_cert(sns_payload): """ Adapted from the solution posted at https://github.com/boto/boto3/issues/2508#issuecomment-992931814 + Modified to swap m2crypto for oscrypto """ if not isinstance(sns_payload, dict): raise ValidationError("Unexpected message type {!r}".format(type(sns_payload).__name__)) diff --git a/app/celery/validate_sns_message.py b/app/celery/validate_sns_message.py new file mode 100644 index 000000000..37295dc7d --- /dev/null +++ b/app/celery/validate_sns_message.py @@ -0,0 +1,66 @@ +import enum +from datetime import timedelta +from json import decoder + +import requests +from flask import current_app, json + +from app.celery.validate_sns_cert import validate_sns_cert +from app.errors import InvalidRequest + + +DEFAULT_MAX_AGE = timedelta(days=10000) + + +class SNSMessageType(enum.Enum): + SubscriptionConfirmation = 'SubscriptionConfirmation' + Notification = 'Notification' + UnsubscribeConfirmation = 'UnsubscribeConfirmation' + + +class InvalidMessageTypeException(Exception): + pass + + +def verify_message_type(message_type: str): + try: + SNSMessageType(message_type) + except ValueError: + raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + + +def sns_notification_handler(data, headers): + message_type = headers.get('x-amz-sns-message-type') + try: + verify_message_type(message_type) + except InvalidMessageTypeException: + current_app.logger.exception(f"Response headers: {headers}\nResponse data: {data}") + raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + + try: + message = json.loads(data.decode('utf-8')) + except decoder.JSONDecodeError: + current_app.logger.exception(f"Response headers: {headers}\nResponse data: {data}") + raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400) + + try: + validate_sns_cert(message) + except Exception as e: + current_app.logger.error(f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}") + raise InvalidRequest("SES-SNS callback failed: validation failed", 400) + + if message.get('Type') == 'SubscriptionConfirmation': + url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') + response = requests.get(url) + try: + response.raise_for_status() + except Exception as e: + current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}") + raise InvalidRequest("SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation Type message failed", 400) + current_app.logger.info("SES-SNS auto-confirm subscription callback succeeded") + return message + + # TODO remove after smoke testing on prod is implemented + current_app.logger.info(f"SNS message: {message} is a valid message. Attempting to process it now.") + + return message diff --git a/app/config.py b/app/config.py index 7aaf81ae9..07e4a0fd8 100644 --- a/app/config.py +++ b/app/config.py @@ -121,7 +121,7 @@ class Config(object): NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' # AWS SNS topics for delivery receipts - VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint'] + VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] # URL of redis instance REDIS_URL = os.environ.get('REDIS_URL') @@ -196,7 +196,7 @@ class Config(object): MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71' MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' - NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' + NOTIFY_INTERNATIONAL_SMS_SENDER = '18446120782' LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea' # we only need real email in Live environment (production) diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py index bf34fc553..9afc90425 100644 --- a/app/inbound_sms/rest.py +++ b/app/inbound_sms/rest.py @@ -30,9 +30,10 @@ def post_inbound_sms_for_service(service_id): form = validate(request.get_json(), get_inbound_sms_for_service_schema) user_number = form.get('phone_number') - if user_number: - # we use this to normalise to an international phone number - but this may fail if it's an alphanumeric - user_number = try_validate_and_format_phone_number(user_number, international=True) + # TODO update this for US formatting + # if user_number: + # # we use this to normalise to an international phone number - but this may fail if it's an alphanumeric + # user_number = try_validate_and_format_phone_number(user_number, international=True) inbound_data_retention = fetch_service_data_retention_by_notification_type(service_id, 'sms') limit_days = inbound_data_retention.days_of_retention if inbound_data_retention else 7 diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 72103c20f..d7f94e755 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -2,15 +2,16 @@ from datetime import datetime from urllib.parse import unquote import iso8601 -from flask import Blueprint, abort, current_app, jsonify, request +from flask import Blueprint, abort, current_app, jsonify, request, json from gds_metrics.metrics import Counter from notifications_utils.recipients import try_validate_and_format_phone_number from app.celery import tasks +from app.celery.validate_sns_message import sns_notification_handler from app.config import QueueNames from app.dao.inbound_sms_dao import dao_create_inbound_sms from app.dao.services_dao import dao_fetch_service_by_inbound_number -from app.errors import register_errors +from app.errors import register_errors, InvalidRequest from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms receive_notifications_blueprint = Blueprint('receive_notifications', __name__) @@ -23,6 +24,64 @@ INBOUND_SMS_COUNTER = Counter( ['provider'] ) +@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST']) +def receive_sns_sms(): + """ + { + "originationNumber":"+14255550182", + "destinationNumber":"+12125550101", + "messageKeyword":"JOIN", # unique to our sending number + "messageBody":"EXAMPLE", + "inboundMessageId":"cae173d2-66b9-564c-8309-21f858e9fb84", + "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + } + """ + + try: + post_data = sns_notification_handler(request.data, request.headers) + except Exception as e: + raise InvalidRequest(f"SMS-SNS callback failed with error: {e}", 400) + + message = json.loads(post_data.get("Message")) + # TODO wrap this up + if "inboundMessageId" in message: + # TODO use standard formatting we use for all US numbers + inbound_number = message['destinationNumber'].replace('+','') + + service = fetch_potential_service(inbound_number, 'sns') + if not service: + # since this is an issue with our service <-> number mapping, or no inbound_sms service permission + # we should still tell SNS that we received it successfully + current_app.logger.warning(f"Mapping between service and inbound number: {inbound_number} is broken, or service does not have permission to receive inbound sms") + return jsonify( + result="success", message="SMS-SNS callback succeeded" + ), 200 + + INBOUND_SMS_COUNTER.labels("sns").inc() + + content = message.get("messageBody") + from_number = message.get('originationNumber') + provider_ref = message.get('inboundMessageId') + date_received = post_data.get('Timestamp') + provider_name = "sns" + + inbound = create_inbound_sms_object(service, + content=content, + from_number=from_number, + provider_ref=provider_ref, + date_received=date_received, + provider_name=provider_name) + + # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users, then uncomment the task below + tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) + + current_app.logger.debug( + '{} received inbound SMS with reference {} from SNS'.format(service.id, inbound.provider_reference)) + + return jsonify( + result="success", message="SMS-SNS callback succeeded" + ), 200 + @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) def receive_mmg_sms(): diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py new file mode 100644 index 000000000..b3641d843 --- /dev/null +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -0,0 +1,52 @@ +"""empty message + +Revision ID: 0377_add_inbound_sms_number +Revises: 0376_add_provider_response +Create Date: 2022-09-30 11:04:15.888017 + +""" +import uuid + +from alembic import op +from flask import current_app + + +revision = '0377_add_inbound_sms_number' +down_revision = '0376_add_provider_response' + +INBOUND_NUMBER_ID = '9b5bc009-b847-4b1f-8a54-f3b5f95cff18' +INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] +DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] + +def upgrade(): + op.get_bind() + + # add the inbound number for the default service to inbound_numbers + table_name = 'inbound_numbers' + provider = 'sns' + active = 'true' + op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") + + # add the inbound number for the default service to service_sms_senders + table_name = 'service_sms_senders' + id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' + is_default = 'true' + sms_sender = INBOUND_NUMBER + inbound_number_id = INBOUND_NUMBER_ID + archived = 'false' + op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") + + # add the inbound number for the default service to inbound_numbers + table_name = 'service_permissions' + permission = 'inbound_sms' + active = 'true' + op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + + +def downgrade(): + delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" + delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" + delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" + op.execute(delete_sms_sender) + op.execute(delete_inbound_number) + op.execute(delete_service_inbound_permission) From c04d1df6b3f9a010c7c7361c72ecc9ecac3d0f1d Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Mon, 3 Oct 2022 17:16:59 -0700 Subject: [PATCH 12/65] fixing tests --- app/__init__.py | 4 +- app/celery/nightly_tasks.py | 4 +- app/celery/process_ses_receipts_tasks.py | 186 +++++++++++++----- .../process_sms_client_response_tasks.py | 4 +- .../notifications_ses_callback.py | 143 +++----------- .../notifications_sms_callback.py | 1 + app/notifications/receive_notifications.py | 142 ++++++------- .../sns_cert_validator.py} | 0 .../sns_handlers.py} | 4 +- devcontainer-api/.devcontainer.json | 3 +- migrations/versions/0375_fix_service_name.py | 8 +- .../versions/0377_add_inbound_sms_number.py | 52 ++--- .../celery/test_process_ses_receipts_tasks.py | 16 +- .../test_notifications_ses_callback.py | 6 +- 14 files changed, 279 insertions(+), 294 deletions(-) rename app/{celery/validate_sns_cert.py => notifications/sns_cert_validator.py} (100%) rename app/{celery/validate_sns_message.py => notifications/sns_handlers.py} (93%) diff --git a/app/__init__.py b/app/__init__.py index 035ce112a..4d3e47721 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -139,7 +139,6 @@ def register_blueprint(application): ) from app.billing.rest import billing_blueprint from app.broadcast_message.rest import broadcast_message_blueprint - from app.celery.process_ses_receipts_tasks import ses_callback_blueprint from app.complaint.complaint_rest import complaint_blueprint from app.email_branding.rest import email_branding_blueprint from app.events.rest import events as events_blueprint @@ -154,6 +153,9 @@ def register_blueprint(application): from app.notifications.notifications_letter_callback import ( letter_callback_blueprint, ) + from app.notifications.notifications_ses_callback import ( + ses_callback_blueprint, + ) from app.notifications.notifications_sms_callback import ( sms_callback_blueprint, ) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index bcca9818e..4dd8f1a78 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -11,6 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError from app import notify_celery, statsd_client, zendesk_client from app.aws import s3 +from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task from app.config import QueueNames from app.cronitor import cronitor from app.dao.fact_processing_time_dao import insert_update_processing_time @@ -37,9 +38,6 @@ from app.models import ( FactProcessingTime, Notification, ) -from app.notifications.notifications_ses_callback import ( - check_and_queue_callback_task, -) from app.utils import get_london_midnight_in_utc diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 6c101893c..95aa92a86 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -6,64 +6,26 @@ from json import decoder import iso8601 import requests from celery.exceptions import Retry -from flask import Blueprint, current_app, json, jsonify, request +from flask import current_app, json from sqlalchemy.orm.exc import NoResultFound from app import notify_celery, statsd_client -from app.celery.validate_sns_message import sns_notification_handler +from app.celery.service_callback_tasks import ( + create_complaint_callback_data, + create_delivery_status_callback_data, + send_complaint_to_service, + send_delivery_status_to_service, +) from app.config import QueueNames from app.dao import notifications_dao -from app.errors import InvalidRequest, register_errors -from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING -from app.notifications.notifications_ses_callback import ( - _check_and_queue_complaint_callback_task, - check_and_queue_callback_task, - determine_notification_bounce_type, - get_aws_responses, - handle_complaint, +from app.dao.complaint_dao import save_complaint +from app.dao.notifications_dao import dao_get_notification_history_by_reference +from app.dao.service_callback_api_dao import ( + get_service_complaint_callback_api_for_service, + get_service_delivery_status_callback_api_for_service, ) - -ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) -DEFAULT_MAX_AGE = timedelta(days=10000) - -register_errors(ses_callback_blueprint) -class SNSMessageType(enum.Enum): - SubscriptionConfirmation = 'SubscriptionConfirmation' - Notification = 'Notification' - UnsubscribeConfirmation = 'UnsubscribeConfirmation' - - -class InvalidMessageTypeException(Exception): - pass - - -def verify_message_type(message_type: str): - try: - SNSMessageType(message_type) - except ValueError: - raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') - - -# 400 counts as a permanent failure so SNS will not retry. -# 500 counts as a failed delivery attempt so SNS will retry. -# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies -# This should not be here, it used to be in notifications/notifications_ses_callback. It then -# got refactored into a task, which is fine, but it created a circular dependency. Will need -# to investigate why GDS extracted this into a lambda -@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) -def email_ses_callback_handler(): - try: - data = sns_notification_handler(request.data, request.headers) - except Exception as e: - raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) - - message = data.get("Message") - if "mail" in message: - process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) - - return jsonify( - result="success", message="SES-SNS callback succeeded" - ), 200 +from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint +from app.notifications.callbacks import create_complaint_callback_data @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) @@ -145,3 +107,123 @@ def process_ses_results(self, response): current_app.logger.exception("Error processing SES results: {}".format(type(e))) self.retry(queue=QueueNames.RETRY) +def determine_notification_bounce_type(ses_message): + notification_type = ses_message["notificationType"] + if notification_type in ["Delivery", "Complaint"]: + return notification_type + + if notification_type != "Bounce": + raise KeyError(f"Unhandled notification type {notification_type}") + + remove_emails_from_bounce(ses_message) + current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + if ses_message["bounce"]["bounceType"] == "Permanent": + return "Permanent" + return "Temporary" + + +def _determine_provider_response(ses_message): + if ses_message["notificationType"] != "Bounce": + return None + + bounce_type = ses_message["bounce"]["bounceType"] + bounce_subtype = ses_message["bounce"]["bounceSubType"] + + # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html + if bounce_type == "Permanent" and bounce_subtype == "Suppressed": + return "The email address is on our email provider suppression list" + elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList": + return "The email address is on the GC Notify suppression list" + elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected": + return "The email was rejected because of its attachments" + + return None + + +def get_aws_responses(ses_message): + status = determine_notification_bounce_type(ses_message) + + base = { + "Permanent": { + "message": "Hard bounced", + "success": False, + "notification_status": "permanent-failure", + }, + "Temporary": { + "message": "Soft bounced", + "success": False, + "notification_status": "temporary-failure", + }, + "Delivery": { + "message": "Delivered", + "success": True, + "notification_status": "delivered", + }, + "Complaint": { + "message": "Complaint", + "success": True, + "notification_status": "delivered", + }, + }[status] + + base["provider_response"] = _determine_provider_response(ses_message) + + return base + + +def handle_complaint(ses_message): + recipient_email = remove_emails_from_complaint(ses_message)[0] + current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + try: + reference = ses_message["mail"]["messageId"] + except KeyError as e: + current_app.logger.exception("Complaint from SES failed to get reference from message", e) + return + notification = dao_get_notification_history_by_reference(reference) + ses_complaint = ses_message.get("complaint", None) + + complaint = Complaint( + notification_id=notification.id, + service_id=notification.service_id, + ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None, + complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None, + complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None, + ) + save_complaint(complaint) + return complaint, notification, recipient_email + + +def remove_mail_headers(dict_to_edit): + if dict_to_edit["mail"].get("headers"): + dict_to_edit["mail"].pop("headers") + if dict_to_edit["mail"].get("commonHeaders"): + dict_to_edit["mail"].pop("commonHeaders") + + +def remove_emails_from_bounce(bounce_dict): + remove_mail_headers(bounce_dict) + bounce_dict["mail"].pop("destination", None) + bounce_dict["bounce"].pop("bouncedRecipients", None) + + +def remove_emails_from_complaint(complaint_dict): + remove_mail_headers(complaint_dict) + complaint_dict["complaint"].pop("complainedRecipients") + return complaint_dict["mail"].pop("destination") + + +def check_and_queue_callback_task(notification): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + notification_data = create_delivery_status_callback_data(notification, service_callback_api) + send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) + + +def _check_and_queue_complaint_callback_task(complaint, notification, recipient): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) + send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) + \ No newline at end of file diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index e7726f305..4da60d477 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -6,13 +6,11 @@ from flask import current_app from notifications_utils.template import SMSMessageTemplate from app import notify_celery, statsd_client +from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task from app.clients import ClientException from app.dao import notifications_dao from app.dao.templates_dao import dao_get_template_by_id from app.models import NOTIFICATION_PENDING -from app.notifications.notifications_ses_callback import ( - check_and_queue_callback_task, -) sms_response_mapper = { # 'MMG': get_mmg_responses, diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index 472891418..d2437b92c 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -1,5 +1,9 @@ -from flask import current_app, json +import enum +from datetime import timedelta +from flask import Blueprint, current_app, json, jsonify, request + +from app.celery.process_ses_receipts_tasks import process_ses_results from app.celery.service_callback_tasks import ( create_complaint_callback_data, create_delivery_status_callback_data, @@ -13,127 +17,28 @@ from app.dao.service_callback_api_dao import ( get_service_complaint_callback_api_for_service, get_service_delivery_status_callback_api_for_service, ) +from app.errors import InvalidRequest from app.models import Complaint from app.notifications.callbacks import create_complaint_callback_data +from app.notifications.sns_handlers import sns_notification_handler +ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) +DEFAULT_MAX_AGE = timedelta(days=10000) -def determine_notification_bounce_type(ses_message): - notification_type = ses_message["notificationType"] - if notification_type in ["Delivery", "Complaint"]: - return notification_type - - if notification_type != "Bounce": - raise KeyError(f"Unhandled notification type {notification_type}") - - remove_emails_from_bounce(ses_message) - current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) - if ses_message["bounce"]["bounceType"] == "Permanent": - return "Permanent" - return "Temporary" - - -def _determine_provider_response(ses_message): - if ses_message["notificationType"] != "Bounce": - return None - - bounce_type = ses_message["bounce"]["bounceType"] - bounce_subtype = ses_message["bounce"]["bounceSubType"] - - # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html - if bounce_type == "Permanent" and bounce_subtype == "Suppressed": - return "The email address is on our email provider suppression list" - elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList": - return "The email address is on the GC Notify suppression list" - elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected": - return "The email was rejected because of its attachments" - - return None - - -def get_aws_responses(ses_message): - status = determine_notification_bounce_type(ses_message) - - base = { - "Permanent": { - "message": "Hard bounced", - "success": False, - "notification_status": "permanent-failure", - }, - "Temporary": { - "message": "Soft bounced", - "success": False, - "notification_status": "temporary-failure", - }, - "Delivery": { - "message": "Delivered", - "success": True, - "notification_status": "delivered", - }, - "Complaint": { - "message": "Complaint", - "success": True, - "notification_status": "delivered", - }, - }[status] - - base["provider_response"] = _determine_provider_response(ses_message) - - return base - - -def handle_complaint(ses_message): - recipient_email = remove_emails_from_complaint(ses_message)[0] - current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) +# 400 counts as a permanent failure so SNS will not retry. +# 500 counts as a failed delivery attempt so SNS will retry. +# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies +@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) +def email_ses_callback_handler(): try: - reference = ses_message["mail"]["messageId"] - except KeyError as e: - current_app.logger.exception("Complaint from SES failed to get reference from message", e) - return - notification = dao_get_notification_history_by_reference(reference) - ses_complaint = ses_message.get("complaint", None) + data = sns_notification_handler(request.data, request.headers) + except Exception as e: + raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + + message = data.get("Message") + if "mail" in message: + process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) - complaint = Complaint( - notification_id=notification.id, - service_id=notification.service_id, - ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None, - complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None, - complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None, - ) - save_complaint(complaint) - return complaint, notification, recipient_email - - -def remove_mail_headers(dict_to_edit): - if dict_to_edit["mail"].get("headers"): - dict_to_edit["mail"].pop("headers") - if dict_to_edit["mail"].get("commonHeaders"): - dict_to_edit["mail"].pop("commonHeaders") - - -def remove_emails_from_bounce(bounce_dict): - remove_mail_headers(bounce_dict) - bounce_dict["mail"].pop("destination", None) - bounce_dict["bounce"].pop("bouncedRecipients", None) - - -def remove_emails_from_complaint(complaint_dict): - remove_mail_headers(complaint_dict) - complaint_dict["complaint"].pop("complainedRecipients") - return complaint_dict["mail"].pop("destination") - - -def check_and_queue_callback_task(notification): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - notification_data = create_delivery_status_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) - - -def _check_and_queue_complaint_callback_task(complaint, notification, recipient): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) - send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) - \ No newline at end of file + return jsonify( + result="success", message="SES-SNS callback succeeded" + ), 200 diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py index cb221d08c..50c345f49 100644 --- a/app/notifications/notifications_sms_callback.py +++ b/app/notifications/notifications_sms_callback.py @@ -9,6 +9,7 @@ from app.errors import InvalidRequest, register_errors sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms") register_errors(sms_callback_blueprint) +# TODO SNS SMS delivery receipts delivered here # @sms_callback_blueprint.route('/mmg', methods=['POST']) # def process_mmg_response(): diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index d7f94e755..4a7ecab50 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -2,17 +2,17 @@ from datetime import datetime from urllib.parse import unquote import iso8601 -from flask import Blueprint, abort, current_app, jsonify, request, json +from flask import Blueprint, abort, current_app, json, jsonify, request from gds_metrics.metrics import Counter from notifications_utils.recipients import try_validate_and_format_phone_number from app.celery import tasks -from app.celery.validate_sns_message import sns_notification_handler from app.config import QueueNames from app.dao.inbound_sms_dao import dao_create_inbound_sms from app.dao.services_dao import dao_fetch_service_by_inbound_number -from app.errors import register_errors, InvalidRequest +from app.errors import InvalidRequest, register_errors from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms +from app.notifications.sns_handlers import sns_notification_handler receive_notifications_blueprint = Blueprint('receive_notifications', __name__) register_errors(receive_notifications_blueprint) @@ -83,90 +83,90 @@ def receive_sns_sms(): ), 200 -@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) -def receive_mmg_sms(): - """ - { - 'MSISDN': '447123456789' - 'Number': '40604', - 'Message': 'some+uri+encoded+message%3A', - 'ID': 'SOME-MMG-SPECIFIC-ID', - 'DateRecieved': '2017-05-21+11%3A56%3A11' - } - """ - post_data = request.get_json() +# @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) +# def receive_mmg_sms(): +# """ +# { +# 'MSISDN': '447123456789' +# 'Number': '40604', +# 'Message': 'some+uri+encoded+message%3A', +# 'ID': 'SOME-MMG-SPECIFIC-ID', +# 'DateRecieved': '2017-05-21+11%3A56%3A11' +# } +# """ +# post_data = request.get_json() - auth = request.authorization +# auth = request.authorization - if not auth: - current_app.logger.warning("Inbound sms (MMG) no auth header") - abort(401) - elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ - or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: - current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) - abort(403) +# if not auth: +# current_app.logger.warning("Inbound sms (MMG) no auth header") +# abort(401) +# elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ +# or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: +# current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) +# abort(403) - inbound_number = strip_leading_forty_four(post_data['Number']) +# inbound_number = strip_leading_forty_four(post_data['Number']) - service = fetch_potential_service(inbound_number, 'mmg') - if not service: - # since this is an issue with our service <-> number mapping, or no inbound_sms service permission - # we should still tell MMG that we received it successfully - return 'RECEIVED', 200 +# service = fetch_potential_service(inbound_number, 'mmg') +# if not service: +# # since this is an issue with our service <-> number mapping, or no inbound_sms service permission +# # we should still tell MMG that we received it successfully +# return 'RECEIVED', 200 - INBOUND_SMS_COUNTER.labels("mmg").inc() +# INBOUND_SMS_COUNTER.labels("mmg").inc() - inbound = create_inbound_sms_object(service, - content=format_mmg_message(post_data["Message"]), - from_number=post_data['MSISDN'], - provider_ref=post_data["ID"], - date_received=post_data.get('DateRecieved'), - provider_name="mmg") +# inbound = create_inbound_sms_object(service, +# content=format_mmg_message(post_data["Message"]), +# from_number=post_data['MSISDN'], +# provider_ref=post_data["ID"], +# date_received=post_data.get('DateRecieved'), +# provider_name="mmg") - tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) +# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) - current_app.logger.debug( - '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) - return jsonify({ - "status": "ok" - }), 200 +# current_app.logger.debug( +# '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) +# return jsonify({ +# "status": "ok" +# }), 200 -@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) -def receive_firetext_sms(): - post_data = request.form +# @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) +# def receive_firetext_sms(): +# post_data = request.form - auth = request.authorization - if not auth: - current_app.logger.warning("Inbound sms (Firetext) no auth header") - abort(401) - elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: - current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) - abort(403) +# auth = request.authorization +# if not auth: +# current_app.logger.warning("Inbound sms (Firetext) no auth header") +# abort(401) +# elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: +# current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) +# abort(403) - inbound_number = strip_leading_forty_four(post_data['destination']) +# inbound_number = strip_leading_forty_four(post_data['destination']) - service = fetch_potential_service(inbound_number, 'firetext') - if not service: - return jsonify({ - "status": "ok" - }), 200 +# service = fetch_potential_service(inbound_number, 'firetext') +# if not service: +# return jsonify({ +# "status": "ok" +# }), 200 - inbound = create_inbound_sms_object(service=service, - content=post_data["message"], - from_number=post_data['source'], - provider_ref=None, - date_received=post_data['time'], - provider_name="firetext") +# inbound = create_inbound_sms_object(service=service, +# content=post_data["message"], +# from_number=post_data['source'], +# provider_ref=None, +# date_received=post_data['time'], +# provider_name="firetext") - INBOUND_SMS_COUNTER.labels("firetext").inc() +# INBOUND_SMS_COUNTER.labels("firetext").inc() - tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) - current_app.logger.debug( - '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) - return jsonify({ - "status": "ok" - }), 200 +# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) +# current_app.logger.debug( +# '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) +# return jsonify({ +# "status": "ok" +# }), 200 def format_mmg_message(message): diff --git a/app/celery/validate_sns_cert.py b/app/notifications/sns_cert_validator.py similarity index 100% rename from app/celery/validate_sns_cert.py rename to app/notifications/sns_cert_validator.py diff --git a/app/celery/validate_sns_message.py b/app/notifications/sns_handlers.py similarity index 93% rename from app/celery/validate_sns_message.py rename to app/notifications/sns_handlers.py index 37295dc7d..a0a6d3b98 100644 --- a/app/celery/validate_sns_message.py +++ b/app/notifications/sns_handlers.py @@ -5,9 +5,8 @@ from json import decoder import requests from flask import current_app, json -from app.celery.validate_sns_cert import validate_sns_cert from app.errors import InvalidRequest - +from app.notifications.sns_cert_validator import validate_sns_cert DEFAULT_MAX_AGE = timedelta(days=10000) @@ -50,6 +49,7 @@ def sns_notification_handler(data, headers): raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': + # NOTE once a request is sent to SubscribeURL, AWS considers Notify a confirmed subscriber to this topic url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) try: diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json index 1d9b2d3d9..bea5f3978 100644 --- a/devcontainer-api/.devcontainer.json +++ b/devcontainer-api/.devcontainer.json @@ -16,7 +16,8 @@ "python.defaultInterpreterPath": "/usr/bin/python3", "python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint", "python.analysis.extraPaths": [ - "/home/vscode/.local/lib/python3.9/site-packages" + "/home/vscode/.local/lib/python3.9/site-packages", + "/home/vscode/.local/bin" ] }, "features": { diff --git a/migrations/versions/0375_fix_service_name.py b/migrations/versions/0375_fix_service_name.py index 6ec34574f..72e93e8ca 100644 --- a/migrations/versions/0375_fix_service_name.py +++ b/migrations/versions/0375_fix_service_name.py @@ -6,17 +6,13 @@ Create Date: 2022-08-29 11:04:15.888017 """ -# revision identifiers, used by Alembic. -from datetime import datetime - revision = '0375_fix_service_name' down_revision = '0374_fix_reg_template_history' from alembic import op -import sqlalchemy as sa +from flask import current_app -service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' -user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6' +service_id = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): op.get_bind() diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py index b3641d843..6b4a74044 100644 --- a/migrations/versions/0377_add_inbound_sms_number.py +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -19,34 +19,36 @@ INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): - op.get_bind() + # op.get_bind() - # add the inbound number for the default service to inbound_numbers - table_name = 'inbound_numbers' - provider = 'sns' - active = 'true' - op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") + # # add the inbound number for the default service to inbound_numbers + # table_name = 'inbound_numbers' + # provider = 'sns' + # active = 'true' + # op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") - # add the inbound number for the default service to service_sms_senders - table_name = 'service_sms_senders' - id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' - is_default = 'true' - sms_sender = INBOUND_NUMBER - inbound_number_id = INBOUND_NUMBER_ID - archived = 'false' - op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") + # # add the inbound number for the default service to service_sms_senders + # table_name = 'service_sms_senders' + # id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' + # is_default = 'true' + # sms_sender = INBOUND_NUMBER + # inbound_number_id = INBOUND_NUMBER_ID + # archived = 'false' + # op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") - # add the inbound number for the default service to inbound_numbers - table_name = 'service_permissions' - permission = 'inbound_sms' - active = 'true' - op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + # # add the inbound number for the default service to inbound_numbers + # table_name = 'service_permissions' + # permission = 'inbound_sms' + # active = 'true' + # op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + pass def downgrade(): - delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" - delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" - delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" - op.execute(delete_sms_sender) - op.execute(delete_inbound_number) - op.execute(delete_service_inbound_permission) + # delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" + # delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" + # delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" + # op.execute(delete_sms_sender) + # op.execute(delete_inbound_number) + # op.execute(delete_service_inbound_permission) + pass diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index c6a9bbf4b..ae6728398 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -4,7 +4,11 @@ from datetime import datetime from freezegun import freeze_time from app import encryption, statsd_client -from app.celery.process_ses_receipts_tasks import process_ses_results +from app.celery.process_ses_receipts_tasks import ( + process_ses_results, + remove_emails_from_bounce, + remove_emails_from_complaint, +) from app.celery.research_mode_tasks import ( ses_hard_bounce_callback, ses_notification_callback, @@ -15,10 +19,6 @@ from app.celery.service_callback_tasks import ( ) from app.dao.notifications_dao import get_notification_by_id from app.models import Complaint, Notification -from app.notifications.notifications_ses_callback import ( - remove_emails_from_bounce, - remove_emails_from_complaint, -) from tests.app.conftest import create_sample_notification from tests.app.db import ( create_notification, @@ -105,7 +105,7 @@ def test_process_ses_results(sample_email_template): assert process_ses_results(response=ses_notification_callback(reference='ref1')) -def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker): +def test_process_ses_results_retry_called(sample_email_template, mocker): create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending') mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED")) mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') @@ -178,7 +178,7 @@ def test_ses_callback_should_not_update_notification_status_if_already_delivered assert mock_upd.call_count == 0 -def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker): +def test_ses_callback_should_retry_if_notification_is_new(mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') with freeze_time('2017-11-17T12:14:03.646Z'): @@ -192,7 +192,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)') -def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker): +def test_ses_callback_should_not_retry_if_notification_is_old(mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') with freeze_time('2017-11-21T12:14:03.646Z'): diff --git a/tests/app/notifications/test_notifications_ses_callback.py b/tests/app/notifications/test_notifications_ses_callback.py index 8e6409eeb..dcec4aa32 100644 --- a/tests/app/notifications/test_notifications_ses_callback.py +++ b/tests/app/notifications/test_notifications_ses_callback.py @@ -2,12 +2,12 @@ import pytest from flask import json from sqlalchemy.exc import SQLAlchemyError -from app.dao.notifications_dao import get_notification_by_id -from app.models import Complaint -from app.notifications.notifications_ses_callback import ( +from app.celery.process_ses_receipts_tasks import ( check_and_queue_callback_task, handle_complaint, ) +from app.dao.notifications_dao import get_notification_by_id +from app.models import Complaint from tests.app.db import ( create_notification, create_notification_history, From fc9e4107c1d3ddc1c0b990f6bf74881622e902a3 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Mon, 3 Oct 2022 20:07:42 -0700 Subject: [PATCH 13/65] all tests passing --- app/celery/process_ses_receipts_tasks.py | 2 +- .../notifications_ses_callback.py | 6 +- app/notifications/receive_notifications.py | 136 +++++++++--------- .../versions/0377_add_inbound_sms_number.py | 54 +++---- .../celery/test_process_ses_receipts_tasks.py | 13 +- tests/app/db.py | 6 +- tests/app/inbound_sms/test_rest.py | 10 +- .../test_notifications_ses_callback.py | 3 +- 8 files changed, 118 insertions(+), 112 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 95aa92a86..22136e7a0 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -177,7 +177,7 @@ def handle_complaint(ses_message): try: reference = ses_message["mail"]["messageId"] except KeyError as e: - current_app.logger.exception("Complaint from SES failed to get reference from message", e) + current_app.logger.exception(f"Complaint from SES failed to get reference from message with error: {e}") return notification = dao_get_notification_history_by_reference(reference) ses_complaint = ses_message.get("complaint", None) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index d2437b92c..16e57384e 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -32,8 +32,10 @@ DEFAULT_MAX_AGE = timedelta(days=10000) def email_ses_callback_handler(): try: data = sns_notification_handler(request.data, request.headers) - except Exception as e: - raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + except InvalidRequest as e: + return jsonify( + result="error", message=str(e.message) + ), e.status_code message = data.get("Message") if "mail" in message: diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 4a7ecab50..f15db5306 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -83,90 +83,90 @@ def receive_sns_sms(): ), 200 -# @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) -# def receive_mmg_sms(): -# """ -# { -# 'MSISDN': '447123456789' -# 'Number': '40604', -# 'Message': 'some+uri+encoded+message%3A', -# 'ID': 'SOME-MMG-SPECIFIC-ID', -# 'DateRecieved': '2017-05-21+11%3A56%3A11' -# } -# """ -# post_data = request.get_json() +@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) +def receive_mmg_sms(): + """ + { + 'MSISDN': '447123456789' + 'Number': '40604', + 'Message': 'some+uri+encoded+message%3A', + 'ID': 'SOME-MMG-SPECIFIC-ID', + 'DateRecieved': '2017-05-21+11%3A56%3A11' + } + """ + post_data = request.get_json() -# auth = request.authorization + auth = request.authorization -# if not auth: -# current_app.logger.warning("Inbound sms (MMG) no auth header") -# abort(401) -# elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ -# or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: -# current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) -# abort(403) + if not auth: + current_app.logger.warning("Inbound sms (MMG) no auth header") + abort(401) + elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ + or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: + current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) + abort(403) -# inbound_number = strip_leading_forty_four(post_data['Number']) + inbound_number = strip_leading_forty_four(post_data['Number']) -# service = fetch_potential_service(inbound_number, 'mmg') -# if not service: -# # since this is an issue with our service <-> number mapping, or no inbound_sms service permission -# # we should still tell MMG that we received it successfully -# return 'RECEIVED', 200 + service = fetch_potential_service(inbound_number, 'mmg') + if not service: + # since this is an issue with our service <-> number mapping, or no inbound_sms service permission + # we should still tell MMG that we received it successfully + return 'RECEIVED', 200 -# INBOUND_SMS_COUNTER.labels("mmg").inc() + INBOUND_SMS_COUNTER.labels("mmg").inc() -# inbound = create_inbound_sms_object(service, -# content=format_mmg_message(post_data["Message"]), -# from_number=post_data['MSISDN'], -# provider_ref=post_data["ID"], -# date_received=post_data.get('DateRecieved'), -# provider_name="mmg") + inbound = create_inbound_sms_object(service, + content=format_mmg_message(post_data["Message"]), + from_number=post_data['MSISDN'], + provider_ref=post_data["ID"], + date_received=post_data.get('DateRecieved'), + provider_name="mmg") -# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) + tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) -# current_app.logger.debug( -# '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) -# return jsonify({ -# "status": "ok" -# }), 200 + current_app.logger.debug( + '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) + return jsonify({ + "status": "ok" + }), 200 -# @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) -# def receive_firetext_sms(): -# post_data = request.form +@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) +def receive_firetext_sms(): + post_data = request.form -# auth = request.authorization -# if not auth: -# current_app.logger.warning("Inbound sms (Firetext) no auth header") -# abort(401) -# elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: -# current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) -# abort(403) + auth = request.authorization + if not auth: + current_app.logger.warning("Inbound sms (Firetext) no auth header") + abort(401) + elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: + current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) + abort(403) -# inbound_number = strip_leading_forty_four(post_data['destination']) + inbound_number = strip_leading_forty_four(post_data['destination']) -# service = fetch_potential_service(inbound_number, 'firetext') -# if not service: -# return jsonify({ -# "status": "ok" -# }), 200 + service = fetch_potential_service(inbound_number, 'firetext') + if not service: + return jsonify({ + "status": "ok" + }), 200 -# inbound = create_inbound_sms_object(service=service, -# content=post_data["message"], -# from_number=post_data['source'], -# provider_ref=None, -# date_received=post_data['time'], -# provider_name="firetext") + inbound = create_inbound_sms_object(service=service, + content=post_data["message"], + from_number=post_data['source'], + provider_ref=None, + date_received=post_data['time'], + provider_name="firetext") -# INBOUND_SMS_COUNTER.labels("firetext").inc() + INBOUND_SMS_COUNTER.labels("firetext").inc() -# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) -# current_app.logger.debug( -# '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) -# return jsonify({ -# "status": "ok" -# }), 200 + tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) + current_app.logger.debug( + '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) + return jsonify({ + "status": "ok" + }), 200 def format_mmg_message(message): diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py index 6b4a74044..6a8ccab0f 100644 --- a/migrations/versions/0377_add_inbound_sms_number.py +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -19,36 +19,36 @@ INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): - # op.get_bind() + op.get_bind() - # # add the inbound number for the default service to inbound_numbers - # table_name = 'inbound_numbers' - # provider = 'sns' - # active = 'true' - # op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") + # add the inbound number for the default service to inbound_numbers + table_name = 'inbound_numbers' + provider = 'sns' + active = 'true' + op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") - # # add the inbound number for the default service to service_sms_senders - # table_name = 'service_sms_senders' - # id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' - # is_default = 'true' - # sms_sender = INBOUND_NUMBER - # inbound_number_id = INBOUND_NUMBER_ID - # archived = 'false' - # op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") + # add the inbound number for the default service to service_sms_senders + table_name = 'service_sms_senders' + id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' + is_default = 'true' + sms_sender = INBOUND_NUMBER + inbound_number_id = INBOUND_NUMBER_ID + archived = 'false' + op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") - # # add the inbound number for the default service to inbound_numbers - # table_name = 'service_permissions' - # permission = 'inbound_sms' - # active = 'true' - # op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") - pass + # add the inbound number for the default service to inbound_numbers + table_name = 'service_permissions' + permission = 'inbound_sms' + active = 'true' + op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + # pass def downgrade(): - # delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" - # delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" - # delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" - # op.execute(delete_sms_sender) - # op.execute(delete_inbound_number) - # op.execute(delete_service_inbound_permission) - pass + delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" + delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" + delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" + op.execute(delete_sms_sender) + op.execute(delete_inbound_number) + op.execute(delete_service_inbound_permission) + # pass diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index ae6728398..0cc82cddd 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -71,9 +71,9 @@ def test_notifications_ses_400_with_certificate(client): def test_notifications_ses_200_autoconfirms_subscription(client, mocker): - mocker.patch("app.celery.process_ses_receipts_tasks.validate_sns_message", return_value=True) + mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True) requests_mock = mocker.patch("requests.get") - data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo"}) + data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo", "Message": "foo"}) response = client.post( path='/notifications/email/ses', data=data, @@ -85,9 +85,10 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker): def test_notifications_ses_200_call_process_task(client, mocker): - mocker.patch("app.celery.process_ses_receipts_tasks.validate_sns_message", return_value=True) - process_mock = mocker.patch("app.celery.process_ses_receipts_tasks.process_ses_results.apply_async") - data = {"Type": "Notification", "foo": "bar"} + process_mock = mocker.patch("app.notifications.notifications_ses_callback.process_ses_results.apply_async") + mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True) + data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"} } + mocker.patch("app.notifications.sns_handlers.sns_notification_handler", return_value=data) json_data = json.dumps(data) response = client.post( path='/notifications/email/ses', @@ -95,7 +96,7 @@ def test_notifications_ses_200_call_process_task(client, mocker): headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')] ) - process_mock.assert_called_once_with([{'Message': None}], queue='notify-internal-tasks') + process_mock.assert_called_once_with([{'Message': {"mail": "baz"}}], queue='notify-internal-tasks') assert response.status_code == 200 diff --git a/tests/app/db.py b/tests/app/db.py index 30864e876..8dac3f22c 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -443,17 +443,17 @@ def create_service_permission(service_id, permission=EMAIL_TYPE): def create_inbound_sms( service, notify_number=None, - user_number='447700900111', + user_number='12025550104', provider_date=None, provider_reference=None, content='Hello', - provider="mmg", + provider="sns", created_at=None ): if not service.inbound_number: create_inbound_number( # create random inbound number - notify_number or '07{:09}'.format(random.randint(0, 1e9 - 1)), + notify_number or '1'+str(random.randint(1001001000, 9999999999)), provider=provider, service_id=service.id ) diff --git a/tests/app/inbound_sms/test_rest.py b/tests/app/inbound_sms/test_rest.py index a0b364267..d6ee9f2d8 100644 --- a/tests/app/inbound_sms/test_rest.py +++ b/tests/app/inbound_sms/test_rest.py @@ -39,6 +39,7 @@ def test_post_to_get_inbound_sms_with_no_params(admin_request, sample_service): '+4407700900001', '447700900001', ]) +@pytest.mark.skip(reason="Needs updating for TTS. Don't need to test UK numbers right now") def test_post_to_get_inbound_sms_filters_user_number(admin_request, sample_service, user_number): # user_number in the db is international and normalised one = create_inbound_sms(sample_service, user_number='447700900001') @@ -65,7 +66,7 @@ def test_post_to_get_inbound_sms_filters_international_user_number(admin_request create_inbound_sms(sample_service) data = { - 'phone_number': '+1 (202) 555-0104' + 'phone_number': '12025550104' } sms = admin_request.post( @@ -74,9 +75,10 @@ def test_post_to_get_inbound_sms_filters_international_user_number(admin_request _data=data )['data'] - assert len(sms) == 1 - assert sms[0]['id'] == str(one.id) - assert sms[0]['user_number'] == str(one.user_number) + assert len(sms) == 2 + print(f'sms is: {sms}') + assert sms[1]['id'] == str(one.id) + assert sms[1]['user_number'] == str(one.user_number) def test_post_to_get_inbound_sms_allows_badly_formatted_number(admin_request, sample_service): diff --git a/tests/app/notifications/test_notifications_ses_callback.py b/tests/app/notifications/test_notifications_ses_callback.py index dcec4aa32..1bff3bab2 100644 --- a/tests/app/notifications/test_notifications_ses_callback.py +++ b/tests/app/notifications/test_notifications_ses_callback.py @@ -72,7 +72,7 @@ def test_process_ses_results_in_complaint_save_complaint_with_null_complaint_typ def test_check_and_queue_callback_task(mocker, sample_notification): mock_create = mocker.patch( - 'app.notifications.notifications_ses_callback.create_delivery_status_callback_data' + 'app.celery.process_ses_receipts_tasks.create_delivery_status_callback_data' ) mock_send = mocker.patch( @@ -86,6 +86,7 @@ def test_check_and_queue_callback_task(mocker, sample_notification): # callback_api doesn't match by equality for some # reason, so we need to take this approach instead + print(f'mock_create.mock_calls is: {mock_create.mock_calls}') mock_create_args = mock_create.mock_calls[0][1] assert mock_create_args[0] == sample_notification assert mock_create_args[1].id == callback_api.id From b87217c3bffd396a9029aa28f38fac04f10004e2 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Mon, 3 Oct 2022 20:32:40 -0700 Subject: [PATCH 14/65] remove .venv --- .venv/bin/Activate.ps1 | 247 -------------------------------------- .venv/bin/activate | 69 ----------- .venv/bin/activate.csh | 26 ---- .venv/bin/activate.fish | 66 ---------- .venv/bin/pip | 8 -- .venv/bin/pip-compile | 8 -- .venv/bin/pip-sync | 8 -- .venv/bin/pip3 | 8 -- .venv/bin/pip3.10 | 8 -- .venv/bin/pyproject-build | 8 -- .venv/bin/python | 1 - .venv/bin/python3 | 1 - .venv/bin/python3.10 | 1 - .venv/bin/wheel | 8 -- .venv/pyvenv.cfg | 3 - 15 files changed, 470 deletions(-) delete mode 100644 .venv/bin/Activate.ps1 delete mode 100644 .venv/bin/activate delete mode 100644 .venv/bin/activate.csh delete mode 100644 .venv/bin/activate.fish delete mode 100755 .venv/bin/pip delete mode 100755 .venv/bin/pip-compile delete mode 100755 .venv/bin/pip-sync delete mode 100755 .venv/bin/pip3 delete mode 100755 .venv/bin/pip3.10 delete mode 100755 .venv/bin/pyproject-build delete mode 120000 .venv/bin/python delete mode 120000 .venv/bin/python3 delete mode 120000 .venv/bin/python3.10 delete mode 100755 .venv/bin/wheel delete mode 100644 .venv/pyvenv.cfg diff --git a/.venv/bin/Activate.ps1 b/.venv/bin/Activate.ps1 deleted file mode 100644 index b49d77ba4..000000000 --- a/.venv/bin/Activate.ps1 +++ /dev/null @@ -1,247 +0,0 @@ -<# -.Synopsis -Activate a Python virtual environment for the current PowerShell session. - -.Description -Pushes the python executable for a virtual environment to the front of the -$Env:PATH environment variable and sets the prompt to signify that you are -in a Python virtual environment. Makes use of the command line switches as -well as the `pyvenv.cfg` file values present in the virtual environment. - -.Parameter VenvDir -Path to the directory that contains the virtual environment to activate. The -default value for this is the parent of the directory that the Activate.ps1 -script is located within. - -.Parameter Prompt -The prompt prefix to display when this virtual environment is activated. By -default, this prompt is the name of the virtual environment folder (VenvDir) -surrounded by parentheses and followed by a single space (ie. '(.venv) '). - -.Example -Activate.ps1 -Activates the Python virtual environment that contains the Activate.ps1 script. - -.Example -Activate.ps1 -Verbose -Activates the Python virtual environment that contains the Activate.ps1 script, -and shows extra information about the activation as it executes. - -.Example -Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv -Activates the Python virtual environment located in the specified location. - -.Example -Activate.ps1 -Prompt "MyPython" -Activates the Python virtual environment that contains the Activate.ps1 script, -and prefixes the current prompt with the specified string (surrounded in -parentheses) while the virtual environment is active. - -.Notes -On Windows, it may be required to enable this Activate.ps1 script by setting the -execution policy for the user. You can do this by issuing the following PowerShell -command: - -PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - -For more information on Execution Policies: -https://go.microsoft.com/fwlink/?LinkID=135170 - -#> -Param( - [Parameter(Mandatory = $false)] - [String] - $VenvDir, - [Parameter(Mandatory = $false)] - [String] - $Prompt -) - -<# Function declarations --------------------------------------------------- #> - -<# -.Synopsis -Remove all shell session elements added by the Activate script, including the -addition of the virtual environment's Python executable from the beginning of -the PATH variable. - -.Parameter NonDestructive -If present, do not remove this function from the global namespace for the -session. - -#> -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - - # The prior prompt: - if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { - Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt - Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT - } - - # The prior PYTHONHOME: - if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { - Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME - Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME - } - - # The prior PATH: - if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { - Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH - Remove-Item -Path Env:_OLD_VIRTUAL_PATH - } - - # Just remove the VIRTUAL_ENV altogether: - if (Test-Path -Path Env:VIRTUAL_ENV) { - Remove-Item -Path env:VIRTUAL_ENV - } - - # Just remove VIRTUAL_ENV_PROMPT altogether. - if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { - Remove-Item -Path env:VIRTUAL_ENV_PROMPT - } - - # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: - if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { - Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force - } - - # Leave deactivate function in the global namespace if requested: - if (-not $NonDestructive) { - Remove-Item -Path function:deactivate - } -} - -<# -.Description -Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the -given folder, and returns them in a map. - -For each line in the pyvenv.cfg file, if that line can be parsed into exactly -two strings separated by `=` (with any amount of whitespace surrounding the =) -then it is considered a `key = value` line. The left hand string is the key, -the right hand is the value. - -If the value starts with a `'` or a `"` then the first and last character is -stripped from the value before being captured. - -.Parameter ConfigDir -Path to the directory that contains the `pyvenv.cfg` file. -#> -function Get-PyVenvConfig( - [String] - $ConfigDir -) { - Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" - - # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). - $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue - - # An empty map will be returned if no config file is found. - $pyvenvConfig = @{ } - - if ($pyvenvConfigPath) { - - Write-Verbose "File exists, parse `key = value` lines" - $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath - - $pyvenvConfigContent | ForEach-Object { - $keyval = $PSItem -split "\s*=\s*", 2 - if ($keyval[0] -and $keyval[1]) { - $val = $keyval[1] - - # Remove extraneous quotations around a string value. - if ("'""".Contains($val.Substring(0, 1))) { - $val = $val.Substring(1, $val.Length - 2) - } - - $pyvenvConfig[$keyval[0]] = $val - Write-Verbose "Adding Key: '$($keyval[0])'='$val'" - } - } - } - return $pyvenvConfig -} - - -<# Begin Activate script --------------------------------------------------- #> - -# Determine the containing directory of this script -$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$VenvExecDir = Get-Item -Path $VenvExecPath - -Write-Verbose "Activation script is located in path: '$VenvExecPath'" -Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" -Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" - -# Set values required in priority: CmdLine, ConfigFile, Default -# First, get the location of the virtual environment, it might not be -# VenvExecDir if specified on the command line. -if ($VenvDir) { - Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" -} -else { - Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." - $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") - Write-Verbose "VenvDir=$VenvDir" -} - -# Next, read the `pyvenv.cfg` file to determine any required value such -# as `prompt`. -$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir - -# Next, set the prompt from the command line, or the config file, or -# just use the name of the virtual environment folder. -if ($Prompt) { - Write-Verbose "Prompt specified as argument, using '$Prompt'" -} -else { - Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" - if ($pyvenvCfg -and $pyvenvCfg['prompt']) { - Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" - $Prompt = $pyvenvCfg['prompt']; - } - else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" - Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" - $Prompt = Split-Path -Path $venvDir -Leaf - } -} - -Write-Verbose "Prompt = '$Prompt'" -Write-Verbose "VenvDir='$VenvDir'" - -# Deactivate any currently active virtual environment, but leave the -# deactivate function in place. -deactivate -nondestructive - -# Now set the environment variable VIRTUAL_ENV, used by many tools to determine -# that there is an activated venv. -$env:VIRTUAL_ENV = $VenvDir - -if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { - - Write-Verbose "Setting prompt to '$Prompt'" - - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT { "" } - Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT - New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt - - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " - _OLD_VIRTUAL_PROMPT - } - $env:VIRTUAL_ENV_PROMPT = $Prompt -} - -# Clear PYTHONHOME -if (Test-Path -Path Env:PYTHONHOME) { - Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME - Remove-Item -Path Env:PYTHONHOME -} - -# Add the venv to the PATH -Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH -$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/.venv/bin/activate b/.venv/bin/activate deleted file mode 100644 index d8aaaac7e..000000000 --- a/.venv/bin/activate +++ /dev/null @@ -1,69 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - unset VIRTUAL_ENV_PROMPT - if [ ! "${1:-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/Users/jamesdmoffet/notifications-api/.venv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="(.venv) ${PS1:-}" - export PS1 - VIRTUAL_ENV_PROMPT="(.venv) " - export VIRTUAL_ENV_PROMPT -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi diff --git a/.venv/bin/activate.csh b/.venv/bin/activate.csh deleted file mode 100644 index de77f1e28..000000000 --- a/.venv/bin/activate.csh +++ /dev/null @@ -1,26 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . -# Ported to Python 3.3 venv by Andrew Svetlov - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/Users/jamesdmoffet/notifications-api/.venv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "(.venv) $prompt" - setenv VIRTUAL_ENV_PROMPT "(.venv) " -endif - -alias pydoc python -m pydoc - -rehash diff --git a/.venv/bin/activate.fish b/.venv/bin/activate.fish deleted file mode 100644 index 579f1fae1..000000000 --- a/.venv/bin/activate.fish +++ /dev/null @@ -1,66 +0,0 @@ -# This file must be used with "source /bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. - -function deactivate -d "Exit virtual environment and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt - set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - - set -e VIRTUAL_ENV - set -e VIRTUAL_ENV_PROMPT - if test "$argv[1]" != "nondestructive" - # Self-destruct! - functions -e deactivate - end -end - -# Unset irrelevant variables. -deactivate nondestructive - -set -gx VIRTUAL_ENV "/Users/jamesdmoffet/notifications-api/.venv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# Unset PYTHONHOME if set. -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # Save the current fish_prompt function as the function _old_fish_prompt. - functions -c fish_prompt _old_fish_prompt - - # With the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command. - set -l old_status $status - - # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "(.venv) " (set_color normal) - - # Restore the return status of the previous command. - echo "exit $old_status" | . - # Output the original/"old" prompt. - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "(.venv) " -end diff --git a/.venv/bin/pip b/.venv/bin/pip deleted file mode 100755 index d9be3c77d..000000000 --- a/.venv/bin/pip +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/pip-compile b/.venv/bin/pip-compile deleted file mode 100755 index a5e7ad8af..000000000 --- a/.venv/bin/pip-compile +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from piptools.scripts.compile import cli -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(cli()) diff --git a/.venv/bin/pip-sync b/.venv/bin/pip-sync deleted file mode 100755 index 07dcdc6d6..000000000 --- a/.venv/bin/pip-sync +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from piptools.scripts.sync import cli -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(cli()) diff --git a/.venv/bin/pip3 b/.venv/bin/pip3 deleted file mode 100755 index d9be3c77d..000000000 --- a/.venv/bin/pip3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/pip3.10 b/.venv/bin/pip3.10 deleted file mode 100755 index d9be3c77d..000000000 --- a/.venv/bin/pip3.10 +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/bin/pyproject-build b/.venv/bin/pyproject-build deleted file mode 100755 index d045f4ee0..000000000 --- a/.venv/bin/pyproject-build +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from build.__main__ import entrypoint -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(entrypoint()) diff --git a/.venv/bin/python b/.venv/bin/python deleted file mode 120000 index 15b49b769..000000000 --- a/.venv/bin/python +++ /dev/null @@ -1 +0,0 @@ -/Users/jamesdmoffet/.pyenv/versions/3.10.1/bin/python \ No newline at end of file diff --git a/.venv/bin/python3 b/.venv/bin/python3 deleted file mode 120000 index d8654aa0e..000000000 --- a/.venv/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -python \ No newline at end of file diff --git a/.venv/bin/python3.10 b/.venv/bin/python3.10 deleted file mode 120000 index d8654aa0e..000000000 --- a/.venv/bin/python3.10 +++ /dev/null @@ -1 +0,0 @@ -python \ No newline at end of file diff --git a/.venv/bin/wheel b/.venv/bin/wheel deleted file mode 100755 index 5dc0d7448..000000000 --- a/.venv/bin/wheel +++ /dev/null @@ -1,8 +0,0 @@ -#!/Users/jamesdmoffet/notifications-api/.venv/bin/python -# -*- coding: utf-8 -*- -import re -import sys -from wheel.cli import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/.venv/pyvenv.cfg b/.venv/pyvenv.cfg deleted file mode 100644 index 728fbf7d7..000000000 --- a/.venv/pyvenv.cfg +++ /dev/null @@ -1,3 +0,0 @@ -home = /Users/jamesdmoffet/.pyenv/versions/3.10.1/bin -include-system-site-packages = false -version = 3.10.1 From 57f4df8ed1fa2328bee483c54d704e77f2d142c5 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 4 Oct 2022 15:28:27 +0000 Subject: [PATCH 15/65] remove broadcast-related code, except migrations --- .github/workflows/checks.yml | 1 - .github/workflows/daily_checks.yml | 1 - app/__init__.py | 17 - app/authentication/auth.py | 5 - app/broadcast_message/__init__.py | 0 .../broadcast_message_schema.py | 60 -- app/broadcast_message/rest.py | 165 ---- app/broadcast_message/translators.py | 37 - app/broadcast_message/utils.py | 140 ---- app/celery/broadcast_message_tasks.py | 230 ------ app/celery/scheduled_tasks.py | 37 - app/clients/cbc_proxy.py | 299 -------- app/commands.py | 30 - app/config.py | 38 - app/dao/broadcast_message_dao.py | 105 --- app/dao/broadcast_service_dao.py | 100 --- app/dao/templates_dao.py | 1 - app/govuk_alerts/__init__.py | 0 app/govuk_alerts/rest.py | 31 - app/models.py | 423 +--------- app/schemas.py | 23 +- app/service/rest.py | 29 - .../service_broadcast_settings_schema.py | 12 - app/service_invite/rest.py | 7 +- app/template/rest.py | 9 +- app/utils.py | 9 +- app/v2/broadcast/__init__.py | 11 - app/v2/broadcast/broadcast_schemas.py | 123 --- app/v2/broadcast/post_broadcast.py | 148 ---- app/xml_schemas/CAP-v1.2.xsd | 218 ------ app/xml_schemas/__init__.py | 15 - docs/writing-public-apis.md | 4 +- sample.env | 1 - scripts/paas_app_wrapper.sh | 4 - .../app/authentication/test_authentication.py | 8 - tests/app/broadcast_message/__init__.py | 0 tests/app/broadcast_message/test_rest.py | 658 ---------------- tests/app/broadcast_message/test_utils.py | 455 ----------- .../celery/test_broadcast_message_tasks.py | 726 ------------------ tests/app/celery/test_scheduled_tasks.py | 82 -- tests/app/clients/test_cbc_proxy.py | 637 --------------- tests/app/conftest.py | 81 -- tests/app/dao/test_broadcast_message_dao.py | 131 ---- tests/app/db.py | 105 --- tests/app/govuk_alerts/__init__.py | 0 tests/app/govuk_alerts/test_get_broadcasts.py | 41 - tests/app/service/test_rest.py | 522 ------------- .../test_service_invite_rest.py | 54 -- tests/app/template/test_rest.py | 7 +- tests/app/test_commands.py | 21 - tests/app/test_config.py | 1 - tests/app/test_model.py | 4 - tests/app/v2/broadcast/__init__.py | 0 .../v2/broadcast/sample_cap_xml_documents.py | 252 ------ tests/app/v2/broadcast/test_post_broadcast.py | 487 ------------ tests/app/v2/templates/test_get_templates.py | 2 +- .../v2/templates/test_templates_schemas.py | 2 +- tests/conftest.py | 5 +- 58 files changed, 14 insertions(+), 6600 deletions(-) delete mode 100644 app/broadcast_message/__init__.py delete mode 100644 app/broadcast_message/broadcast_message_schema.py delete mode 100644 app/broadcast_message/rest.py delete mode 100644 app/broadcast_message/translators.py delete mode 100644 app/broadcast_message/utils.py delete mode 100644 app/celery/broadcast_message_tasks.py delete mode 100644 app/clients/cbc_proxy.py delete mode 100644 app/dao/broadcast_message_dao.py delete mode 100644 app/dao/broadcast_service_dao.py delete mode 100644 app/govuk_alerts/__init__.py delete mode 100644 app/govuk_alerts/rest.py delete mode 100644 app/service/service_broadcast_settings_schema.py delete mode 100644 app/v2/broadcast/__init__.py delete mode 100644 app/v2/broadcast/broadcast_schemas.py delete mode 100644 app/v2/broadcast/post_broadcast.py delete mode 100644 app/xml_schemas/CAP-v1.2.xsd delete mode 100644 app/xml_schemas/__init__.py delete mode 100644 tests/app/broadcast_message/__init__.py delete mode 100644 tests/app/broadcast_message/test_rest.py delete mode 100644 tests/app/broadcast_message/test_utils.py delete mode 100644 tests/app/celery/test_broadcast_message_tasks.py delete mode 100644 tests/app/clients/test_cbc_proxy.py delete mode 100644 tests/app/dao/test_broadcast_message_dao.py delete mode 100644 tests/app/govuk_alerts/__init__.py delete mode 100644 tests/app/govuk_alerts/test_get_broadcasts.py delete mode 100644 tests/app/v2/broadcast/__init__.py delete mode 100644 tests/app/v2/broadcast/sample_cap_xml_documents.py delete mode 100644 tests/app/v2/broadcast/test_post_broadcast.py diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8cbbda589..79ddb2d32 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,7 +16,6 @@ env: NOTIFY_LOG_PATH: /workspace/logs/app.log ADMIN_CLIENT_ID: notify-admin ADMIN_CLIENT_SECRET: dev-notify-secret-key - GOVUK_ALERTS_CLIENT_ID: govuk-alerts FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 3846c3a79..f33178988 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -20,7 +20,6 @@ env: NOTIFY_LOG_PATH: /workspace/logs/app.log ADMIN_CLIENT_ID: notify-admin ADMIN_CLIENT_SECRET: dev-notify-secret-key - GOVUK_ALERTS_CLIENT_ID: govuk-alerts FLASK_APP: application.py FLASK_ENV: development WERKZEUG_DEBUG_PIN: off diff --git a/app/__init__.py b/app/__init__.py index 2fffe2673..aa08a1a0f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -30,7 +30,6 @@ from werkzeug.exceptions import HTTPException as WerkzeugHTTPException from werkzeug.local import LocalProxy from app.clients import NotificationProviderClients -from app.clients.cbc_proxy import CBCProxyClient from app.clients.document_download import DocumentDownloadClient from app.clients.email.aws_ses import AwsSesClient from app.clients.email.aws_ses_stub import AwsSesStubClient @@ -61,7 +60,6 @@ encryption = Encryption() zendesk_client = ZendeskClient() statsd_client = StatsdClient() redis_store = RedisClient() -cbc_proxy_client = CBCProxyClient() document_download_client = DocumentDownloadClient() metrics = GDSMetrics() @@ -115,8 +113,6 @@ def create_app(application): redis_store.init_app(application) document_download_client.init_app(application) - cbc_proxy_client.init_app(application) - register_blueprint(application) register_v2_blueprints(application) @@ -134,15 +130,12 @@ def register_blueprint(application): from app.authentication.auth import ( requires_admin_auth, requires_auth, - requires_govuk_alerts_auth, requires_no_auth, ) from app.billing.rest import billing_blueprint - from app.broadcast_message.rest import broadcast_message_blueprint from app.complaint.complaint_rest import complaint_blueprint from app.email_branding.rest import email_branding_blueprint from app.events.rest import events as events_blueprint - from app.govuk_alerts.rest import govuk_alerts_blueprint from app.inbound_number.rest import inbound_number_blueprint from app.inbound_sms.rest import inbound_sms as inbound_sms_blueprint from app.job.rest import job_blueprint @@ -269,16 +262,9 @@ def register_blueprint(application): upload_blueprint.before_request(requires_admin_auth) application.register_blueprint(upload_blueprint) - broadcast_message_blueprint.before_request(requires_admin_auth) - application.register_blueprint(broadcast_message_blueprint) - - govuk_alerts_blueprint.before_request(requires_govuk_alerts_auth) - application.register_blueprint(govuk_alerts_blueprint) - def register_v2_blueprints(application): from app.authentication.auth import requires_auth - from app.v2.broadcast.post_broadcast import v2_broadcast_blueprint from app.v2.inbound_sms.get_inbound_sms import v2_inbound_sms_blueprint from app.v2.notifications import ( # noqa get_notifications, @@ -304,9 +290,6 @@ def register_v2_blueprints(application): v2_inbound_sms_blueprint.before_request(requires_auth) application.register_blueprint(v2_inbound_sms_blueprint) - v2_broadcast_blueprint.before_request(requires_auth) - application.register_blueprint(v2_broadcast_blueprint) - def init_app(app): diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 301ed7853..9849bda3e 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -59,11 +59,6 @@ class InternalApiKey(): def requires_no_auth(): pass - -def requires_govuk_alerts_auth(): - requires_internal_auth(current_app.config.get('GOVUK_ALERTS_CLIENT_ID')) - - def requires_admin_auth(): requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID')) diff --git a/app/broadcast_message/__init__.py b/app/broadcast_message/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/broadcast_message/broadcast_message_schema.py b/app/broadcast_message/broadcast_message_schema.py deleted file mode 100644 index 823d3ab1b..000000000 --- a/app/broadcast_message/broadcast_message_schema.py +++ /dev/null @@ -1,60 +0,0 @@ -from app.models import BroadcastStatusType -from app.schema_validation.definitions import uuid - -create_broadcast_message_schema = { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'description': 'POST create broadcast_message schema', - 'type': 'object', - 'title': 'Create broadcast_message', - 'properties': { - 'template_id': uuid, - 'service_id': uuid, - 'created_by': uuid, - 'personalisation': {'type': 'object'}, - 'starts_at': {'type': 'string', 'format': 'datetime'}, - 'finishes_at': {'type': 'string', 'format': 'datetime'}, - 'areas': {'type': 'object'}, - 'content': {'type': 'string', 'minLength': 1}, - 'reference': {'type': 'string', 'minLength': 1, 'maxLength': 255}, - }, - 'required': ['service_id', 'created_by'], - 'allOf': [ - {'oneOf': [ - {'required': ['template_id']}, - {'required': ['content']}, - ]}, - {'oneOf': [ - {'required': ['template_id']}, - {'required': ['reference']}, - ]}, - ], - 'additionalProperties': False -} - -update_broadcast_message_schema = { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'description': 'POST update broadcast_message schema', - 'type': 'object', - 'title': 'Update broadcast_message', - 'properties': { - 'personalisation': {'type': 'object'}, - 'starts_at': {'type': 'string', 'format': 'datetime'}, - 'finishes_at': {'type': 'string', 'format': 'datetime'}, - 'areas': {'type': 'object'}, - }, - 'required': [], - 'additionalProperties': False -} - -update_broadcast_message_status_schema = { - '$schema': 'http://json-schema.org/draft-07/schema#', - 'description': 'POST update broadcast_message status schema', - 'type': 'object', - 'title': 'Update broadcast_message', - 'properties': { - 'status': {'type': 'string', 'enum': BroadcastStatusType.STATUSES}, - 'created_by': uuid, - }, - 'required': ['status', 'created_by'], - 'additionalProperties': False -} diff --git a/app/broadcast_message/rest.py b/app/broadcast_message/rest.py deleted file mode 100644 index aca1c408f..000000000 --- a/app/broadcast_message/rest.py +++ /dev/null @@ -1,165 +0,0 @@ -import iso8601 -from flask import Blueprint, jsonify, request -from notifications_utils.template import BroadcastMessageTemplate - -from app.broadcast_message import utils as broadcast_utils -from app.broadcast_message.broadcast_message_schema import ( - create_broadcast_message_schema, - update_broadcast_message_schema, - update_broadcast_message_status_schema, -) -from app.dao.broadcast_message_dao import ( - dao_get_broadcast_message_by_id_and_service_id, - dao_get_broadcast_messages_for_service, -) -from app.dao.dao_utils import dao_save_object -from app.dao.services_dao import dao_fetch_service_by_id -from app.dao.templates_dao import dao_get_template_by_id_and_service_id -from app.dao.users_dao import get_user_by_id -from app.errors import InvalidRequest, register_errors -from app.models import BroadcastMessage, BroadcastStatusType -from app.schema_validation import validate - -broadcast_message_blueprint = Blueprint( - 'broadcast_message', - __name__, - url_prefix='/service//broadcast-message' -) -register_errors(broadcast_message_blueprint) - - -def _parse_nullable_datetime(dt): - if dt: - return iso8601.parse_date(dt).replace(tzinfo=None) - return dt - - -@broadcast_message_blueprint.route('', methods=['GET']) -def get_broadcast_messages_for_service(service_id): - # TODO: should this return template content/data in some way? or can we rely on them being cached admin side. - # we might need stuff like template name for showing on the dashboard. - # TODO: should this paginate or filter on dates or anything? - broadcast_messages = [o.serialize() for o in dao_get_broadcast_messages_for_service(service_id)] - return jsonify(broadcast_messages=broadcast_messages) - - -@broadcast_message_blueprint.route('/', methods=['GET']) -def get_broadcast_message(service_id, broadcast_message_id): - return jsonify(dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id).serialize()) - - -@broadcast_message_blueprint.route('', methods=['POST']) -def create_broadcast_message(service_id): - data = request.get_json() - - validate(data, create_broadcast_message_schema) - service = dao_fetch_service_by_id(data['service_id']) - user = get_user_by_id(data['created_by']) - personalisation = data.get('personalisation', {}) - template_id = data.get('template_id') - - if template_id: - template = dao_get_template_by_id_and_service_id( - template_id, data['service_id'] - ) - content = str(template._as_utils_template_with_personalisation( - personalisation - )) - reference = None - else: - temporary_template = BroadcastMessageTemplate.from_content(data['content']) - if temporary_template.content_too_long: - raise InvalidRequest( - ( - f'Content must be ' - f'{temporary_template.max_content_count:,.0f} ' - f'characters or fewer' - ) + ( - ' (because it could not be GSM7 encoded)' - if temporary_template.non_gsm_characters else '' - ), - status_code=400, - ) - template = None - content = str(temporary_template) - reference = data['reference'] - - broadcast_message = BroadcastMessage( - service_id=service.id, - template_id=template_id, - template_version=template.version if template else None, - personalisation=personalisation, - areas=data.get("areas", {}), - status=BroadcastStatusType.DRAFT, - starts_at=_parse_nullable_datetime(data.get('starts_at')), - finishes_at=_parse_nullable_datetime(data.get('finishes_at')), - created_by_id=user.id, - content=content, - reference=reference, - stubbed=service.restricted - ) - - dao_save_object(broadcast_message) - - return jsonify(broadcast_message.serialize()), 201 - - -@broadcast_message_blueprint.route('/', methods=['POST']) -def update_broadcast_message(service_id, broadcast_message_id): - data = request.get_json() - validate(data, update_broadcast_message_schema) - - broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id) - - if broadcast_message.status not in BroadcastStatusType.PRE_BROADCAST_STATUSES: - raise InvalidRequest( - f'Cannot update broadcast_message {broadcast_message.id} while it has status {broadcast_message.status}', - status_code=400 - ) - - areas = data.get("areas", {}) - - if ('ids' in areas and 'simple_polygons' not in areas) or ('ids' not in areas and 'simple_polygons' in areas): - raise InvalidRequest( - f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.', - status_code=400 - ) - - if 'personalisation' in data: - broadcast_message.personalisation = data['personalisation'] - if 'starts_at' in data: - broadcast_message.starts_at = _parse_nullable_datetime(data['starts_at']) - if 'finishes_at' in data: - broadcast_message.finishes_at = _parse_nullable_datetime(data['finishes_at']) - if 'ids' in areas and 'simple_polygons' in areas: - broadcast_message.areas = areas - - dao_save_object(broadcast_message) - - return jsonify(broadcast_message.serialize()), 200 - - -@broadcast_message_blueprint.route('//status', methods=['POST']) -def update_broadcast_message_status(service_id, broadcast_message_id): - data = request.get_json() - - validate(data, update_broadcast_message_status_schema) - broadcast_message = dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id) - - if not broadcast_message.service.active: - raise InvalidRequest("Updating broadcast message is not allowed: service is inactive ", 403) - - new_status = data['status'] - updating_user = get_user_by_id(data['created_by']) - - if updating_user not in broadcast_message.service.users: - # we allow platform admins to cancel broadcasts, and we don't check user if request was done via API - if not (new_status == BroadcastStatusType.CANCELLED and updating_user.platform_admin): - raise InvalidRequest( - f'User {updating_user.id} cannot update broadcast_message {broadcast_message.id} from other service', - status_code=400 - ) - - broadcast_utils.update_broadcast_message_status(broadcast_message, new_status, updating_user) - - return jsonify(broadcast_message.serialize()), 200 diff --git a/app/broadcast_message/translators.py b/app/broadcast_message/translators.py deleted file mode 100644 index bb999fccb..000000000 --- a/app/broadcast_message/translators.py +++ /dev/null @@ -1,37 +0,0 @@ -from bs4 import BeautifulSoup - - -def cap_xml_to_dict(cap_xml): - # This function assumes that it’s being passed valid CAP XML - cap = BeautifulSoup(cap_xml, "xml") - return { - "msgType": cap.alert.msgType.text, - "reference": cap.alert.identifier.text, - "references": ( - # references to previous events belonging to the same alert - cap.alert.references.text if cap.alert.references else None - ), - "cap_event": cap.alert.info.event.text, - "category": cap.alert.info.category.text, - "expires": cap.alert.info.expires.text, - "content": cap.alert.info.description.text, - "areas": [ - { - "name": area.areaDesc.text, - "polygons": [ - cap_xml_polygon_to_list(polygon.text) - for polygon in area.find_all('polygon') - ] - } - for area in cap.alert.info.find_all('area') - ] - } - - -def cap_xml_polygon_to_list(polygon_string): - return [ - [ - float(coordinate) for coordinate in pair.split(',') - ] - for pair in polygon_string.strip().split(' ') - ] diff --git a/app/broadcast_message/utils.py b/app/broadcast_message/utils.py deleted file mode 100644 index 0aa500eca..000000000 --- a/app/broadcast_message/utils.py +++ /dev/null @@ -1,140 +0,0 @@ -import inspect -from datetime import datetime - -from flask import current_app -from notifications_utils.clients.zendesk.zendesk_client import ( - NotifySupportTicket, -) - -from app import zendesk_client -from app.celery.broadcast_message_tasks import send_broadcast_event -from app.config import QueueNames -from app.dao.dao_utils import dao_save_object -from app.errors import InvalidRequest -from app.models import ( - BroadcastEvent, - BroadcastEventMessageType, - BroadcastStatusType, -) - - -def update_broadcast_message_status(broadcast_message, new_status, updating_user=None, api_key_id=None): - _validate_broadcast_update(broadcast_message, new_status, updating_user) - - if new_status == BroadcastStatusType.BROADCASTING: - broadcast_message.approved_at = datetime.utcnow() - broadcast_message.approved_by = updating_user - - if new_status == BroadcastStatusType.CANCELLED: - broadcast_message.cancelled_at = datetime.utcnow() - broadcast_message.cancelled_by = updating_user - broadcast_message.cancelled_by_api_key_id = api_key_id - - current_app.logger.info( - f'broadcast_message {broadcast_message.id} moving from {broadcast_message.status} to {new_status}' - ) - broadcast_message.status = new_status - - dao_save_object(broadcast_message) - _create_p1_zendesk_alert(broadcast_message) - - if new_status in {BroadcastStatusType.BROADCASTING, BroadcastStatusType.CANCELLED}: - _create_broadcast_event(broadcast_message) - - -def _validate_broadcast_update(broadcast_message, new_status, updating_user): - if new_status not in BroadcastStatusType.ALLOWED_STATUS_TRANSITIONS[broadcast_message.status]: - raise InvalidRequest( - f'Cannot move broadcast_message {broadcast_message.id} from {broadcast_message.status} to {new_status}', - status_code=400 - ) - - if new_status == BroadcastStatusType.BROADCASTING: - # training mode services can approve their own broadcasts - if updating_user == broadcast_message.created_by and not broadcast_message.service.restricted: - raise InvalidRequest( - f'User {updating_user.id} cannot approve their own broadcast_message {broadcast_message.id}', - status_code=400 - ) - elif len(broadcast_message.areas['simple_polygons']) == 0: - raise InvalidRequest( - f'broadcast_message {broadcast_message.id} has no selected areas and so cannot be broadcasted.', - status_code=400 - ) - - -def _create_p1_zendesk_alert(broadcast_message): - if current_app.config['NOTIFY_ENVIRONMENT'] != 'live': - return - - if broadcast_message.status != BroadcastStatusType.BROADCASTING: - return - - message = inspect.cleandoc(f""" - Broadcast Sent - - https://www.notifications.service.gov.uk/services/{broadcast_message.service_id}/current-alerts/{broadcast_message.id} - - Sent on channel {broadcast_message.service.broadcast_channel} to {broadcast_message.areas["names"]}. - - Content starts "{broadcast_message.content[:100]}". - - Follow the runbook to check the broadcast went out OK: - https://docs.google.com/document/d/1J99yOlfp4nQz6et0w5oJVqi-KywtIXkxrEIyq_g2XUs/edit#heading=h.lzr9aq5b4wg - """) - - ticket = NotifySupportTicket( - subject='Live broadcast sent', - message=message, - ticket_type=NotifySupportTicket.TYPE_INCIDENT, - technical_ticket=True, - org_id=current_app.config['BROADCAST_ORGANISATION_ID'], - org_type='central', - service_id=str(broadcast_message.service_id), - p1=True - ) - zendesk_client.send_ticket_to_zendesk(ticket) - - -def _create_broadcast_event(broadcast_message): - """ - If the service is live and the broadcast message is not stubbed, creates a broadcast event, stores it in the - database, and triggers the task to send the CAP XML off. - """ - service = broadcast_message.service - - if not broadcast_message.stubbed and not service.restricted: - msg_types = { - BroadcastStatusType.BROADCASTING: BroadcastEventMessageType.ALERT, - BroadcastStatusType.CANCELLED: BroadcastEventMessageType.CANCEL, - } - - event = BroadcastEvent( - service=service, - broadcast_message=broadcast_message, - message_type=msg_types[broadcast_message.status], - transmitted_content={"body": broadcast_message.content}, - transmitted_areas=broadcast_message.areas, - # TODO: Probably move this somewhere more standalone too and imply that it shouldn't change. Should it - # include a service based identifier too? eg "flood-warnings@notifications.service.gov.uk" or similar - transmitted_sender='notifications.service.gov.uk', - - # TODO: Should this be set to now? Or the original starts_at? - transmitted_starts_at=broadcast_message.starts_at, - transmitted_finishes_at=broadcast_message.finishes_at, - ) - - dao_save_object(event) - - send_broadcast_event.apply_async( - kwargs={'broadcast_event_id': str(event.id)}, - queue=QueueNames.BROADCASTS - ) - elif broadcast_message.stubbed != service.restricted: - # It's possible for a service to create a broadcast in trial mode, and then approve it after the - # service is live (or vice versa). We don't think it's safe to send such broadcasts, as the service - # has changed since they were created. Log an error instead. - current_app.logger.error( - f'Broadcast event not created. Stubbed status of broadcast message was {broadcast_message.stubbed}' - f' but service was {"in trial mode" if service.restricted else "live"}' - ) diff --git a/app/celery/broadcast_message_tasks.py b/app/celery/broadcast_message_tasks.py deleted file mode 100644 index 9cdea6375..000000000 --- a/app/celery/broadcast_message_tasks.py +++ /dev/null @@ -1,230 +0,0 @@ -from datetime import datetime - -from flask import current_app - -from app import cbc_proxy_client, notify_celery -from app.clients.cbc_proxy import CBCProxyRetryableException -from app.config import QueueNames, TaskNames -from app.dao.broadcast_message_dao import ( - create_broadcast_provider_message, - dao_get_broadcast_event_by_id, - update_broadcast_provider_message_status, -) -from app.models import ( - BroadcastEventMessageType, - BroadcastProvider, - BroadcastProviderMessageStatus, -) -from app.utils import format_sequential_number - - -class BroadcastIntegrityError(Exception): - pass - - -def get_retry_delay(retry_count): - """ - Given a count of retries so far, return a delay for the next one. - `retry_count` should be 0 the first time a task fails. - """ - # TODO: replace with celery's built in exponential backoff - - # 2 to the power of x. 1, 2, 4, 8, 16, 32, ... - delay = 2**retry_count - # never wait longer than 4 minutes - return min(delay, 240) - - -def check_event_is_authorised_to_be_sent(broadcast_event, provider): - if not broadcast_event.service.active: - raise BroadcastIntegrityError( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: the service is suspended' - ) - - if broadcast_event.service.restricted: - raise BroadcastIntegrityError( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: the service is not live' - ) - - if broadcast_event.broadcast_message.stubbed: - raise BroadcastIntegrityError( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: the broadcast message is stubbed' - ) - - -def check_event_makes_sense_in_sequence(broadcast_event, provider): - """ - If any previous event hasn't sent yet for that provider, then we shouldn't send the current event. Instead, fail and - raise a zendesk ticket - so that a notify team member can assess the state of the previous messages, and if - necessary, can replay the `send_broadcast_provider_message` task if the previous message has now been sent. - - Note: This is called before the new broadcast_provider_message is created. - - # Help, I've come across this code following a pagerduty alert, what should I do? - - 1. Find the failing broadcast_provider_message associated with the previous event that caused this to trip. - 2. If that provider message is still failing to send, fix the issue causing that. The task to send that previous - message might still be retrying in the background - look for logs related to that task. - 3. If that provider message has sent succesfully, you might need to send this task off depending on context. This - might not always be true though, for example, it may not be necessary to send a cancel if the original alert has - already expired. - 4. If you need to re-send this task off again, you'll need to run the following command on paas: - `send_broadcast_provider_message.apply_async(args=(broadcast_event_id, provider), queue=QueueNames.BROADCASTS)` - """ - current_provider_message = broadcast_event.get_provider_message(provider) - # if this is the first time a task is being executed, it won't have a provider message yet - if current_provider_message and current_provider_message.status != BroadcastProviderMessageStatus.SENDING: - raise BroadcastIntegrityError( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: ' + - f'It is in status {current_provider_message.status}' - ) - - if broadcast_event.transmitted_finishes_at < datetime.utcnow(): - raise BroadcastIntegrityError( - f'Cannot send broadcast_event {broadcast_event.id} ' + - f'to provider {provider}: ' + - f'The expiry time of {broadcast_event.transmitted_finishes_at} has already passed' - ) - - # get events sorted from earliest to latest - events = sorted(broadcast_event.broadcast_message.events, key=lambda x: x.sent_at) - - for prev_event in events: - if prev_event.id != broadcast_event.id and prev_event.sent_at < broadcast_event.sent_at: - # get the record from when that event was sent to the same provider - prev_provider_message = prev_event.get_provider_message(provider) - - # the previous message hasn't even got round to running `send_broadcast_provider_message` yet. - if not prev_provider_message: - raise BroadcastIntegrityError( - f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' + - f'(type {prev_event.message_type}) has no provider_message for provider {provider} yet.\n' + - 'You must ensure that the other event sends succesfully, then manually kick off this event ' + - 'again by re-running send_broadcast_provider_message for this event and provider.' - ) - - # if there's a previous message that has started but not finished sending (whether it fatally errored or is - # currently retrying) - if prev_provider_message.status != BroadcastProviderMessageStatus.ACK: - raise BroadcastIntegrityError( - f'Cannot send {broadcast_event.id}. Previous event {prev_event.id} ' + - f'(type {prev_event.message_type}) has not finished sending to provider {provider} yet.\n' + - f'It is currently in status "{prev_provider_message.status}".\n' + - 'You must ensure that the other event sends succesfully, then manually kick off this event ' + - 'again by re-running send_broadcast_provider_message for this event and provider.' - ) - - -@notify_celery.task(name="send-broadcast-event") -def send_broadcast_event(broadcast_event_id): - broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id) - - notify_celery.send_task( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) - - for provider in broadcast_event.service.get_available_broadcast_providers(): - send_broadcast_provider_message.apply_async( - kwargs={'broadcast_event_id': broadcast_event_id, 'provider': provider}, - queue=QueueNames.BROADCASTS - ) - - -# max_retries=None: retry forever -@notify_celery.task(bind=True, name="send-broadcast-provider-message", max_retries=None) -def send_broadcast_provider_message(self, broadcast_event_id, provider): - if not current_app.config['CBC_PROXY_ENABLED']: - current_app.logger.info( - "CBC Proxy disabled, not sending broadcast_provider_message for " - f"broadcast_event_id {broadcast_event_id} with provider {provider}" - ) - return - - broadcast_event = dao_get_broadcast_event_by_id(broadcast_event_id) - - check_event_is_authorised_to_be_sent(broadcast_event, provider) - check_event_makes_sense_in_sequence(broadcast_event, provider) - - # the broadcast_provider_message may already exist if we retried previously - broadcast_provider_message = broadcast_event.get_provider_message(provider) - if broadcast_provider_message is None: - broadcast_provider_message = create_broadcast_provider_message(broadcast_event, provider) - - formatted_message_number = None - if provider == BroadcastProvider.VODAFONE: - formatted_message_number = format_sequential_number(broadcast_provider_message.message_number) - - current_app.logger.info( - f'Invoking cbc proxy to send broadcast_provider_message with ID of {broadcast_provider_message.id} ' - f'and broadcast_event ID of {broadcast_event_id} ' - f'msgType {broadcast_event.message_type}' - ) - - areas = [ - {"polygon": polygon} - for polygon in broadcast_event.transmitted_areas["simple_polygons"] - ] - - cbc_proxy_provider_client = cbc_proxy_client.get_proxy(provider) - - try: - if broadcast_event.message_type == BroadcastEventMessageType.ALERT: - cbc_proxy_provider_client.create_and_send_broadcast( - identifier=str(broadcast_provider_message.id), - message_number=formatted_message_number, - headline="GOV.UK Notify Broadcast", - description=broadcast_event.transmitted_content['body'], - areas=areas, - sent=broadcast_event.sent_at_as_cap_datetime_string, - expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string, - channel=broadcast_event.service.broadcast_channel - ) - elif broadcast_event.message_type == BroadcastEventMessageType.UPDATE: - cbc_proxy_provider_client.update_and_send_broadcast( - identifier=str(broadcast_provider_message.id), - message_number=formatted_message_number, - headline="GOV.UK Notify Broadcast", - description=broadcast_event.transmitted_content['body'], - areas=areas, - previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider), - sent=broadcast_event.sent_at_as_cap_datetime_string, - expires=broadcast_event.transmitted_finishes_at_as_cap_datetime_string, - # We think an alert update should always go out on the same channel that created the alert - # We recognise there is a small risk with this code here that if the services channel was - # changed between an alert being sent out and then updated, then something might go wrong - # but we are relying on service channels changing almost never, and not mid incident - # We may consider in the future, changing this such that we store the channel a broadcast was - # sent on on the broadcast message itself and pick the value from there instead of the service - channel=broadcast_event.service.broadcast_channel - ) - elif broadcast_event.message_type == BroadcastEventMessageType.CANCEL: - cbc_proxy_provider_client.cancel_broadcast( - identifier=str(broadcast_provider_message.id), - message_number=formatted_message_number, - previous_provider_messages=broadcast_event.get_earlier_provider_messages(provider), - sent=broadcast_event.sent_at_as_cap_datetime_string, - ) - except CBCProxyRetryableException as exc: - delay = get_retry_delay(self.request.retries) - current_app.logger.exception( - f'Retrying send_broadcast_provider_message for broadcast event {broadcast_event_id}, ' - f'provider message {broadcast_provider_message.id}, provider {provider} in {delay} seconds' - ) - - self.retry( - exc=exc, - countdown=delay, - queue=QueueNames.BROADCASTS, - ) - - update_broadcast_provider_message_status(broadcast_provider_message, status=BroadcastProviderMessageStatus.ACK) - - -@notify_celery.task(name='trigger-link-test') -def trigger_link_test(provider): - cbc_proxy_client.get_proxy(provider).send_link_test() diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index d02faa0ab..be09f3a74 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -9,7 +9,6 @@ from sqlalchemy.exc import SQLAlchemyError from app import db, notify_celery, zendesk_client from app.aws import s3 -from app.celery.broadcast_message_tasks import trigger_link_test from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter from app.celery.tasks import ( get_recipient_csv_and_template_and_sender_id, @@ -53,8 +52,6 @@ from app.models import ( JOB_STATUS_IN_PROGRESS, JOB_STATUS_PENDING, SMS_TYPE, - BroadcastMessage, - BroadcastStatusType, Job, ) from app.notifications.process_notifications import send_notification_to_queue @@ -333,37 +330,3 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(): technical_ticket=True ) zendesk_client.send_ticket_to_zendesk(ticket) - - -@notify_celery.task(name='trigger-link-tests') -def trigger_link_tests(): - if current_app.config['CBC_PROXY_ENABLED']: - for cbc_name in current_app.config['ENABLED_CBCS']: - trigger_link_test.apply_async(kwargs={'provider': cbc_name}, queue=QueueNames.BROADCASTS) - - -@notify_celery.task(name='auto-expire-broadcast-messages') -def auto_expire_broadcast_messages(): - expired_broadcasts = BroadcastMessage.query.filter( - BroadcastMessage.finishes_at <= datetime.now(), - BroadcastMessage.status == BroadcastStatusType.BROADCASTING, - ).all() - - for broadcast in expired_broadcasts: - broadcast.status = BroadcastStatusType.COMPLETED - - db.session.commit() - - if expired_broadcasts: - notify_celery.send_task( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) - - -@notify_celery.task(name='remove-yesterdays-planned-tests-on-govuk-alerts') -def remove_yesterdays_planned_tests_on_govuk_alerts(): - notify_celery.send_task( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) diff --git a/app/clients/cbc_proxy.py b/app/clients/cbc_proxy.py deleted file mode 100644 index a33b0b284..000000000 --- a/app/clients/cbc_proxy.py +++ /dev/null @@ -1,299 +0,0 @@ -import json -import uuid -from abc import ABC, abstractmethod - -import boto3 -import botocore -from flask import current_app -from notifications_utils.template import non_gsm_characters -from sqlalchemy.schema import Sequence - -from app.config import BroadcastProvider -from app.utils import DATETIME_FORMAT, format_sequential_number - -# The variable names in this file have specific meaning in a CAP message -# -# identifier is a unique field for each CAP message -# -# headline is a field which we are not sure if we will use -# -# description is the body of the message - -# areas is a list of dicts, with the following items -# * description is a string which populates the areaDesc field -# * polygon is a list of lat/long pairs -# -# previous_provider_messages is a list of previous events (models.py::BroadcastProviderMessage) -# ie a Cancel message would have a unique event but have the event of -# the preceeding Alert message in the previous_provider_messages field - - -class CBCProxyRetryableException(Exception): - pass - - -class CBCProxyClient: - _lambda_client = None - - def init_app(self, app): - if app.config.get('CBC_PROXY_ENABLED'): - self._lambda_client = boto3.client( - 'lambda', - region_name='us-west-2', - aws_access_key_id=app.config['CBC_PROXY_AWS_ACCESS_KEY_ID'], - aws_secret_access_key=app.config['CBC_PROXY_AWS_SECRET_ACCESS_KEY'], - ) - - def get_proxy(self, provider): - proxy_classes = { - BroadcastProvider.EE: CBCProxyEE, - BroadcastProvider.THREE: CBCProxyThree, - BroadcastProvider.O2: CBCProxyO2, - BroadcastProvider.VODAFONE: CBCProxyVodafone, - } - return proxy_classes[provider](self._lambda_client) - - -class CBCProxyClientBase(ABC): - @property - @abstractmethod - def lambda_name(self): - pass - - @property - @abstractmethod - def failover_lambda_name(self): - pass - - @property - @abstractmethod - def LANGUAGE_ENGLISH(self): - pass - - @property - @abstractmethod - def LANGUAGE_WELSH(self): - pass - - def __init__(self, lambda_client): - self._lambda_client = lambda_client - - def send_link_test(self): - self._send_link_test(self.lambda_name) - self._send_link_test(self.failover_lambda_name) - - def _send_link_test( - self, - lambda_name, - ): pass - - def create_and_send_broadcast( - self, identifier, headline, description, areas, sent, expires, channel, message_number=None - ): - pass - - # We have not implementated updating a broadcast - def update_and_send_broadcast( - self, - identifier, previous_provider_messages, headline, description, areas, - sent, expires, channel, message_number=None - ): - pass - - def cancel_broadcast( - self, - identifier, previous_provider_messages, headline, description, areas, - sent, expires, message_number=None - ): - pass - - def _invoke_lambda_with_failover(self, payload): - result = self._invoke_lambda(self.lambda_name, payload) - - if not result: - failover_result = self._invoke_lambda(self.failover_lambda_name, payload) - if not failover_result: - raise CBCProxyRetryableException( - f'Lambda failed for both {self.lambda_name} and {self.failover_lambda_name}' - ) - - return result - - def _invoke_lambda(self, lambda_name, payload): - payload_bytes = bytes(json.dumps(payload), encoding='utf8') - try: - current_app.logger.info( - f"Calling lambda {lambda_name} with payload {str(payload)[:1000]}" - ) - - result = self._lambda_client.invoke( - FunctionName=lambda_name, - InvocationType='RequestResponse', - Payload=payload_bytes, - ) - except botocore.exceptions.ClientError: - current_app.logger.exception(f'Boto ClientError calling lambda {lambda_name}') - success = False - return success - - if result['StatusCode'] > 299: - current_app.logger.info( - f"Error calling lambda {lambda_name} with status code { result['StatusCode']}, {result.get('Payload')}" - ) - success = False - - elif 'FunctionError' in result: - current_app.logger.info( - f"Error calling lambda {lambda_name} with function error { result['Payload'].read() }" - ) - success = False - - else: - success = True - - return success - - def infer_language_from(self, content): - if non_gsm_characters(content): - return self.LANGUAGE_WELSH - return self.LANGUAGE_ENGLISH - - -class CBCProxyOne2ManyClient(CBCProxyClientBase): - LANGUAGE_ENGLISH = 'en-GB' - LANGUAGE_WELSH = 'cy-GB' - - def _send_link_test( - self, - lambda_name, - ): - """ - link test - open up a connection to a specific provider, and send them an xml payload with a of - test. - """ - payload = { - 'message_type': 'test', - 'identifier': str(uuid.uuid4()), - 'message_format': 'cap' - } - - self._invoke_lambda(lambda_name=lambda_name, payload=payload) - - def create_and_send_broadcast( - self, identifier, headline, description, areas, sent, expires, channel, message_number=None - ): - payload = { - 'message_type': 'alert', - 'identifier': identifier, - 'message_format': 'cap', - 'headline': headline, - 'description': description, - 'areas': areas, - 'sent': sent, - 'expires': expires, - 'language': self.infer_language_from(description), - 'channel': channel, - } - self._invoke_lambda_with_failover(payload=payload) - - def cancel_broadcast( - self, - identifier, previous_provider_messages, - sent, message_number=None - ): - payload = { - 'message_type': 'cancel', - 'identifier': identifier, - 'message_format': 'cap', - "references": [ - { - "message_id": str(message.id), - "sent": message.created_at.strftime(DATETIME_FORMAT) - } for message in previous_provider_messages - ], - 'sent': sent, - } - self._invoke_lambda_with_failover(payload=payload) - - -class CBCProxyEE(CBCProxyOne2ManyClient): - lambda_name = 'ee-1-proxy' - failover_lambda_name = 'ee-2-proxy' - - -class CBCProxyThree(CBCProxyOne2ManyClient): - lambda_name = 'three-1-proxy' - failover_lambda_name = 'three-2-proxy' - - -class CBCProxyO2(CBCProxyOne2ManyClient): - lambda_name = 'o2-1-proxy' - failover_lambda_name = 'o2-2-proxy' - - -class CBCProxyVodafone(CBCProxyClientBase): - lambda_name = 'vodafone-1-proxy' - failover_lambda_name = 'vodafone-2-proxy' - - LANGUAGE_ENGLISH = 'English' - LANGUAGE_WELSH = 'Welsh' - - def _send_link_test( - self, - lambda_name, - ): - """ - link test - open up a connection to a specific provider, and send them an xml payload with a of - test. - """ - from app import db - sequence = Sequence('broadcast_provider_message_number_seq') - sequential_number = db.session.connection().execute(sequence) - formatted_seq_number = format_sequential_number(sequential_number) - - payload = { - 'message_type': 'test', - 'identifier': str(uuid.uuid4()), - 'message_number': formatted_seq_number, - 'message_format': 'ibag' - } - - self._invoke_lambda(lambda_name=lambda_name, payload=payload) - - def create_and_send_broadcast( - self, identifier, message_number, headline, description, areas, sent, expires, channel - ): - payload = { - 'message_type': 'alert', - 'identifier': identifier, - 'message_number': message_number, - 'message_format': 'ibag', - 'headline': headline, - 'description': description, - 'areas': areas, - 'sent': sent, - 'expires': expires, - 'language': self.infer_language_from(description), - 'channel': channel, - } - self._invoke_lambda_with_failover(payload=payload) - - def cancel_broadcast( - self, identifier, previous_provider_messages, sent, message_number - ): - - payload = { - 'message_type': 'cancel', - 'identifier': identifier, - 'message_number': message_number, - 'message_format': 'ibag', - "references": [ - { - "message_id": str(message.id), - "message_number": format_sequential_number(message.message_number), - "sent": message.created_at.strftime(DATETIME_FORMAT) - } for message in previous_provider_messages - ], - 'sent': sent, - } - self._invoke_lambda_with_failover(payload=payload) diff --git a/app/commands.py b/app/commands.py index ad944178d..803908c4f 100644 --- a/app/commands.py +++ b/app/commands.py @@ -811,33 +811,3 @@ def populate_annual_billing_with_defaults(year, missing_services_only): else: print(f'update service {service.id} with default') set_default_free_allowance_for_service(service, year) - - -@click.option('-u', '--user-id', required=True) -@notify_command(name='local-dev-broadcast-permissions') -def local_dev_broadcast_permissions(user_id): - if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']: - current_app.logger.error('Can only be run in development') - return - - user = User.query.filter_by(id=user_id).one() - - user_broadcast_services = Service.query.filter( - Service.permissions.any(permission='broadcast'), - Service.users.any(id=user_id) - ) - - for service in user_broadcast_services: - permission_list = [ - Permission(service_id=service.id, user_id=user_id, permission=permission) - for permission in [ - 'reject_broadcasts', 'cancel_broadcasts', # required to create / approve - 'create_broadcasts', 'approve_broadcasts', # minimum for testing - 'manage_templates', # unlikely but might be useful - 'view_activity', # normally added on invite / service creation - ] - ] - - permission_dao.set_user_service_permission( - user, service, permission_list, _commit=True, replace=True - ) diff --git a/app/config.py b/app/config.py index 0cbd2067d..6897dcf65 100644 --- a/app/config.py +++ b/app/config.py @@ -34,8 +34,6 @@ class QueueNames(object): SANITISE_LETTERS = 'sanitise-letter-tasks' SAVE_API_EMAIL = 'save-api-email-tasks' SAVE_API_SMS = 'save-api-sms-tasks' - BROADCASTS = 'broadcast-tasks' - GOVUK_ALERTS = 'govuk-alerts' @staticmethod def all_queues(): @@ -57,26 +55,15 @@ class QueueNames(object): QueueNames.SMS_CALLBACKS, QueueNames.SAVE_API_EMAIL, QueueNames.SAVE_API_SMS, - QueueNames.BROADCASTS, ] -class BroadcastProvider: - EE = 'ee' - VODAFONE = 'vodafone' - THREE = 'three' - O2 = 'o2' - - PROVIDERS = [EE, VODAFONE, THREE, O2] - - class TaskNames(object): PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs' ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs' SCAN_FILE = 'scan-file' SANITISE_LETTER = 'sanitise-and-upload-letter' CREATE_PDF_FOR_TEMPLATED_LETTER = 'create-pdf-for-templated-letter' - PUBLISH_GOVUK_ALERTS = 'publish-govuk-alerts' RECREATE_PDF_FOR_PRECOMPILED_LETTER = 'recreate-pdf-for-precompiled-letter' @@ -89,7 +76,6 @@ class Config(object): # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID = 'notify-admin' - GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove? INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') @@ -175,7 +161,6 @@ class Config(object): NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' NOTIFY_USER_ID = '6af522d0-2915-4e52-83a3-3690455a5fe6' INVITATION_EMAIL_TEMPLATE_ID = '4f46df42-f795-4cc4-83bb-65ca312f49cc' - BROADCAST_INVITATION_EMAIL_TEMPLATE_ID = '46152f7c-6901-41d5-8590-a5624d0d4359' SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246' EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164' NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27' @@ -333,16 +318,6 @@ class Config(object): 'schedule': timedelta(minutes=15), 'options': {'queue': QueueNames.PERIODIC} }, - 'auto-expire-broadcast-messages': { - 'task': 'auto-expire-broadcast-messages', - 'schedule': timedelta(minutes=5), - 'options': {'queue': QueueNames.PERIODIC} - }, - 'remove-yesterdays-planned-tests-on-govuk-alerts': { - 'task': 'remove-yesterdays-planned-tests-on-govuk-alerts', - 'schedule': crontab(hour=00, minute=00), - 'options': {'queue': QueueNames.PERIODIC} - }, } } @@ -389,15 +364,6 @@ class Config(object): AWS_REGION = 'us-west-2' - CBC_PROXY_ENABLED = True - CBC_PROXY_AWS_ACCESS_KEY_ID = os.environ.get('CBC_PROXY_AWS_ACCESS_KEY_ID', '') - CBC_PROXY_AWS_SECRET_ACCESS_KEY = os.environ.get('CBC_PROXY_AWS_SECRET_ACCESS_KEY', '') - - ENABLED_CBCS = {BroadcastProvider.EE, BroadcastProvider.THREE, BroadcastProvider.O2, BroadcastProvider.VODAFONE} - - # as defined in api db migration 0331_add_broadcast_org.py - BROADCAST_ORGANISATION_ID = '38e4bf69-93b0-445d-acee-53ea53fe02df' - ###################### # Config overrides ### @@ -427,7 +393,6 @@ class Development(Config): # INTERNAL_CLIENT_API_KEYS = { # Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'], - # Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] # } SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development @@ -452,8 +417,6 @@ class Development(Config): API_RATE_LIMIT_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com'] - CBC_PROXY_ENABLED = False - class Test(Development): NOTIFY_EMAIL_DOMAIN = 'test.notify.com' @@ -498,7 +461,6 @@ class Test(Development): MMG_URL = 'https://example.com/mmg' FIRETEXT_URL = 'https://example.com/firetext' - CBC_PROXY_ENABLED = True DVLA_EMAIL_ADDRESSES = ['success@simulator.amazonses.com', 'success+2@simulator.amazonses.com'] diff --git a/app/dao/broadcast_message_dao.py b/app/dao/broadcast_message_dao.py deleted file mode 100644 index 1281c0e75..000000000 --- a/app/dao/broadcast_message_dao.py +++ /dev/null @@ -1,105 +0,0 @@ -import uuid -from datetime import datetime - -from sqlalchemy import desc - -from app import db -from app.dao.dao_utils import autocommit -from app.models import ( - BroadcastEvent, - BroadcastMessage, - BroadcastProvider, - BroadcastProviderMessage, - BroadcastProviderMessageNumber, - BroadcastProviderMessageStatus, - BroadcastStatusType, - ServiceBroadcastSettings, -) - - -def dao_get_broadcast_message_by_id_and_service_id(broadcast_message_id, service_id): - return BroadcastMessage.query.filter( - BroadcastMessage.id == broadcast_message_id, - BroadcastMessage.service_id == service_id - ).one() - - -def dao_get_broadcast_message_by_references_and_service_id(references_to_original_broadcast, service_id): - return BroadcastMessage.query.filter( - BroadcastMessage.status.in_(( - BroadcastStatusType.PENDING_APPROVAL, - BroadcastStatusType.BROADCASTING, - )), - BroadcastMessage.reference.in_(references_to_original_broadcast), - BroadcastMessage.service_id == service_id - ).one() - - -def dao_get_broadcast_event_by_id(broadcast_event_id): - return BroadcastEvent.query.filter(BroadcastEvent.id == broadcast_event_id).one() - - -def dao_get_broadcast_messages_for_service(service_id): - return BroadcastMessage.query.filter( - BroadcastMessage.service_id == service_id - ).order_by(BroadcastMessage.created_at) - - -def dao_get_all_broadcast_messages(): - return db.session.query( - BroadcastMessage.id, - BroadcastMessage.reference, - ServiceBroadcastSettings.channel, - BroadcastMessage.content, - BroadcastMessage.areas, - BroadcastMessage.status, - BroadcastMessage.starts_at, - BroadcastMessage.finishes_at, - BroadcastMessage.approved_at, - BroadcastMessage.cancelled_at, - ).join( - ServiceBroadcastSettings, ServiceBroadcastSettings.service_id == BroadcastMessage.service_id - ).filter( - BroadcastMessage.starts_at >= datetime(2021, 5, 25, 0, 0, 0), - BroadcastMessage.stubbed == False, # noqa - BroadcastMessage.status.in_(BroadcastStatusType.LIVE_STATUSES) - ).order_by(desc(BroadcastMessage.starts_at)).all() - - -def get_earlier_events_for_broadcast_event(broadcast_event_id): - """ - This is used to build up the references list. - """ - this_event = BroadcastEvent.query.get(broadcast_event_id) - - return BroadcastEvent.query.filter( - BroadcastEvent.broadcast_message_id == this_event.broadcast_message_id, - BroadcastEvent.sent_at < this_event.sent_at - ).order_by( - BroadcastEvent.sent_at.asc() - ).all() - - -@autocommit -def create_broadcast_provider_message(broadcast_event, provider): - broadcast_provider_message_id = uuid.uuid4() - provider_message = BroadcastProviderMessage( - id=broadcast_provider_message_id, - broadcast_event=broadcast_event, - provider=provider, - status=BroadcastProviderMessageStatus.SENDING, - ) - db.session.add(provider_message) - db.session.commit() - provider_message_number = None - if provider == BroadcastProvider.VODAFONE: - provider_message_number = BroadcastProviderMessageNumber( - broadcast_provider_message_id=broadcast_provider_message_id) - db.session.add(provider_message_number) - db.session.commit() - return provider_message - - -@autocommit -def update_broadcast_provider_message_status(broadcast_provider_message, *, status): - broadcast_provider_message.status = status diff --git a/app/dao/broadcast_service_dao.py b/app/dao/broadcast_service_dao.py deleted file mode 100644 index 6166b0634..000000000 --- a/app/dao/broadcast_service_dao.py +++ /dev/null @@ -1,100 +0,0 @@ -from datetime import datetime - -from flask import current_app - -from app import db -from app.dao.dao_utils import autocommit, version_class -from app.models import ( - BROADCAST_TYPE, - EMAIL_AUTH_TYPE, - INVITE_PENDING, - VIEW_ACTIVITY, - ApiKey, - InvitedUser, - Organisation, - Permission, - Service, - ServiceBroadcastSettings, - ServicePermission, -) - - -@autocommit -@version_class(Service) -def set_broadcast_service_type(service, service_mode, broadcast_channel, provider_restriction): - insert_or_update_service_broadcast_settings( - service, channel=broadcast_channel, provider_restriction=provider_restriction - ) - - # Remove all permissions and add broadcast permission - if not service.has_permission(BROADCAST_TYPE): - service_permission = ServicePermission(service_id=service.id, permission=BROADCAST_TYPE) - db.session.add(service_permission) - - ServicePermission.query.filter( - ServicePermission.service_id == service.id, - ServicePermission.permission != BROADCAST_TYPE, - # Email auth is an exception to the other service permissions (which relate to what type - # of notifications a service can send) where a broadcast service is allowed to have the - # email auth permission (but doesn't have to) - ServicePermission.permission != EMAIL_AUTH_TYPE - ).delete() - - # Refresh the service object as it has references to the service permissions but we don't yet - # want to commit the permission changes incase all of this needs to rollback - db.session.refresh(service) - - # Set service count as live false always - service.count_as_live = False - - # Set service into training mode or live mode - if service_mode == "live": - if service.restricted: - # Only update the go live at timestamp if this if moving from training mode - # to live mode, not if it's moving from one type of live mode service to another - service.go_live_at = datetime.utcnow() - service.restricted = False - else: - service.restricted = True - service.go_live_at = None - - # Remove all user permissions apart from view_activity for the service users and invited users - Permission.query.filter( - Permission.service_id == service.id, - Permission.permission != VIEW_ACTIVITY - ).delete() - InvitedUser.query.filter_by( - service_id=service.id, - status=INVITE_PENDING - ).update({'permissions': VIEW_ACTIVITY}) - - # Revoke any API keys to avoid a regular API key being used to send alerts - ApiKey.query.filter_by( - service_id=service.id, - expiry_date=None, - ).update({ - ApiKey.expiry_date: datetime.utcnow() - }) - - # Add service to organisation - organisation = Organisation.query.filter_by( - id=current_app.config['BROADCAST_ORGANISATION_ID'] - ).one() - service.organisation_id = organisation.id - service.organisation_type = organisation.organisation_type - service.crown = organisation.crown - - db.session.add(service) - - -def insert_or_update_service_broadcast_settings(service, channel, provider_restriction="all"): - if not service.service_broadcast_settings: - settings = ServiceBroadcastSettings() - settings.service = service - settings.channel = channel - settings.provider = provider_restriction - db.session.add(settings) - else: - service.service_broadcast_settings.channel = channel - service.service_broadcast_settings.provider = provider_restriction - db.session.add(service.service_broadcast_settings) diff --git a/app/dao/templates_dao.py b/app/dao/templates_dao.py index 19361fce1..fb669565f 100644 --- a/app/dao/templates_dao.py +++ b/app/dao/templates_dao.py @@ -72,7 +72,6 @@ def dao_update_template_reply_to(template_id, reply_to): "archived": template.archived, "process_type": template.process_type, "service_letter_contact_id": template.service_letter_contact_id, - "broadcast_data": template.broadcast_data, }) db.session.add(history) return template diff --git a/app/govuk_alerts/__init__.py b/app/govuk_alerts/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/app/govuk_alerts/rest.py b/app/govuk_alerts/rest.py deleted file mode 100644 index 8ec7a35e0..000000000 --- a/app/govuk_alerts/rest.py +++ /dev/null @@ -1,31 +0,0 @@ -from flask import Blueprint, jsonify - -from app.dao.broadcast_message_dao import dao_get_all_broadcast_messages -from app.errors import register_errors -from app.utils import get_dt_string_or_none - -govuk_alerts_blueprint = Blueprint( - "govuk-alerts", - __name__, - url_prefix='/govuk-alerts', -) - -register_errors(govuk_alerts_blueprint) - - -@govuk_alerts_blueprint.route('') -def get_broadcasts(): - broadcasts = dao_get_all_broadcast_messages() - broadcasts_dict = {"alerts": [{ - "id": broadcast.id, - "reference": broadcast.reference, - "channel": broadcast.channel, - "content": broadcast.content, - "areas": broadcast.areas, - "status": broadcast.status, - "starts_at": get_dt_string_or_none(broadcast.starts_at), - "finishes_at": get_dt_string_or_none(broadcast.finishes_at), - "approved_at": get_dt_string_or_none(broadcast.approved_at), - "cancelled_at": get_dt_string_or_none(broadcast.cancelled_at), - } for broadcast in broadcasts]} - return jsonify(broadcasts_dict), 200 diff --git a/app/models.py b/app/models.py index ff6a8a8d9..d9735f533 100644 --- a/app/models.py +++ b/app/models.py @@ -16,7 +16,6 @@ from notifications_utils.recipients import ( validate_phone_number, ) from notifications_utils.template import ( - BroadcastMessageTemplate, LetterPrintTemplate, PlainTextEmailTemplate, SMSMessageTemplate, @@ -50,10 +49,9 @@ from app.utils import ( SMS_TYPE = 'sms' EMAIL_TYPE = 'email' LETTER_TYPE = 'letter' -BROADCAST_TYPE = 'broadcast' -TEMPLATE_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, BROADCAST_TYPE] -NOTIFICATION_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE] # not broadcast +TEMPLATE_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE] +NOTIFICATION_TYPES = [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE] template_types = db.Enum(*TEMPLATE_TYPES, name='template_type') @@ -152,7 +150,6 @@ class User(db.Model): return True return any( - str(service.organisation_id) == current_app.config['BROADCAST_ORGANISATION_ID'] or str(service.id) == current_app.config['NOTIFY_SERVICE_ID'] for service in self.services ) @@ -326,7 +323,6 @@ SERVICE_PERMISSION_TYPES = [ EMAIL_TYPE, SMS_TYPE, LETTER_TYPE, - BROADCAST_TYPE, INTERNATIONAL_SMS_TYPE, INBOUND_SMS_TYPE, SCHEDULE_NOTIFICATIONS, @@ -531,9 +527,6 @@ class Service(db.Model, Versioned): uselist=False, backref=db.backref('services', lazy='dynamic')) - allowed_broadcast_provider = association_proxy('service_broadcast_settings', 'provider') - broadcast_channel = association_proxy('service_broadcast_settings', 'channel') - @classmethod def from_json(cls, data): """ @@ -577,13 +570,6 @@ class Service(db.Model, Versioned): 'research_mode': self.research_mode } - def get_available_broadcast_providers(self): - # There may be future checks here if we add, for example, platform admin level provider killswitches. - if self.allowed_broadcast_provider != ALL_BROADCAST_PROVIDERS: - return [x for x in current_app.config['ENABLED_CBCS'] if x == self.allowed_broadcast_provider] - else: - return current_app.config['ENABLED_CBCS'] - class AnnualBilling(db.Model): __tablename__ = "annual_billing" @@ -953,7 +939,6 @@ class TemplateBase(db.Model): hidden = db.Column(db.Boolean, nullable=False, default=False) subject = db.Column(db.Text) postage = db.Column(db.String, nullable=True) - broadcast_data = db.Column(JSONB(none_as_null=True), nullable=True) @declared_attr def service_id(cls): @@ -1026,8 +1011,6 @@ class TemplateBase(db.Model): return PlainTextEmailTemplate(self.__dict__) if self.template_type == SMS_TYPE: return SMSMessageTemplate(self.__dict__) - if self.template_type == BROADCAST_TYPE: - return BroadcastMessageTemplate(self.__dict__) if self.template_type == LETTER_TYPE: return LetterPrintTemplate( self.__dict__, @@ -1891,10 +1874,6 @@ SEND_LETTERS = 'send_letters' MANAGE_API_KEYS = 'manage_api_keys' PLATFORM_ADMIN = 'platform_admin' VIEW_ACTIVITY = 'view_activity' -CREATE_BROADCASTS = 'create_broadcasts' -APPROVE_BROADCASTS = 'approve_broadcasts' -CANCEL_BROADCASTS = 'cancel_broadcasts' -REJECT_BROADCASTS = 'reject_broadcasts' # List of permissions PERMISSION_LIST = [ @@ -1907,10 +1886,6 @@ PERMISSION_LIST = [ MANAGE_API_KEYS, PLATFORM_ADMIN, VIEW_ACTIVITY, - CREATE_BROADCASTS, - APPROVE_BROADCASTS, - CANCEL_BROADCASTS, - REJECT_BROADCASTS, ] @@ -2263,400 +2238,6 @@ class ServiceContactList(db.Model): return contact_list -class BroadcastStatusType(db.Model): - __tablename__ = 'broadcast_status_type' - DRAFT = 'draft' - PENDING_APPROVAL = 'pending-approval' - REJECTED = 'rejected' - BROADCASTING = 'broadcasting' - COMPLETED = 'completed' - CANCELLED = 'cancelled' - TECHNICAL_FAILURE = 'technical-failure' - - STATUSES = [DRAFT, PENDING_APPROVAL, REJECTED, BROADCASTING, COMPLETED, CANCELLED, TECHNICAL_FAILURE] - - # a broadcast message can be edited while in one of these states - PRE_BROADCAST_STATUSES = [DRAFT, PENDING_APPROVAL, REJECTED] - LIVE_STATUSES = [BROADCASTING, COMPLETED, CANCELLED] - - # these are only the transitions we expect to administer via the API code. - ALLOWED_STATUS_TRANSITIONS = { - DRAFT: {PENDING_APPROVAL}, - PENDING_APPROVAL: {REJECTED, DRAFT, BROADCASTING}, - REJECTED: {DRAFT, PENDING_APPROVAL}, - BROADCASTING: {COMPLETED, CANCELLED}, - COMPLETED: {}, - CANCELLED: {}, - TECHNICAL_FAILURE: {}, - } - - name = db.Column(db.String, primary_key=True) - - -class BroadcastMessage(db.Model): - """ - This is for creating a message, viewing it in notify, adding areas, approvals, drafts, etc. Notify logic before - hitting send. - """ - __tablename__ = 'broadcast_message' - __table_args__ = ( - db.ForeignKeyConstraint( - ['template_id', 'template_version'], - ['templates_history.id', 'templates_history.version'], - ), - {} - ) - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id')) - service = db.relationship('Service', backref='broadcast_messages') - - template_id = db.Column(UUID(as_uuid=True), nullable=True) - template_version = db.Column(db.Integer, nullable=True) - template = db.relationship('TemplateHistory', backref='broadcast_messages') - - _personalisation = db.Column(db.String, nullable=True) - content = db.Column(db.String, nullable=False) - # defaults to empty list - areas = db.Column(JSONB(none_as_null=True), nullable=False, default=list) - - status = db.Column( - db.String, - db.ForeignKey('broadcast_status_type.name'), - nullable=False, - default=BroadcastStatusType.DRAFT - ) - - # these times are related to the actual broadcast, rather than auditing purposes - starts_at = db.Column(db.DateTime, nullable=True) - finishes_at = db.Column(db.DateTime, nullable=True) # isn't updated if user cancels - - # these times correspond to when - created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - approved_at = db.Column(db.DateTime, nullable=True) - cancelled_at = db.Column(db.DateTime, nullable=True) - updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) - - created_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), nullable=True) - approved_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), nullable=True) - cancelled_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), nullable=True) - - created_by = db.relationship('User', foreign_keys=[created_by_id]) - approved_by = db.relationship('User', foreign_keys=[approved_by_id]) - cancelled_by = db.relationship('User', foreign_keys=[cancelled_by_id]) - - created_by_api_key_id = db.Column(UUID(as_uuid=True), db.ForeignKey('api_keys.id'), nullable=True) - cancelled_by_api_key_id = db.Column(UUID(as_uuid=True), db.ForeignKey('api_keys.id'), nullable=True) - created_by_api_key = db.relationship('ApiKey', foreign_keys=[created_by_api_key_id]) - cancelled_by_api_key = db.relationship('ApiKey', foreign_keys=[cancelled_by_api_key_id]) - - reference = db.Column(db.String(255), nullable=True) - cap_event = db.Column(db.String(255), nullable=True) - - stubbed = db.Column(db.Boolean, nullable=False) - - CheckConstraint("created_by_id is not null or created_by_api_key_id is not null") - - @property - def personalisation(self): - if self._personalisation: - return encryption.decrypt(self._personalisation) - return {} - - @personalisation.setter - def personalisation(self, personalisation): - self._personalisation = encryption.encrypt(personalisation or {}) - - def serialize(self): - return { - 'id': str(self.id), - 'reference': self.reference, - 'cap_event': self.cap_event, - - 'service_id': str(self.service_id), - - 'template_id': str(self.template_id) if self.template else None, - 'template_version': self.template_version, - 'template_name': self.template.name if self.template else None, - 'personalisation': self.personalisation if self.template else None, - 'content': self.content, - - 'areas': self.areas, - 'status': self.status, - - 'starts_at': get_dt_string_or_none(self.starts_at), - 'finishes_at': get_dt_string_or_none(self.finishes_at), - - 'created_at': get_dt_string_or_none(self.created_at), - 'approved_at': get_dt_string_or_none(self.approved_at), - 'cancelled_at': get_dt_string_or_none(self.cancelled_at), - 'updated_at': get_dt_string_or_none(self.updated_at), - - 'created_by_id': get_uuid_string_or_none(self.created_by_id), - 'approved_by_id': get_uuid_string_or_none(self.approved_by_id), - 'cancelled_by_id': get_uuid_string_or_none(self.cancelled_by_id), - } - - -class BroadcastEventMessageType: - ALERT = 'alert' - UPDATE = 'update' - CANCEL = 'cancel' - - MESSAGE_TYPES = [ALERT, UPDATE, CANCEL] - - -class BroadcastEvent(db.Model): - """ - This table represents an instruction that we will send to the broadcast providers. It directly correlates with an - instruction from the admin - to broadcast a message, to cancel an existing message, or to update an existing one. - - We should be able to create the complete CAP message without joining from this to any other tables, eg - template, service, or broadcast_message. - - The only exception to this is that we will have to join to itself to find other broadcast_events with the - same broadcast_message_id when building up the `` xml field for updating/cancelling an existing message. - - As such, this shouldn't have foreign keys to things that can change or be deleted. - """ - __tablename__ = 'broadcast_event' - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id')) - service = db.relationship('Service') - - broadcast_message_id = db.Column(UUID(as_uuid=True), db.ForeignKey('broadcast_message.id'), nullable=False) - broadcast_message = db.relationship('BroadcastMessage', backref='events') - - # this is used for in the cap xml - sent_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - - # msgType. alert, cancel, or update. (other options in the spec are "ack" and "error") - message_type = db.Column(db.String, nullable=False) - - # this will be json containing anything that isnt hardcoded in utils/cbc proxy. for now just body but may grow to - # include, eg, title, headline, instructions. - transmitted_content = db.Column( - JSONB(none_as_null=True), - nullable=True - ) - # unsubstantiated reckon: even if we're sending a cancel, we'll still need to provide areas - transmitted_areas = db.Column(JSONB(none_as_null=True), nullable=False, default=list) - transmitted_sender = db.Column(db.String(), nullable=False) - - # we may only need this starts_at if this is scheduled for the future. Interested to see how this affects - # updates/cancels (ie: can you schedule an update for the future?) - transmitted_starts_at = db.Column(db.DateTime, nullable=True) - transmitted_finishes_at = db.Column(db.DateTime, nullable=True) - - @property - def reference(self): - notify_email_domain = current_app.config['NOTIFY_EMAIL_DOMAIN'] - return ( - f'https://www.{notify_email_domain}/,' - f'{self.id},' - f'{self.sent_at_as_cap_datetime_string}' - ) - - @property - def sent_at_as_cap_datetime_string(self): - return self.formatted_datetime_for('sent_at') - - @property - def transmitted_finishes_at_as_cap_datetime_string(self): - return self.formatted_datetime_for('transmitted_finishes_at') - - def formatted_datetime_for(self, property_name): - return self.convert_naive_utc_datetime_to_cap_standard_string( - getattr(self, property_name) - ) - - @staticmethod - def convert_naive_utc_datetime_to_cap_standard_string(dt): - """ - As defined in section 3.3.2 of - http://docs.oasis-open.org/emergency/cap/v1.2/CAP-v1.2-os.html - They define the standard "YYYY-MM-DDThh:mm:ssXzh:zm", where X is - `+` if the timezone is > UTC, otherwise `-` - """ - return f"{dt.strftime('%Y-%m-%dT%H:%M:%S')}-00:00" - - def get_provider_message(self, provider): - return next( - ( - provider_message - for provider_message in self.provider_messages - if provider_message.provider == provider - ), - None - ) - - def get_earlier_provider_messages(self, provider): - """ - Get the previous message for a provider. These are different per provider, as the identifiers are different. - Return the full provider_message object rather than just an identifier, since the different providers expect - reference to contain different things - let the cbc_proxy work out what information is relevant. - """ - from app.dao.broadcast_message_dao import ( - get_earlier_events_for_broadcast_event, - ) - earlier_events = [ - event for event in get_earlier_events_for_broadcast_event(self.id) - ] - ret = [] - for event in earlier_events: - provider_message = event.get_provider_message(provider) - if provider_message is None: - # TODO: We should figure out what to do if a previous message hasn't been sent out yet. - # We don't want to not cancel a message just because it's stuck in a queue somewhere. - # This exception should probably be named, and then should be caught further up and handled - # appropriately. - raise Exception( - f'Cannot get earlier message references for event {self.id}, previous event {event.id} has not ' + - f' been sent to provider "{provider}" yet' - ) - ret.append(provider_message) - return ret - - def serialize(self): - return { - 'id': str(self.id), - - 'service_id': str(self.service_id), - - 'broadcast_message_id': str(self.broadcast_message_id), - # sent_at is required by BroadcastMessageTemplate.from_broadcast_event - 'sent_at': self.sent_at.strftime(DATETIME_FORMAT), - 'message_type': self.message_type, - - 'transmitted_content': self.transmitted_content, - 'transmitted_areas': self.transmitted_areas, - 'transmitted_sender': self.transmitted_sender, - - 'transmitted_starts_at': get_dt_string_or_none(self.transmitted_starts_at), - # transmitted_finishes_at is required by BroadcastMessageTemplate.from_broadcast_event - 'transmitted_finishes_at': self.transmitted_finishes_at.strftime(DATETIME_FORMAT), - - } - - -class BroadcastProvider: - EE = 'ee' - VODAFONE = 'vodafone' - THREE = 'three' - O2 = 'o2' - - PROVIDERS = [EE, VODAFONE, THREE, O2] - - -ALL_BROADCAST_PROVIDERS = 'all' - - -class BroadcastProviderMessageStatus: - TECHNICAL_FAILURE = 'technical-failure' # Couldn’t send (cbc proxy 5xx/4xx) - SENDING = 'sending' # Sent to cbc, awaiting response - ACK = 'returned-ack' # Received ack response - ERR = 'returned-error' # Received error response - - STATES = [TECHNICAL_FAILURE, SENDING, ACK, ERR] - - -class BroadcastProviderMessage(db.Model): - """ - A row in this table represents the XML blob sent to a single provider. - """ - __tablename__ = 'broadcast_provider_message' - - id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) - - broadcast_event_id = db.Column(UUID(as_uuid=True), db.ForeignKey('broadcast_event.id')) - broadcast_event = db.relationship('BroadcastEvent', backref='provider_messages') - - # 'ee', 'three', 'vodafone', etc - provider = db.Column(db.String) - - status = db.Column(db.String) - - created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) - - UniqueConstraint(broadcast_event_id, provider) - - message_number = association_proxy('broadcast_provider_message_number', 'broadcast_provider_message_number') - - -class BroadcastProviderMessageNumber(db.Model): - """ - To send IBAG messages via the CBC proxy to Nokia CBC appliances, Notify must generate and store a numeric - message_number alongside the message ID (GUID). - Subsequent messages (Update, Cancel) in IBAG format must reference the original message_number & message_id. - This model relates broadcast_provider_message_id to that numeric message_number. - """ - __tablename__ = 'broadcast_provider_message_number' - - sequence = Sequence('broadcast_provider_message_number_seq') - broadcast_provider_message_number = db.Column( - db.Integer, sequence, server_default=sequence.next_value(), primary_key=True - ) - broadcast_provider_message_id = db.Column( - UUID(as_uuid=True), db.ForeignKey('broadcast_provider_message.id'), nullable=False - ) - broadcast_provider_message = db.relationship( - 'BroadcastProviderMessage', backref=db.backref("broadcast_provider_message_number", uselist=False) - ) - - -class ServiceBroadcastSettings(db.Model): - """ - Every broadcast service should have one and only one row in this table. - """ - __tablename__ = "service_broadcast_settings" - - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), primary_key=True, nullable=False) - service = db.relationship(Service, backref=db.backref("service_broadcast_settings", uselist=False)) - channel = db.Column( - db.String(255), db.ForeignKey('broadcast_channel_types.name'), nullable=False - ) - provider = db.Column(db.String, db.ForeignKey('broadcast_provider_types.name'), nullable=False) - created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - updated_at = db.Column(db.DateTime, nullable=True, onupdate=datetime.datetime.utcnow) - - -class BroadcastChannelTypes(db.Model): - __tablename__ = 'broadcast_channel_types' - - name = db.Column(db.String(255), primary_key=True) - - -class BroadcastProviderTypes(db.Model): - __tablename__ = 'broadcast_provider_types' - - name = db.Column(db.String(255), primary_key=True) - - -class ServiceBroadcastProviderRestriction(db.Model): - """ - TODO: Drop this table as no longer used - - Most services don't send broadcasts. Of those that do, most send to all broadcast providers. - However, some services don't send to all providers. These services are test services that we or the providers - themselves use. - - This table links those services. There should only be one row per service in this table, and this is enforced by - the service_id being a primary key. - """ - __tablename__ = "service_broadcast_provider_restriction" - - service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), primary_key=True, nullable=False) - service = db.relationship(Service, backref=db.backref("service_broadcast_provider_restriction", uselist=False)) - - provider = db.Column(db.String, nullable=False) - - created_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow) - - class WebauthnCredential(db.Model): """ A table that stores data for registered webauthn credentials. diff --git a/app/schemas.py b/app/schemas.py index 5fc97bb7c..6ea50d16f 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -234,15 +234,7 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin): email_branding = field_for(models.Service, 'email_branding') organisation = field_for(models.Service, 'organisation') go_live_at = field_for(models.Service, 'go_live_at', format=DATETIME_FORMAT_NO_TIMEZONE) - allowed_broadcast_provider = fields.Method(dump_only=True, serialize='_get_allowed_broadcast_provider') - broadcast_channel = fields.Method(dump_only=True, serialize='_get_broadcast_channel') - - def _get_allowed_broadcast_provider(self, service): - return service.allowed_broadcast_provider - - def _get_broadcast_channel(self, service): - return service.broadcast_channel - + def get_letter_logo_filename(self, service): return service.letter_branding and service.letter_branding.filename @@ -270,7 +262,6 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin): 'all_template_folders', 'annual_billing', 'api_keys', - 'broadcast_messages', 'complaints', 'contact_list', 'created_at', @@ -284,8 +275,6 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin): 'letter_logo_filename', 'reply_to_email_addresses', 'returned_letters', - 'service_broadcast_provider_restriction', - 'service_broadcast_settings', 'service_sms_senders', 'templates', 'updated_at', @@ -331,7 +320,6 @@ class DetailedServiceSchema(BaseSchema): 'all_template_folders', 'annual_billing', 'api_keys', - 'broadcast_messages', 'contact_list', 'created_by', 'crown', @@ -413,7 +401,6 @@ class TemplateSchemaNoDetail(TemplateSchema): class Meta(TemplateSchema.Meta): exclude = TemplateSchema.Meta.exclude + ( 'archived', - 'broadcast_data', 'created_at', 'created_by', 'created_by_id', @@ -431,13 +418,6 @@ class TemplateSchemaNoDetail(TemplateSchema): 'version', ) - @pre_dump - def remove_content_for_non_broadcast_templates(self, template, **kwargs): - if template.template_type != models.BROADCAST_TYPE: - template.content = None - - return template - class TemplateHistorySchema(BaseSchema): @@ -457,7 +437,6 @@ class TemplateHistorySchema(BaseSchema): class Meta(BaseSchema.Meta): model = models.TemplateHistory - exclude = ('broadcast_messages',) class ApiKeySchema(BaseSchema): diff --git a/app/service/rest.py b/app/service/rest.py index 4b9eb4e19..7773b8dcd 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -21,7 +21,6 @@ from app.dao.api_key_dao import ( get_unsigned_secret, save_model_api_key, ) -from app.dao.broadcast_service_dao import set_broadcast_service_type from app.dao.dao_utils import dao_rollback, transaction from app.dao.date_util import get_financial_year from app.dao.fact_notification_status_dao import ( @@ -129,9 +128,6 @@ from app.service.send_notification import ( ) from app.service.send_pdf_letter_schema import send_pdf_letter_request from app.service.sender import send_notification_to_service_users -from app.service.service_broadcast_settings_schema import ( - service_broadcast_settings_schema, -) from app.service.service_contact_list_schema import ( create_service_contact_list_schema, ) @@ -1149,28 +1145,3 @@ def create_contact_list(service_id): save_service_contact_list(list_to_save) return jsonify(list_to_save.serialize()), 201 - - -@service_blueprint.route('//set-as-broadcast-service', methods=['POST']) -def set_as_broadcast_service(service_id): - """ - This route does the following - - adds a service broadcast settings to define which channel broadcasts should go out on - - removes all current service permissions and adds the broadcast service permission - - sets the services `count_as_live` to false - - adds the service to the broadcast organisation - - puts the service into training mode or live mode - - removes all permissions from current users and invited users - """ - data = validate(request.get_json(), service_broadcast_settings_schema) - service = dao_fetch_service_by_id(service_id) - - set_broadcast_service_type( - service, - service_mode=data["service_mode"], - broadcast_channel=data["broadcast_channel"], - provider_restriction=data["provider_restriction"] - ) - - data = service_schema.dump(service) - return jsonify(data=data) diff --git a/app/service/service_broadcast_settings_schema.py b/app/service/service_broadcast_settings_schema.py deleted file mode 100644 index 38c5e1d53..000000000 --- a/app/service/service_broadcast_settings_schema.py +++ /dev/null @@ -1,12 +0,0 @@ -service_broadcast_settings_schema = { - "$schema": "http://json-schema.org/draft-07/schema#", - "description": "Set a services broadcast settings", - "type": "object", - "title": "Set a services broadcast settings", - "properties": { - "broadcast_channel": {"enum": ["operator", "test", "severe", "government"]}, - "service_mode": {"enum": ["training", "live"]}, - "provider_restriction": {"enum": ["three", "o2", "vodafone", "ee", "all"]} - }, - "required": ["broadcast_channel", "service_mode", "provider_restriction"] -} diff --git a/app/service_invite/rest.py b/app/service_invite/rest.py index 2bb331d7d..4813e7bef 100644 --- a/app/service_invite/rest.py +++ b/app/service_invite/rest.py @@ -11,7 +11,7 @@ from app.dao.invited_user_dao import ( ) from app.dao.templates_dao import dao_get_template_by_id from app.errors import InvalidRequest, register_errors -from app.models import BROADCAST_TYPE, EMAIL_TYPE, KEY_TYPE_NORMAL, Service +from app.models import EMAIL_TYPE, KEY_TYPE_NORMAL, Service from app.notifications.process_notifications import ( persist_notification, send_notification_to_queue, @@ -29,10 +29,7 @@ def create_invited_user(service_id): invited_user = invited_user_schema.load(request_json) save_invited_user(invited_user) - if invited_user.service.has_permission(BROADCAST_TYPE): - template_id = current_app.config['BROADCAST_INVITATION_EMAIL_TEMPLATE_ID'] - else: - template_id = current_app.config['INVITATION_EMAIL_TEMPLATE_ID'] + template_id = current_app.config['INVITATION_EMAIL_TEMPLATE_ID'] template = dao_get_template_by_id(template_id) service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) diff --git a/app/template/rest.py b/app/template/rest.py index 3dc35c49c..45675eba9 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -5,10 +5,7 @@ import botocore from flask import Blueprint, current_app, jsonify, request from notifications_utils import SMS_CHAR_COUNT_LIMIT from notifications_utils.pdf import extract_page_from_pdf -from notifications_utils.template import ( - BroadcastMessageTemplate, - SMSMessageTemplate, -) +from notifications_utils.template import SMSMessageTemplate from PyPDF2.errors import PdfReadError from requests import post as requests_post from sqlalchemy.orm.exc import NoResultFound @@ -32,7 +29,6 @@ from app.dao.templates_dao import ( from app.errors import InvalidRequest, register_errors from app.letters.utils import get_letter_pdf_and_metadata from app.models import ( - BROADCAST_TYPE, LETTER_TYPE, SECOND_CLASS, SMS_TYPE, @@ -60,9 +56,6 @@ def _content_count_greater_than_limit(content, template_type): if template_type == SMS_TYPE: template = SMSMessageTemplate({'content': content, 'template_type': template_type}) return template.is_message_too_long() - if template_type == BROADCAST_TYPE: - template = BroadcastMessageTemplate({'content': content, 'template_type': template_type}) - return template.is_message_too_long() return False diff --git a/app/utils.py b/app/utils.py index abc0b0aa5..4ed476d58 100644 --- a/app/utils.py +++ b/app/utils.py @@ -3,7 +3,6 @@ from datetime import datetime, timedelta import pytz from flask import url_for from notifications_utils.template import ( - BroadcastMessageTemplate, HTMLEmailTemplate, LetterPrintTemplate, SMSMessageTemplate, @@ -48,12 +47,11 @@ def url_with_token(data, url, config, base_url=None): def get_template_instance(template, values): - from app.models import BROADCAST_TYPE, EMAIL_TYPE, LETTER_TYPE, SMS_TYPE + from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE return { SMS_TYPE: SMSMessageTemplate, EMAIL_TYPE: HTMLEmailTemplate, LETTER_TYPE: LetterPrintTemplate, - BROADCAST_TYPE: BroadcastMessageTemplate, }[template['template_type']](template, values) @@ -90,7 +88,6 @@ def get_london_month_from_utc_column(column): def get_public_notify_type_text(notify_type, plural=False): from app.models import ( - BROADCAST_TYPE, PRECOMPILED_LETTER, SMS_TYPE, UPLOAD_DOCUMENT, @@ -102,9 +99,7 @@ def get_public_notify_type_text(notify_type, plural=False): notify_type_text = 'document' elif notify_type == PRECOMPILED_LETTER: notify_type_text = 'precompiled letter' - elif notify_type == BROADCAST_TYPE: - notify_type_text = 'broadcast message' - + return '{}{}'.format(notify_type_text, 's' if plural else '') diff --git a/app/v2/broadcast/__init__.py b/app/v2/broadcast/__init__.py deleted file mode 100644 index 767cdb956..000000000 --- a/app/v2/broadcast/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from flask import Blueprint - -from app.v2.errors import register_errors - -v2_broadcast_blueprint = Blueprint( - "v2_broadcast_blueprint", - __name__, - url_prefix='/v2/broadcast', -) - -register_errors(v2_broadcast_blueprint) diff --git a/app/v2/broadcast/broadcast_schemas.py b/app/v2/broadcast/broadcast_schemas.py deleted file mode 100644 index f7d97f8a2..000000000 --- a/app/v2/broadcast/broadcast_schemas.py +++ /dev/null @@ -1,123 +0,0 @@ -post_broadcast_schema = { - "$schema": "http://json-schema.org/draft-07/schema", - "type": "object", - "required": [ - "msgType", - "reference", - "cap_event", - "category", - "content", - "areas", - ], - "additionalProperties": False, - "properties": { - "reference": { - "type": [ - "string", - "null", - ], - }, - "references": { - "type": [ - "string", - "null", - ], - }, - "cap_event": { - "type": [ - "string", - "null", - ], - }, - "category": { - "type": "string", - "enum": [ - "Geo", - "Met", - "Safety", - "Security", - "Rescue", - "Fire", - "Health", - "Env", - "Transport", - "Infra", - "CBRNE", - "Other", - ], - }, - "expires": { - "type": "string", - "format": "date-time", - }, - "content": { - "type": "string", - "minLength": 0, - }, - "web": { - "type": "string", - "format": "uri", - }, - "areas": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/definitions/area", - }, - }, - "msgType": { - "type": "string", - "enum": [ - "Alert", - "Cancel", - # The following are valid CAP but not supported by our - # API at the moment - # "Update", - # "Ack", - # "Error", - ], - } - }, - "definitions": { - "area": { - "type": "object", - "required": [ - "name", - "polygons", - ], - "additionalProperties": False, - "properties": { - "name": { - "type": "string", - "pattern": "([a-zA-Z1-9]+ )*[a-zA-Z1-9]+", - }, - "polygons": { - "type": "array", - "minItems": 1, - "items": { - "oneOf": [ - { - "$ref": "#/definitions/polygon", - }, - ], - }, - }, - }, - }, - "polygon": { - "type": "array", - "minItems": 4, - "items": { - "$ref": "#/definitions/coordinatePair", - }, - }, - "coordinatePair": { - "type": "array", - "items": { - "type": "number" - }, - "minItems": 2, - "maxItems": 2, - }, - }, -} diff --git a/app/v2/broadcast/post_broadcast.py b/app/v2/broadcast/post_broadcast.py deleted file mode 100644 index c3062efe9..000000000 --- a/app/v2/broadcast/post_broadcast.py +++ /dev/null @@ -1,148 +0,0 @@ -from itertools import chain - -from flask import current_app, jsonify, request -from notifications_utils.polygons import Polygons -from notifications_utils.template import BroadcastMessageTemplate -from sqlalchemy.orm.exc import MultipleResultsFound - -from app import api_user, authenticated_service, redis_store -from app.broadcast_message import utils as broadcast_utils -from app.broadcast_message.translators import cap_xml_to_dict -from app.dao.broadcast_message_dao import ( - dao_get_broadcast_message_by_references_and_service_id, -) -from app.dao.dao_utils import dao_save_object -from app.models import BROADCAST_TYPE, BroadcastMessage, BroadcastStatusType -from app.notifications.validators import check_service_has_permission -from app.schema_validation import validate -from app.v2.broadcast import v2_broadcast_blueprint -from app.v2.broadcast.broadcast_schemas import post_broadcast_schema -from app.v2.errors import BadRequestError, ValidationError -from app.xml_schemas import validate_xml - - -@v2_broadcast_blueprint.route("", methods=['POST']) -def create_broadcast(): - - check_service_has_permission( - BROADCAST_TYPE, - authenticated_service.permissions, - ) - - if request.content_type != 'application/cap+xml': - raise BadRequestError( - message=f'Content type {request.content_type} not supported', - status_code=415, - ) - - cap_xml = request.get_data() - - if not validate_xml(cap_xml, 'CAP-v1.2.xsd'): - raise BadRequestError( - message='Request data is not valid CAP XML', - status_code=400, - ) - broadcast_json = cap_xml_to_dict(cap_xml) - - validate(broadcast_json, post_broadcast_schema) - - if broadcast_json["msgType"] == "Cancel": - if broadcast_json["references"] is None: - raise BadRequestError( - message='Missing ', - status_code=400, - ) - broadcast_message = _cancel_or_reject_broadcast( - broadcast_json["references"].split(","), - authenticated_service.id - ) - return jsonify(broadcast_message.serialize()), 201 - - else: - _validate_template(broadcast_json) - - polygons = Polygons(list(chain.from_iterable(( - [ - [[y, x] for x, y in polygon] - for polygon in area['polygons'] - ] for area in broadcast_json['areas'] - )))) - - if len(polygons) > 12 or polygons.point_count > 250: - simple_polygons = polygons.smooth.simplify - else: - simple_polygons = polygons - - broadcast_message = BroadcastMessage( - service_id=authenticated_service.id, - content=broadcast_json['content'], - reference=broadcast_json['reference'], - cap_event=broadcast_json['cap_event'], - areas={ - 'names': [ - area['name'] for area in broadcast_json['areas'] - ], - 'simple_polygons': simple_polygons.as_coordinate_pairs_lat_long, - }, - status=BroadcastStatusType.PENDING_APPROVAL, - created_by_api_key_id=api_user.id, - stubbed=authenticated_service.restricted - # The client may pass in broadcast_json['expires'] but it’s - # simpler for now to ignore it and have the rules around expiry - # for broadcasts created with the API match those created from - # the admin app - ) - - dao_save_object(broadcast_message) - - current_app.logger.info( - f'Broadcast message {broadcast_message.id} created for service ' - f'{authenticated_service.id} with reference {broadcast_json["reference"]}' - ) - - return jsonify(broadcast_message.serialize()), 201 - - -def _cancel_or_reject_broadcast(references_to_original_broadcast, service_id): - try: - broadcast_message = dao_get_broadcast_message_by_references_and_service_id( - references_to_original_broadcast, - service_id - ) - except MultipleResultsFound: - raise BadRequestError( - message='Multiple alerts found - unclear which one to cancel', - status_code=400, - ) - - if broadcast_message.status == BroadcastStatusType.PENDING_APPROVAL: - new_status = BroadcastStatusType.REJECTED - else: - new_status = BroadcastStatusType.CANCELLED - broadcast_utils.update_broadcast_message_status( - broadcast_message, - new_status, - api_key_id=api_user.id - ) - redis_store.delete( - f'service-{broadcast_message.service_id}-broadcast-message-{broadcast_message.id}' - ) - return broadcast_message - - -def _validate_template(broadcast_json): - template = BroadcastMessageTemplate.from_content( - broadcast_json['content'] - ) - - if template.content_too_long: - raise ValidationError( - message=( - f'description must be {template.max_content_count:,.0f} ' - f'characters or fewer' - ) + ( - ' (because it could not be GSM7 encoded)' - if template.non_gsm_characters else '' - ), - status_code=400, - ) diff --git a/app/xml_schemas/CAP-v1.2.xsd b/app/xml_schemas/CAP-v1.2.xsd deleted file mode 100644 index ed97952be..000000000 --- a/app/xml_schemas/CAP-v1.2.xsd +++ /dev/null @@ -1,218 +0,0 @@ - - - - - - CAP Alert Message (version 1.2) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/app/xml_schemas/__init__.py b/app/xml_schemas/__init__.py deleted file mode 100644 index fc02965d0..000000000 --- a/app/xml_schemas/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from pathlib import Path - -from defusedxml.lxml import fromstring -# there is no equivalent in defusedxml to validate a schema -from lxml.etree import XMLSchema # nosec B410 - - -def validate_xml(document, schema_file_name): - - path = Path(__file__).resolve().parent / schema_file_name - contents = path.read_text() - - schema_root = fromstring(contents.encode('utf-8')) - schema = XMLSchema(schema_root) - return schema.validate(fromstring(document)) diff --git a/docs/writing-public-apis.md b/docs/writing-public-apis.md index 3bca46683..f37f6e2d7 100644 --- a/docs/writing-public-apis.md +++ b/docs/writing-public-apis.md @@ -45,6 +45,6 @@ Each adapter should be documented in each client ([example](https://github.com/a This is done as part of registering the blueprint in `app/__init__.py` e.g. ``` -post_broadcast.before_request(requires_auth) -application.register_blueprint(post_broadcast) +post_letter.before_request(requires_auth) +application.register_blueprint(post_letter) ``` diff --git a/sample.env b/sample.env index 78761beac..187c36056 100644 --- a/sample.env +++ b/sample.env @@ -13,7 +13,6 @@ NOTIFY_LOG_PATH=/workspace/logs/app.log # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID=notify-admin ADMIN_CLIENT_SECRET=dev-notify-secret-key -GOVUK_ALERTS_CLIENT_ID=govuk-alerts # Flask FLASK_APP=application.py diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh index 7aeb46868..c06906c1b 100755 --- a/scripts/paas_app_wrapper.sh +++ b/scripts/paas_app_wrapper.sh @@ -41,10 +41,6 @@ case $NOTIFY_APP_NAME in exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ -Q notify-internal-tasks 2> /dev/null ;; - delivery-worker-broadcasts) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ - -Q broadcast-tasks 2> /dev/null - ;; delivery-worker-receipts) exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ -Q ses-callbacks,sms-callbacks 2> /dev/null diff --git a/tests/app/authentication/test_authentication.py b/tests/app/authentication/test_authentication.py index d5ceaac84..55bc59fc3 100644 --- a/tests/app/authentication/test_authentication.py +++ b/tests/app/authentication/test_authentication.py @@ -78,14 +78,6 @@ def test_requires_admin_auth_should_allow_valid_token_for_request(client): assert response.status_code == 200 -@pytest.mark.skip(reason="Needs updating for TTS") -def test_requires_govuk_alerts_auth_should_allow_valid_token_for_request(client): - jwt_client_id = current_app.config['GOVUK_ALERTS_CLIENT_ID'] - header = create_internal_authorization_header(jwt_client_id) - response = client.get('/govuk-alerts', headers=[header]) - assert response.status_code == 200 - - def test_get_auth_token_should_not_allow_request_with_no_token(client): request.headers = {} with pytest.raises(AuthError) as exc: diff --git a/tests/app/broadcast_message/__init__.py b/tests/app/broadcast_message/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/app/broadcast_message/test_rest.py b/tests/app/broadcast_message/test_rest.py deleted file mode 100644 index d26093851..000000000 --- a/tests/app/broadcast_message/test_rest.py +++ /dev/null @@ -1,658 +0,0 @@ -import uuid - -import pytest -from freezegun import freeze_time - -from app.dao.broadcast_message_dao import ( - dao_get_broadcast_message_by_id_and_service_id, -) -from app.models import ( - BROADCAST_TYPE, - BroadcastEventMessageType, - BroadcastStatusType, -) -from tests.app.db import ( - create_broadcast_message, - create_service, - create_template, - create_user, -) - - -def test_get_broadcast_message( - admin_request, - sample_broadcast_service -): - t = create_template( - sample_broadcast_service, - BROADCAST_TYPE, - content='This is a ((thing))' - ) - bm = create_broadcast_message( - t, - areas={ - "ids": ["place A", "region B"], - "simple_polygons": [[[50.1, 1.2], [50.12, 1.2], [50.13, 1.2]]], - }, - personalisation={ - 'thing': 'test', - }, - ) - - response = admin_request.get( - 'broadcast_message.get_broadcast_message', - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['id'] == str(bm.id) - assert response['template_id'] == str(t.id) - assert response['content'] == 'This is a test' - assert response['template_name'] == t.name - assert response['status'] == BroadcastStatusType.DRAFT - assert response['created_at'] is not None - assert response['starts_at'] is None - assert response['areas']['ids'] == ['place A', 'region B'] - assert response['areas']['simple_polygons'] == [[[50.1, 1.2], [50.12, 1.2], [50.13, 1.2]]] - assert response['personalisation'] == {'thing': 'test'} - - -def test_get_broadcast_message_without_template( - admin_request, - sample_broadcast_service -): - bm = create_broadcast_message( - service=sample_broadcast_service, - content='emergency broadcast content', - areas={ - "ids": ["place A", "region B"], - "simple_polygons": [[[50.1, 1.2], [50.12, 1.2], [50.13, 1.2]]], - }, - ) - - response = admin_request.get( - 'broadcast_message.get_broadcast_message', - service_id=sample_broadcast_service.id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['id'] == str(bm.id) - assert response['template_id'] is None - assert response['template_version'] is None - assert response['template_name'] is None - assert response['content'] == 'emergency broadcast content' - assert response['status'] == BroadcastStatusType.DRAFT - assert response['created_at'] is not None - assert response['starts_at'] is None - assert response['areas']['ids'] == ['place A', 'region B'] - assert response['areas']['simple_polygons'] == [[[50.1, 1.2], [50.12, 1.2], [50.13, 1.2]]] - assert response['personalisation'] is None - - -def test_get_broadcast_message_with_event( - admin_request, - sample_broadcast_service -): - bm = create_broadcast_message( - service=sample_broadcast_service, - content='emergency broadcast content', - cap_event='001 example event', - ) - - response = admin_request.get( - 'broadcast_message.get_broadcast_message', - service_id=sample_broadcast_service.id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['cap_event'] == '001 example event' - - -def test_get_broadcast_message_404s_if_message_doesnt_exist(admin_request, sample_broadcast_service): - err = admin_request.get( - 'broadcast_message.get_broadcast_message', - service_id=sample_broadcast_service.id, - broadcast_message_id=uuid.uuid4(), - _expected_status=404 - ) - assert err == {'message': 'No result found', 'result': 'error'} - - -def test_get_broadcast_message_404s_if_message_is_for_different_service(admin_request, sample_broadcast_service): - other_service = create_service(service_name='other') - other_template = create_template(other_service, BROADCAST_TYPE) - bm = create_broadcast_message(other_template) - - err = admin_request.get( - 'broadcast_message.get_broadcast_message', - service_id=sample_broadcast_service.id, - broadcast_message_id=bm.id, - _expected_status=404 - ) - assert err == {'message': 'No result found', 'result': 'error'} - - -@freeze_time('2020-01-01') -def test_get_broadcast_messages_for_service(admin_request, sample_broadcast_service): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - - with freeze_time('2020-01-01 12:00'): - bm1 = create_broadcast_message(t, personalisation={'foo': 'bar'}) - with freeze_time('2020-01-01 13:00'): - bm2 = create_broadcast_message(t, personalisation={'foo': 'baz'}) - - response = admin_request.get( - 'broadcast_message.get_broadcast_messages_for_service', - service_id=t.service_id, - _expected_status=200 - ) - - assert response['broadcast_messages'][0]['id'] == str(bm1.id) - assert response['broadcast_messages'][1]['id'] == str(bm2.id) - - -@freeze_time('2020-01-01') -@pytest.mark.parametrize('training_mode_service', [True, False]) -def test_create_broadcast_message(admin_request, sample_broadcast_service, training_mode_service): - sample_broadcast_service.restricted = training_mode_service - t = create_template( - sample_broadcast_service, - BROADCAST_TYPE, - content='Some content\r\n€ŷš~\r\nβ€˜β€™β€œβ€β€”β€“-', - ) - - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'template_id': str(t.id), - 'service_id': str(t.service_id), - 'created_by': str(t.created_by_id), - }, - service_id=t.service_id, - _expected_status=201 - ) - - assert response['template_name'] == t.name - assert response['status'] == BroadcastStatusType.DRAFT - assert response['created_at'] is not None - assert response['created_by_id'] == str(t.created_by_id) - assert response['personalisation'] == {} - assert response['areas'] == {} - assert response['content'] == 'Some content\n€ŷš~\n\'\'""---' - - broadcast_message = dao_get_broadcast_message_by_id_and_service_id(response["id"], sample_broadcast_service.id) - assert broadcast_message.stubbed == training_mode_service - - -@pytest.mark.parametrize('data, expected_errors', [ - ( - {}, - [ - {'error': 'ValidationError', 'message': 'service_id is a required property'}, - {'error': 'ValidationError', 'message': 'created_by is a required property'}, - {'error': 'ValidationError', 'message': '{} is not valid under any of the given schemas'}, - ] - ), - ( - { - 'template_id': str(uuid.uuid4()), - 'service_id': str(uuid.uuid4()), - 'created_by': str(uuid.uuid4()), - 'foo': 'something else' - }, - [ - {'error': 'ValidationError', 'message': 'Additional properties are not allowed (foo was unexpected)'} - ] - ) -]) -def test_create_broadcast_message_400s_if_json_schema_fails_validation( - admin_request, - sample_broadcast_service, - data, - expected_errors -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data=data, - service_id=t.service_id, - _expected_status=400 - ) - assert response['errors'] == expected_errors - - -@pytest.mark.parametrize('content, expected_status, expected_errors', ( - ('a', 201, None), - ('a' * 1_395, 201, None), - ('a\r\n' * 697, 201, None), # 1,394 chars – new lines normalised to \n - ('a' * 1_396, 400, ( - 'Content must be 1,395 characters or fewer' - )), - ('Ε΅' * 615, 201, None), - ('Ε΅' * 616, 400, ( - 'Content must be 615 characters or fewer ' - '(because it could not be GSM7 encoded)' - )), -)) -def test_create_broadcast_message_400s_if_content_too_long( - admin_request, - sample_broadcast_service, - content, - expected_status, - expected_errors, -): - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - service_id=sample_broadcast_service.id, - _data={ - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - 'reference': 'abc123', - 'content': content, - }, - _expected_status=expected_status, - ) - assert response.get('message') == expected_errors - - -@freeze_time('2020-01-01') -def test_create_broadcast_message_can_be_created_from_content(admin_request, sample_broadcast_service): - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'content': 'Some content\r\n€ŷš~\r\nβ€˜β€™β€œβ€β€”β€“-', - 'reference': 'abc123', - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - }, - service_id=sample_broadcast_service.id, - _expected_status=201 - ) - assert response['content'] == 'Some content\n€ŷš~\n\'\'""---' - assert response['reference'] == 'abc123' - assert response['template_id'] is None - assert response['cap_event'] is None - - -def test_create_broadcast_message_400s_if_content_and_template_provided( - admin_request, - sample_broadcast_service, -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'template_id': str(template.id), - 'content': 'Some tailor made broadcast content', - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - }, - service_id=sample_broadcast_service.id, - _expected_status=400 - ) - - assert len(response['errors']) == 1 - assert response['errors'][0]['error'] == 'ValidationError' - # The error message for oneOf is ugly, non-deterministic in ordering - # and contains some UUID, so let’s just pick out the important bits - assert ( - ' is valid under each of ' - ) in response['errors'][0]['message'] - assert ( - '{required: [content]}' - ) in response['errors'][0]['message'] - assert ( - '{required: [template_id]}' - ) in response['errors'][0]['message'] - - -def test_create_broadcast_message_400s_if_reference_and_template_provided( - admin_request, - sample_broadcast_service, -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'template_id': str(template.id), - 'reference': 'abc123', - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - }, - service_id=sample_broadcast_service.id, - _expected_status=400 - ) - - assert len(response['errors']) == 1 - assert response['errors'][0]['error'] == 'ValidationError' - # The error message for oneOf is ugly, non-deterministic in ordering - # and contains some UUID, so let’s just pick out the important bits - assert ( - ' is valid under each of ' - ) in response['errors'][0]['message'] - assert ( - '{required: [reference]}' - ) in response['errors'][0]['message'] - assert ( - '{required: [template_id]}' - ) in response['errors'][0]['message'] - - -def test_create_broadcast_message_400s_if_reference_not_provided_with_content( - admin_request, - sample_broadcast_service, -): - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'content': 'Some tailor made broadcast content', - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - }, - service_id=sample_broadcast_service.id, - _expected_status=400 - ) - assert len(response['errors']) == 1 - assert response['errors'][0]['error'] == 'ValidationError' - assert response['errors'][0]['message'].endswith( - 'is not valid under any of the given schemas' - ) - - -def test_create_broadcast_message_400s_if_no_content_or_template( - admin_request, - sample_broadcast_service, -): - response = admin_request.post( - 'broadcast_message.create_broadcast_message', - _data={ - 'service_id': str(sample_broadcast_service.id), - 'created_by': str(sample_broadcast_service.created_by_id), - }, - service_id=sample_broadcast_service.id, - _expected_status=400 - ) - assert len(response['errors']) == 1 - assert response['errors'][0]['error'] == 'ValidationError' - assert response['errors'][0]['message'].endswith( - 'is not valid under any of the given schemas' - ) - - -@pytest.mark.parametrize('status', [ - BroadcastStatusType.DRAFT, - BroadcastStatusType.PENDING_APPROVAL, - BroadcastStatusType.REJECTED, -]) -def test_update_broadcast_message_allows_edit_while_not_yet_live( - admin_request, - sample_broadcast_service, - status -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message( - t, - areas={ - "ids": ['manchester'], - "simple_polygons": [[[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]]] - }, - status=status - ) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data={ - "starts_at": "2020-06-01 20:00:01", - "areas": { - "ids": ["london", "glasgow"], - "simple_polygons": [[[51.12, 0.2], [50.13, 0.4], [50.14, 0.45]]] - }, - }, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' - assert response['areas']['ids'] == ['london', 'glasgow'] - assert response['areas']['simple_polygons'] == [[[51.12, 0.2], [50.13, 0.4], [50.14, 0.45]]] - assert response['updated_at'] is not None - - -@pytest.mark.parametrize('status', [ - BroadcastStatusType.BROADCASTING, - BroadcastStatusType.CANCELLED, - BroadcastStatusType.COMPLETED, - BroadcastStatusType.TECHNICAL_FAILURE, -]) -def test_update_broadcast_message_doesnt_allow_edits_after_broadcast_goes_live( - admin_request, - sample_broadcast_service, - status -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t, status=status) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data={'areas': {'ids': ['london', 'glasgow']}}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=400 - ) - assert f'status {status}' in response['message'] - - -def test_update_broadcast_message_sets_finishes_at_separately(admin_request, sample_broadcast_service): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message( - t, - areas={ - "ids": ["london"], - "simple_polygons": [[[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]]] - } - ) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data={'starts_at': '2020-06-01 20:00:01', 'finishes_at': '2020-06-02 20:00:01'}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' - assert response['finishes_at'] == '2020-06-02T20:00:01.000000Z' - assert response['updated_at'] is not None - - -@pytest.mark.parametrize('input_dt', [ - '2020-06-01 20:00:01', - '2020-06-01T20:00:01', - '2020-06-01 20:00:01Z', - '2020-06-01T20:00:01+00:00', -]) -def test_update_broadcast_message_allows_sensible_datetime_formats(admin_request, sample_broadcast_service, input_dt): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data={'starts_at': input_dt}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['starts_at'] == '2020-06-01T20:00:01.000000Z' - assert response['updated_at'] is not None - - -def test_update_broadcast_message_doesnt_let_you_update_status(admin_request, sample_broadcast_service): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data={ - "areas": { - "ids": ["glasgow"], - "simple_polygons": [[[55.86, -4.25], [55.85, -4.25], [55.87, -4.24]]], - }, - "status": BroadcastStatusType.BROADCASTING}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=400 - ) - - assert response['errors'] == [{ - 'error': 'ValidationError', - 'message': 'Additional properties are not allowed (status was unexpected)' - }] - - -@pytest.mark.parametrize("incomplete_area_data", [ - {"areas": {"ids": ["cardiff"]}}, - {"areas": {"simple_polygons": [[[51.28, -3.11], [51.29, -3.12], [51.27, -3.10]]]}}, -]) -def test_update_broadcast_message_doesnt_let_you_update_areas_but_not_polygons( - admin_request, sample_broadcast_service, incomplete_area_data -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message', - _data=incomplete_area_data, - service_id=template.service_id, - broadcast_message_id=broadcast_message.id, - _expected_status=400 - ) - - assert response[ - 'message' - ] == f'Cannot update broadcast_message {broadcast_message.id}, area IDs or polygons are missing.' - - -def test_update_broadcast_message_status(admin_request, sample_broadcast_service): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t, status=BroadcastStatusType.DRAFT) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message_status', - _data={'status': BroadcastStatusType.PENDING_APPROVAL, 'created_by': str(t.created_by_id)}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert response['status'] == BroadcastStatusType.PENDING_APPROVAL - assert response['updated_at'] is not None - - -def test_update_broadcast_message_status_doesnt_let_you_update_other_things( - admin_request, - sample_broadcast_service -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t) - - response = admin_request.post( - 'broadcast_message.update_broadcast_message_status', - _data={ - 'areas': {'ids': ['glasgow']}, - 'status': BroadcastStatusType.BROADCASTING, - 'created_by': str(t.created_by_id) - }, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=400 - ) - - assert response['errors'] == [{ - 'error': 'ValidationError', - 'message': 'Additional properties are not allowed (areas was unexpected)', - }] - - -@pytest.mark.parametrize('user_is_platform_admin', [True, False]) -def test_update_broadcast_message_allows_service_user_and_platform_admin_to_cancel( - admin_request, sample_broadcast_service, mocker, user_is_platform_admin -): - """ - Only platform admins and users belonging to that service should be able to cancel broadcasts. - """ - t = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - bm = create_broadcast_message(t, status=BroadcastStatusType.BROADCASTING) - canceller = create_user(email='canceller@gov.uk') - if user_is_platform_admin: - canceller.platform_admin = True - else: - sample_broadcast_service.users.append(canceller) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - response = admin_request.post( - 'broadcast_message.update_broadcast_message_status', - _data={'status': BroadcastStatusType.CANCELLED, 'created_by': str(canceller.id)}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=200 - ) - - assert len(bm.events) == 1 - cancel_event = bm.events[0] - - cancel_id = str(cancel_event.id) - - mock_task.assert_called_once_with(kwargs={'broadcast_event_id': cancel_id}, queue='broadcast-tasks') - assert response['status'] == BroadcastStatusType.CANCELLED - assert response['cancelled_at'] is not None - assert response['cancelled_by_id'] == str(canceller.id) - - assert cancel_event.service_id == sample_broadcast_service.id - assert cancel_event.transmitted_areas == bm.areas - assert cancel_event.message_type == BroadcastEventMessageType.CANCEL - assert cancel_event.transmitted_finishes_at == bm.finishes_at - assert cancel_event.transmitted_content == {"body": "emergency broadcast"} - - -def test_update_broadcast_message_status_aborts_if_service_is_suspended( - admin_request, - sample_broadcast_service, -): - bm = create_broadcast_message(service=sample_broadcast_service, content='test') - sample_broadcast_service.active = False - - admin_request.post( - 'broadcast_message.update_broadcast_message_status', - _data={'status': BroadcastStatusType.BROADCASTING, 'created_by': str(uuid.uuid4())}, - service_id=sample_broadcast_service.id, - broadcast_message_id=bm.id, - _expected_status=403 - ) - - -def test_update_broadcast_message_status_rejects_approval_from_user_not_on_that_service( - admin_request, - sample_broadcast_service, - mocker -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - bm = create_broadcast_message(t, status=BroadcastStatusType.PENDING_APPROVAL) - approver = create_user(email='approver@gov.uk') - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - response = admin_request.post( - 'broadcast_message.update_broadcast_message_status', - _data={'status': BroadcastStatusType.BROADCASTING, 'created_by': str(approver.id)}, - service_id=t.service_id, - broadcast_message_id=bm.id, - _expected_status=400 - ) - - assert mock_task.called is False - assert 'cannot update broadcast' in response['message'] diff --git a/tests/app/broadcast_message/test_utils.py b/tests/app/broadcast_message/test_utils.py deleted file mode 100644 index b29bbd0a2..000000000 --- a/tests/app/broadcast_message/test_utils.py +++ /dev/null @@ -1,455 +0,0 @@ -import pytest - -from app.broadcast_message.utils import ( - _create_p1_zendesk_alert, - update_broadcast_message_status, -) -from app.errors import InvalidRequest -from app.models import ( - BROADCAST_TYPE, - BroadcastEventMessageType, - BroadcastStatusType, -) -from tests.app.db import ( - create_api_key, - create_broadcast_message, - create_template, - create_user, -) -from tests.conftest import set_config - - -def test_update_broadcast_message_status_stores_approved_by_and_approved_at_and_queues_task( - sample_broadcast_service, - mocker -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.PENDING_APPROVAL, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, approver - ) - - assert broadcast_message.status == BroadcastStatusType.BROADCASTING - assert broadcast_message.approved_at is not None - assert broadcast_message.approved_by_id == approver.id - - assert len(broadcast_message.events) == 1 - alert_event = broadcast_message.events[0] - - mock_task.assert_called_once_with(kwargs={'broadcast_event_id': str(alert_event.id)}, queue='broadcast-tasks') - - assert alert_event.service_id == sample_broadcast_service.id - assert alert_event.transmitted_areas == broadcast_message.areas - assert alert_event.message_type == BroadcastEventMessageType.ALERT - assert alert_event.transmitted_finishes_at == broadcast_message.finishes_at - assert alert_event.transmitted_content == {"body": "emergency broadcast"} - - -def test_update_broadcast_message_status_for_cancelling_broadcast_from_admin_interface( - sample_broadcast_service, - mocker, -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.BROADCASTING, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - canceller = sample_broadcast_service.created_by - - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.CANCELLED, updating_user=canceller, api_key_id=None - ) - - assert broadcast_message.status == BroadcastStatusType.CANCELLED - assert broadcast_message.cancelled_at is not None - assert broadcast_message.cancelled_by_id == canceller.id - assert broadcast_message.cancelled_by_api_key_id is None - - assert len(broadcast_message.events) == 1 - alert_event = broadcast_message.events[0] - - mock_task.assert_called_once_with(kwargs={'broadcast_event_id': str(alert_event.id)}, queue='broadcast-tasks') - - assert alert_event.service_id == sample_broadcast_service.id - assert alert_event.message_type == BroadcastEventMessageType.CANCEL - - -def test_update_broadcast_message_status_for_cancelling_broadcast_from_API_call( - sample_broadcast_service, - mocker, -): - api_key = create_api_key(service=sample_broadcast_service) - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.BROADCASTING, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.CANCELLED, updating_user=None, api_key_id=api_key.id - ) - - assert broadcast_message.status == BroadcastStatusType.CANCELLED - assert broadcast_message.cancelled_at is not None - assert broadcast_message.cancelled_by_id is None - assert broadcast_message.cancelled_by_api_key_id == api_key.id - - assert len(broadcast_message.events) == 1 - alert_event = broadcast_message.events[0] - - mock_task.assert_called_once_with(kwargs={'broadcast_event_id': str(alert_event.id)}, queue='broadcast-tasks') - - assert alert_event.service_id == sample_broadcast_service.id - assert alert_event.message_type == BroadcastEventMessageType.CANCEL - - -def test_update_broadcast_message_status_for_rejecting_broadcast_via_admin_interface( - sample_broadcast_service, - mocker -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.PENDING_APPROVAL, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.REJECTED, updating_user=sample_broadcast_service.created_by - ) - - assert broadcast_message.status == BroadcastStatusType.REJECTED - assert broadcast_message.cancelled_at is None - assert broadcast_message.cancelled_by_id is None - assert broadcast_message.updated_at is not None - - assert not mock_task.called - assert len(broadcast_message.events) == 0 - - -def test_update_broadcast_message_status_for_rejecting_broadcast_from_API_call( - sample_broadcast_service, - mocker -): - api_key = create_api_key(service=sample_broadcast_service) - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.PENDING_APPROVAL, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.REJECTED, api_key_id=api_key.id - ) - - assert broadcast_message.status == BroadcastStatusType.REJECTED - assert broadcast_message.cancelled_at is None - assert broadcast_message.cancelled_by_id is None - assert broadcast_message.cancelled_by_api_key_id is None - assert broadcast_message.updated_at is not None - - assert not mock_task.called - assert len(broadcast_message.events) == 0 - - -@pytest.mark.parametrize('current_status, new_status', [ - (BroadcastStatusType.DRAFT, BroadcastStatusType.DRAFT), - (BroadcastStatusType.DRAFT, BroadcastStatusType.BROADCASTING), - (BroadcastStatusType.DRAFT, BroadcastStatusType.CANCELLED), - - (BroadcastStatusType.PENDING_APPROVAL, BroadcastStatusType.PENDING_APPROVAL), - (BroadcastStatusType.PENDING_APPROVAL, BroadcastStatusType.CANCELLED), - (BroadcastStatusType.PENDING_APPROVAL, BroadcastStatusType.COMPLETED), - - (BroadcastStatusType.REJECTED, BroadcastStatusType.REJECTED), - (BroadcastStatusType.REJECTED, BroadcastStatusType.BROADCASTING), - (BroadcastStatusType.REJECTED, BroadcastStatusType.CANCELLED), - (BroadcastStatusType.REJECTED, BroadcastStatusType.COMPLETED), - - (BroadcastStatusType.BROADCASTING, BroadcastStatusType.DRAFT), - (BroadcastStatusType.BROADCASTING, BroadcastStatusType.PENDING_APPROVAL), - (BroadcastStatusType.BROADCASTING, BroadcastStatusType.BROADCASTING), - - (BroadcastStatusType.COMPLETED, BroadcastStatusType.DRAFT), - (BroadcastStatusType.COMPLETED, BroadcastStatusType.PENDING_APPROVAL), - (BroadcastStatusType.COMPLETED, BroadcastStatusType.BROADCASTING), - (BroadcastStatusType.COMPLETED, BroadcastStatusType.CANCELLED), - - (BroadcastStatusType.CANCELLED, BroadcastStatusType.DRAFT), - (BroadcastStatusType.CANCELLED, BroadcastStatusType.PENDING_APPROVAL), - (BroadcastStatusType.CANCELLED, BroadcastStatusType.BROADCASTING), - (BroadcastStatusType.CANCELLED, BroadcastStatusType.COMPLETED), -]) -def test_update_broadcast_message_status_restricts_status_transitions_to_explicit_list( - sample_broadcast_service, - mocker, - current_status, - new_status -): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(t, status=current_status) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - with pytest.raises(expected_exception=InvalidRequest) as e: - update_broadcast_message_status(broadcast_message, new_status, approver) - - assert mock_task.called is False - assert f'from {current_status} to {new_status}' in str(e.value) - - -@pytest.mark.parametrize('is_platform_admin', [True, False]) -def test_update_broadcast_message_status_rejects_approval_from_creator( - sample_broadcast_service, - mocker, - is_platform_admin -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.PENDING_APPROVAL) - creator_and_approver = sample_broadcast_service.created_by - creator_and_approver.platform_admin = is_platform_admin - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - with pytest.raises(expected_exception=InvalidRequest) as e: - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, creator_and_approver - ) - - assert mock_task.called is False - assert 'cannot approve their own broadcast' in str(e.value) - - -def test_update_broadcast_message_status_rejects_approval_of_broadcast_with_no_areas( - admin_request, - sample_broadcast_service, - mocker -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast = create_broadcast_message(template, status=BroadcastStatusType.PENDING_APPROVAL) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - with pytest.raises(expected_exception=InvalidRequest) as e: - update_broadcast_message_status(broadcast, BroadcastStatusType.BROADCASTING, approver) - - assert mock_task.called is False - assert f'broadcast_message {broadcast.id} has no selected areas and so cannot be broadcasted.' in str(e.value) - - -def test_update_broadcast_message_status_allows_trial_mode_services_to_approve_own_message( - sample_broadcast_service, - mocker -): - sample_broadcast_service.restricted = True - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.PENDING_APPROVAL, - areas={"ids": ["london"], "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]]} - ) - creator_and_approver = sample_broadcast_service.created_by - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, creator_and_approver - ) - - assert broadcast_message.status == BroadcastStatusType.BROADCASTING - assert broadcast_message.approved_at is not None - assert broadcast_message.created_by_id == template.created_by_id - assert broadcast_message.approved_by_id == template.created_by_id - assert not mock_task.called - - -@pytest.mark.parametrize('broadcast_message_stubbed, service_restricted_before_approval', [ - (True, True), - (True, False), - (False, True), -]) -def test_update_broadcast_message_status_when_broadcast_message_is_stubbed_or_service_not_live( - admin_request, - sample_broadcast_service, - mocker, - broadcast_message_stubbed, - service_restricted_before_approval, -): - sample_broadcast_service.restricted = broadcast_message_stubbed - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='emergency broadcast') - broadcast_message = create_broadcast_message( - template, - status=BroadcastStatusType.PENDING_APPROVAL, - areas={"ids": ["london"], "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]]}, - stubbed=broadcast_message_stubbed - ) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - sample_broadcast_service.restricted = service_restricted_before_approval - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, approver - ) - assert broadcast_message.status == BroadcastStatusType.BROADCASTING - assert broadcast_message.approved_at is not None - assert broadcast_message.approved_by_id == approver.id - - # The broadcast can be approved, but does not create a broadcast_event in the database or put a task on the queue - assert len(broadcast_message.events) == 0 - assert len(mock_task.mock_calls) == 0 - - -def test_update_broadcast_message_status_creates_event_with_correct_content_if_broadcast_has_no_template( - admin_request, - sample_broadcast_service, - mocker -): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - template=None, - content='tailor made emergency broadcast content', - status=BroadcastStatusType.PENDING_APPROVAL, - areas={ - "ids": ["london"], - "simple_polygons": [[[51.30, 0.7], [51.28, 0.8], [51.25, -0.7]]] - } - ) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - mock_task = mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, approver - ) - - assert broadcast_message.status == BroadcastStatusType.BROADCASTING - - assert len(broadcast_message.events) == 1 - alert_event = broadcast_message.events[0] - - mock_task.assert_called_once_with(kwargs={'broadcast_event_id': str(alert_event.id)}, queue='broadcast-tasks') - - assert alert_event.transmitted_content == {"body": "tailor made emergency broadcast content"} - - -def test_update_broadcast_message_status_creates_zendesk_ticket( - mocker, - notify_api, - sample_broadcast_service -): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - content='tailor made emergency broadcast content', - status=BroadcastStatusType.PENDING_APPROVAL, - areas={"names": ["England", "Scotland"], "simple_polygons": ['polygons']} - ) - approver = create_user(email='approver@gov.uk') - sample_broadcast_service.users.append(approver) - - mocker.patch('app.celery.broadcast_message_tasks.send_broadcast_event.apply_async') - mock_send_ticket_to_zendesk = mocker.patch( - 'app.broadcast_message.utils.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'live'): - update_broadcast_message_status( - broadcast_message, BroadcastStatusType.BROADCASTING, approver - ) - - mock_send_ticket_to_zendesk.assert_called_once() - - -def test_create_p1_zendesk_alert(sample_broadcast_service, mocker, notify_api): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - content='tailor made emergency broadcast content', - status=BroadcastStatusType.BROADCASTING, - areas={"names": ["England", "Scotland"]} - ) - - mock_send_ticket_to_zendesk = mocker.patch( - 'app.broadcast_message.utils.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'live'): - _create_p1_zendesk_alert(broadcast_message) - - ticket = mock_send_ticket_to_zendesk.call_args_list[0].args[0] - assert ticket.subject == 'Live broadcast sent' - assert ticket.ticket_type == 'incident' - assert str(broadcast_message.id) in ticket.message - assert "Sent on channel severe to ['England', 'Scotland']" in ticket.message - assert 'Content starts "tailor made emergency' in ticket.message - - -def test_create_p1_zendesk_alert_doesnt_alert_when_cancelling(mocker, notify_api, sample_broadcast_service): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - content='tailor made emergency broadcast content', - status=BroadcastStatusType.CANCELLED, - areas={"names": ["England", "Scotland"]} - ) - - mock_send_ticket_to_zendesk = mocker.patch( - 'app.broadcast_message.utils.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'live'): - _create_p1_zendesk_alert(broadcast_message) - - mock_send_ticket_to_zendesk.assert_not_called() - - -def test_create_p1_zendesk_alert_doesnt_alert_on_staging(mocker, notify_api, sample_broadcast_service): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - content='tailor made emergency broadcast content', - status=BroadcastStatusType.BROADCASTING, - areas={"names": ["England", "Scotland"]} - ) - - mock_send_ticket_to_zendesk = mocker.patch( - 'app.broadcast_message.utils.zendesk_client.send_ticket_to_zendesk', - autospec=True, - ) - - with set_config(notify_api, 'NOTIFY_ENVIRONMENT', 'staging'): - _create_p1_zendesk_alert(broadcast_message) - - mock_send_ticket_to_zendesk.assert_not_called() diff --git a/tests/app/celery/test_broadcast_message_tasks.py b/tests/app/celery/test_broadcast_message_tasks.py deleted file mode 100644 index 7efa6190f..000000000 --- a/tests/app/celery/test_broadcast_message_tasks.py +++ /dev/null @@ -1,726 +0,0 @@ -from datetime import datetime -from unittest.mock import ANY, Mock, call - -import pytest -from celery.exceptions import Retry -from freezegun import freeze_time - -from app.celery.broadcast_message_tasks import ( - BroadcastIntegrityError, - check_event_makes_sense_in_sequence, - get_retry_delay, - send_broadcast_event, - send_broadcast_provider_message, - trigger_link_test, -) -from app.clients.cbc_proxy import CBCProxyRetryableException -from app.config import QueueNames, TaskNames -from app.models import ( - BROADCAST_TYPE, - BroadcastEventMessageType, - BroadcastProviderMessageStatus, - BroadcastStatusType, -) -from tests.app.db import ( - create_broadcast_event, - create_broadcast_message, - create_broadcast_provider_message, - create_template, -) -from tests.conftest import set_config - - -def test_send_broadcast_event_queues_up_for_active_providers(mocker, notify_api, sample_broadcast_service): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) - event = create_broadcast_event(broadcast_message) - - mocker.patch('app.celery.broadcast_message_tasks.notify_celery.send_task') - - mock_send_broadcast_provider_message = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', - ) - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']): - send_broadcast_event(event.id) - - assert mock_send_broadcast_provider_message.apply_async.call_args_list == [ - call(kwargs={'broadcast_event_id': event.id, 'provider': 'ee'}, queue='broadcast-tasks'), - call(kwargs={'broadcast_event_id': event.id, 'provider': 'vodafone'}, queue='broadcast-tasks') - ] - - -@pytest.mark.parametrize('message_status', [ - BroadcastStatusType.BROADCASTING, - BroadcastStatusType.CANCELLED, -]) -def test_send_broadcast_event_calls_publish_govuk_alerts_task( - mocker, notify_api, sample_broadcast_service, message_status -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template, status=message_status) - event = create_broadcast_event(broadcast_message) - mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', - ) - - mock_celery = mocker.patch('app.celery.broadcast_message_tasks.notify_celery.send_task') - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']): - send_broadcast_event(event.id) - - mock_celery.assert_called_once_with( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) - - -def test_send_broadcast_event_only_sends_to_one_provider_if_set_on_service( - mocker, - notify_api, - sample_broadcast_service -): - sample_broadcast_service.allowed_broadcast_provider = "vodafone" - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) - event = create_broadcast_event(broadcast_message) - - mock_send_broadcast_provider_message = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', - ) - mocker.patch('app.celery.broadcast_message_tasks.notify_celery.send_task') - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']): - send_broadcast_event(event.id) - - assert mock_send_broadcast_provider_message.apply_async.call_args_list == [ - call(kwargs={'broadcast_event_id': event.id, 'provider': 'vodafone'}, queue='broadcast-tasks') - ] - - -def test_send_broadcast_event_does_nothing_if_provider_set_on_service_isnt_enabled_globally( - mocker, - notify_api, - sample_broadcast_service -): - sample_broadcast_service.allowed_broadcast_provider = "three" - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) - event = create_broadcast_event(broadcast_message) - - mocker.patch('app.celery.broadcast_message_tasks.notify_celery.send_task') - - mock_send_broadcast_provider_message = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message', - ) - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']): - send_broadcast_event(event.id) - - assert mock_send_broadcast_provider_message.apply_async.called is False - - -@freeze_time('2020-08-01 12:00') -@pytest.mark.parametrize('provider,provider_capitalised', [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_send_broadcast_provider_message_sends_data_correctly( - mocker, sample_broadcast_service, provider, provider_capitalised -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message( - template, - areas={ - 'areas': ['london', 'glasgow'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - [[-4.53, 55.72], [-3.88, 55.72], [-3.88, 55.96], [-4.53, 55.96]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - event = create_broadcast_event(broadcast_message) - - mock_create_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.create_and_send_broadcast', - ) - - assert event.get_provider_message(provider) is None - - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(event.id)) - - broadcast_provider_message = event.get_provider_message(provider) - assert broadcast_provider_message.status == BroadcastProviderMessageStatus.ACK - - mock_create_broadcast.assert_called_once_with( - identifier=str(broadcast_provider_message.id), - message_number=mocker.ANY, - headline='GOV.UK Notify Broadcast', - description='this is an emergency broadcast message', - areas=[{ - 'polygon': [ - [50.12, 1.2], [50.13, 1.2], [50.14, 1.21], - ], - }, { - 'polygon': [ - [-4.53, 55.72], [-3.88, 55.72], [-3.88, 55.96], [-4.53, 55.96], - ], - }], - sent=event.sent_at_as_cap_datetime_string, - expires=event.transmitted_finishes_at_as_cap_datetime_string, - channel="severe", - ) - - -@freeze_time('2020-08-01 12:00') -@pytest.mark.parametrize('provider,provider_capitalised', [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -@pytest.mark.parametrize('channel', ['operator', 'test', 'severe', 'government']) -def test_send_broadcast_provider_message_uses_channel_set_on_broadcast_service( - mocker, sample_broadcast_service, provider, provider_capitalised, channel -): - sample_broadcast_service.broadcast_channel = channel - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message( - template, - areas={ - 'areas': ['london', 'glasgow'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - [[-4.53, 55.72], [-3.88, 55.72], [-3.88, 55.96], [-4.53, 55.96]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - event = create_broadcast_event(broadcast_message) - - mock_create_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.create_and_send_broadcast', - ) - - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(event.id)) - - mock_create_broadcast.assert_called_once_with( - identifier=mocker.ANY, - message_number=mocker.ANY, - headline='GOV.UK Notify Broadcast', - description='this is an emergency broadcast message', - areas=mocker.ANY, - sent=mocker.ANY, - expires=mocker.ANY, - channel=channel, - ) - - -def test_send_broadcast_provider_message_works_if_we_retried_previously(mocker, sample_broadcast_service): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message( - template, - areas={'areas': [], 'simple_polygons': [], }, - status=BroadcastStatusType.BROADCASTING - ) - event = create_broadcast_event(broadcast_message) - - # an existing provider message already exists, and previously failed - create_broadcast_provider_message( - broadcast_event=event, - provider='ee', - status=BroadcastProviderMessageStatus.SENDING - ) - - mock_create_broadcast = mocker.patch( - 'app.clients.cbc_proxy.CBCProxyEE.create_and_send_broadcast', - ) - - send_broadcast_provider_message(provider='ee', broadcast_event_id=str(event.id)) - - # make sure we haven't completed a duplicate event - we shouldn't record the failure - assert len(event.provider_messages) == 1 - - broadcast_provider_message = event.get_provider_message('ee') - - assert broadcast_provider_message.status == BroadcastProviderMessageStatus.ACK - assert broadcast_provider_message.updated_at is not None - - mock_create_broadcast.assert_called_once_with( - identifier=str(broadcast_provider_message.id), - message_number=mocker.ANY, - headline='GOV.UK Notify Broadcast', - description='this is an emergency broadcast message', - areas=[], - sent=event.sent_at_as_cap_datetime_string, - expires=event.transmitted_finishes_at_as_cap_datetime_string, - channel='severe', - ) - - -@freeze_time('2020-08-01 12:00') -@pytest.mark.parametrize('provider,provider_capitalised', [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_send_broadcast_provider_message_sends_data_correctly_when_broadcast_message_has_no_template( - mocker, sample_broadcast_service, provider, provider_capitalised -): - broadcast_message = create_broadcast_message( - service=sample_broadcast_service, - template=None, - content='this is an emergency broadcast message', - areas={ - 'areas': ['london', 'glasgow'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - [[-4.53, 55.72], [-3.88, 55.72], [-3.88, 55.96], [-4.53, 55.96]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - event = create_broadcast_event(broadcast_message) - - mock_create_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.create_and_send_broadcast', - ) - - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(event.id)) - - broadcast_provider_message = event.get_provider_message(provider) - - mock_create_broadcast.assert_called_once_with( - identifier=str(broadcast_provider_message.id), - message_number=mocker.ANY, - headline='GOV.UK Notify Broadcast', - description='this is an emergency broadcast message', - areas=mocker.ANY, - sent=mocker.ANY, - expires=mocker.ANY, - channel="severe" - ) - - -@pytest.mark.parametrize('provider,provider_capitalised', [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_send_broadcast_provider_message_sends_update_with_references( - mocker, sample_broadcast_service, provider, provider_capitalised -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='content') - - broadcast_message = create_broadcast_message( - template, - areas={ - 'areas': ['london'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - - alert_event = create_broadcast_event(broadcast_message, message_type=BroadcastEventMessageType.ALERT) - create_broadcast_provider_message(alert_event, provider, status=BroadcastProviderMessageStatus.ACK) - update_event = create_broadcast_event(broadcast_message, message_type=BroadcastEventMessageType.UPDATE) - - mock_update_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.update_and_send_broadcast', - ) - - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(update_event.id)) - - broadcast_provider_message = update_event.get_provider_message(provider) - assert broadcast_provider_message.status == BroadcastProviderMessageStatus.ACK - - mock_update_broadcast.assert_called_once_with( - identifier=str(broadcast_provider_message.id), - message_number=mocker.ANY, - headline="GOV.UK Notify Broadcast", - description='this is an emergency broadcast message', - areas=[{ - "polygon": [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - }], - previous_provider_messages=[ - alert_event.get_provider_message(provider) - ], - sent=update_event.sent_at_as_cap_datetime_string, - expires=update_event.transmitted_finishes_at_as_cap_datetime_string, - channel="severe" - ) - - -@pytest.mark.parametrize('provider,provider_capitalised', [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_send_broadcast_provider_message_sends_cancel_with_references( - mocker, sample_broadcast_service, provider, provider_capitalised -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE, content='content') - - broadcast_message = create_broadcast_message( - template, - areas={ - 'areas': ['london'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - - alert_event = create_broadcast_event(broadcast_message, message_type=BroadcastEventMessageType.ALERT) - update_event = create_broadcast_event(broadcast_message, message_type=BroadcastEventMessageType.UPDATE) - cancel_event = create_broadcast_event(broadcast_message, message_type=BroadcastEventMessageType.CANCEL) - - create_broadcast_provider_message(alert_event, provider, status=BroadcastProviderMessageStatus.ACK) - create_broadcast_provider_message(update_event, provider, status=BroadcastProviderMessageStatus.ACK) - - mock_cancel_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.cancel_broadcast', - ) - - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(cancel_event.id)) - - broadcast_provider_message = cancel_event.get_provider_message(provider) - assert broadcast_provider_message.status == BroadcastProviderMessageStatus.ACK - - mock_cancel_broadcast.assert_called_once_with( - identifier=str(broadcast_provider_message.id), - message_number=mocker.ANY, - previous_provider_messages=[ - alert_event.get_provider_message(provider), - update_event.get_provider_message(provider) - ], - sent=cancel_event.sent_at_as_cap_datetime_string, - ) - - -@pytest.mark.parametrize("provider,provider_capitalised", [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_send_broadcast_provider_message_errors(mocker, sample_broadcast_service, provider, provider_capitalised): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - - broadcast_message = create_broadcast_message( - template, - areas={ - 'areas': ['london'], - 'simple_polygons': [ - [[50.12, 1.2], [50.13, 1.2], [50.14, 1.21]], - ], - }, - status=BroadcastStatusType.BROADCASTING - ) - - event = create_broadcast_event(broadcast_message) - - mock_create_broadcast = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.create_and_send_broadcast', - side_effect=CBCProxyRetryableException('oh no'), - ) - mock_retry = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message.retry', - side_effect=Retry - ) - - with pytest.raises(Retry): - send_broadcast_provider_message(provider=provider, broadcast_event_id=str(event.id)) - - mock_create_broadcast.assert_called_once_with( - identifier=ANY, - message_number=mocker.ANY, - headline="GOV.UK Notify Broadcast", - description='this is an emergency broadcast message', - areas=[{ - 'polygon': [ - [50.12, 1.2], - [50.13, 1.2], - [50.14, 1.21], - ], - }], - sent=event.sent_at_as_cap_datetime_string, - expires=event.transmitted_finishes_at_as_cap_datetime_string, - channel="severe" - ) - mock_retry.assert_called_once_with( - countdown=1, - exc=mock_create_broadcast.side_effect, - queue='broadcast-tasks' - ) - broadcast_provider_message = event.get_provider_message(provider) - assert broadcast_provider_message.status == BroadcastProviderMessageStatus.SENDING - - -@pytest.mark.parametrize('num_retries, expected_countdown', [ - (0, 1), - (5, 32), - (20, 240), -]) -def test_send_broadcast_provider_message_delays_retry_exponentially( - mocker, - sample_broadcast_service, - num_retries, - expected_countdown -): - template = create_template(sample_broadcast_service, BROADCAST_TYPE) - - broadcast_message = create_broadcast_message(template, status=BroadcastStatusType.BROADCASTING) - event = create_broadcast_event(broadcast_message) - - mock_create_broadcast = mocker.patch( - 'app.clients.cbc_proxy.CBCProxyEE.create_and_send_broadcast', - side_effect=CBCProxyRetryableException('oh no'), - ) - mock_retry = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_provider_message.retry', - side_effect=Retry - ) - - # patch celery request context as shown here: https://stackoverflow.com/a/59870468 - mock_celery_task_request_context = mocker.patch("celery.app.task.Task.request") - mock_celery_task_request_context.retries = num_retries - - with pytest.raises(Retry): - send_broadcast_provider_message(provider='ee', broadcast_event_id=str(event.id)) - - mock_create_broadcast.assert_called_once_with( - identifier=ANY, - message_number=mocker.ANY, - headline="GOV.UK Notify Broadcast", - description='this is an emergency broadcast message', - areas=[], - sent=event.sent_at_as_cap_datetime_string, - expires=event.transmitted_finishes_at_as_cap_datetime_string, - channel='severe', - ) - mock_retry.assert_called_once_with( - countdown=expected_countdown, - exc=mock_create_broadcast.side_effect, - queue='broadcast-tasks' - ) - - -@pytest.mark.parametrize("provider,provider_capitalised", [ - ['ee', 'EE'], - ['three', 'Three'], - ['o2', 'O2'], - ['vodafone', 'Vodafone'], -]) -def test_trigger_link_tests_invokes_cbc_proxy_client( - mocker, provider, provider_capitalised, client, -): - mock_send_link_test = mocker.patch( - f'app.clients.cbc_proxy.CBCProxy{provider_capitalised}.send_link_test', - ) - - trigger_link_test(provider) - assert mock_send_link_test.called_once() - - -@pytest.mark.parametrize('retry_count, expected_delay', [ - (0, 1), - (1, 2), - (2, 4), - (7, 128), - (8, 240), - (9, 240), - (1000, 240), -]) -def test_get_retry_delay_has_capped_backoff(retry_count, expected_delay): - assert get_retry_delay(retry_count) == expected_delay - - -@freeze_time('2021-01-01 12:00') -def test_check_event_makes_sense_in_sequence_doesnt_raise_if_event_hasnt_expired_yet(sample_template): - broadcast_message = create_broadcast_message(sample_template) - current_event = create_broadcast_event( - broadcast_message, - transmitted_starts_at=datetime(2021, 1, 1, 0, 0), - transmitted_finishes_at=datetime(2021, 1, 1, 12, 1), - ) - check_event_makes_sense_in_sequence(current_event, 'ee') - - -@freeze_time('2021-01-01 12:00') -def test_send_broadcast_provider_message_raises_if_event_has_expired(sample_template): - broadcast_message = create_broadcast_message(sample_template) - current_event = create_broadcast_event( - broadcast_message, - transmitted_starts_at=datetime(2021, 1, 1, 0, 0), - transmitted_finishes_at=datetime(2021, 1, 1, 11, 59), - ) - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - assert 'The expiry time of 2021-01-01 11:59:00 has already passed' in str(exc.value) - - -@freeze_time('2021-01-01 12:00') -def test_send_broadcast_provider_message_raises_if_older_event_still_sending(sample_template): - broadcast_message = create_broadcast_message(sample_template) - # event approved at midnight - past_succesful_event = create_broadcast_event( - broadcast_message, - message_type='alert', - sent_at=datetime(2021, 1, 1, 0, 0), - ) - # event updated at 5am (this event is still sending) - past_still_sending_event = create_broadcast_event( - broadcast_message, - message_type='update', - sent_at=datetime(2021, 1, 1, 5, 0), - ) - # event updated again at 7am - current_event = create_broadcast_event( - broadcast_message, - message_type='update', - sent_at=datetime(2021, 1, 1, 7, 0), - ) - - create_broadcast_provider_message(past_succesful_event, provider='ee', status=BroadcastProviderMessageStatus.ACK) - create_broadcast_provider_message(past_still_sending_event, provider='ee', status=BroadcastProviderMessageStatus.SENDING) # noqa - - # we havent sent the previous update yet - it's still in sending - so don't try and send this one. - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert f'Previous event {past_still_sending_event.id} (type update) has not finished sending to provider ee' in str(exc.value) # noqa - - -@freeze_time('2021-01-01 12:00') -def test_send_broadcast_provider_message_raises_if_older_event_hasnt_started_sending_yet(sample_template): - broadcast_message = create_broadcast_message(sample_template) - # event approved at midnight - past_succesful_event = create_broadcast_event( - broadcast_message, - message_type='alert', - sent_at=datetime(2021, 1, 1, 0, 0), - ) - # event updated at 5am - past_still_sending_event = create_broadcast_event( - broadcast_message, - message_type='update', - sent_at=datetime(2021, 1, 1, 5, 0), - ) - # event updated at 7am - current_event = create_broadcast_event( - broadcast_message, - message_type='update', - sent_at=datetime(2021, 1, 1, 7, 0), - ) - - # no provider message for past_still_sending_event - create_broadcast_provider_message(past_succesful_event, provider='ee', status=BroadcastProviderMessageStatus.ACK) - - # we shouldn't send the update now, because a previous event is still stuck in sending - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert f'Previous event {past_still_sending_event.id} (type update) has no provider_message for provider ee' in str(exc.value) # noqa - - -@freeze_time('2021-01-01 12:00') -def test_check_event_makes_sense_in_sequence_doesnt_raise_if_newer_event_not_acked_yet(sample_template): - broadcast_message = create_broadcast_message(sample_template) - # event approved at midnight - current_event = create_broadcast_event( - broadcast_message, - message_type='alert', - sent_at=datetime(2021, 1, 1, 0, 0), - ) - # create a future event - create_broadcast_event( - broadcast_message, - message_type='cancel', - sent_at=datetime(2021, 1, 1, 10, 0), - ) - - # this doesn't raise, because the alert event got an ack. The cancel doesn't have an event yet - # but this task is only interested in the current task (the update) so doesn't worry about that - check_event_makes_sense_in_sequence(current_event, 'ee') - - -@pytest.mark.parametrize('existing_message_status', [ - BroadcastProviderMessageStatus.ACK, - BroadcastProviderMessageStatus.ERR, - BroadcastProviderMessageStatus.TECHNICAL_FAILURE, -]) -def test_send_broadcast_provider_message_raises_if_current_event_already_has_provider_message_not_in_sending( - sample_template, - existing_message_status -): - broadcast_message = create_broadcast_message(sample_template) - current_event = create_broadcast_event(broadcast_message, message_type='alert') - create_broadcast_provider_message(current_event, provider='ee', status=existing_message_status) - - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert f'in status {existing_message_status}' in str(exc.value) - - -def test_send_broadcast_provider_message_raises_if_service_is_suspended( - sample_broadcast_service, -): - sample_broadcast_service.active = False - broadcast_message = create_broadcast_message(service=sample_broadcast_service, content='test') - current_event = create_broadcast_event(broadcast_message, message_type='alert') - - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert 'service is suspended' in str(exc.value) - - -def test_send_broadcast_provider_message_raises_if_service_is_not_live( - sample_broadcast_service, -): - sample_broadcast_service.restricted = True - broadcast_message = create_broadcast_message(service=sample_broadcast_service, content='test') - current_event = create_broadcast_event(broadcast_message, message_type='alert') - - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert 'service is not live' in str(exc.value) - - -def test_send_broadcast_provider_message_raises_if_message_is_stubbed( - sample_template, -): - broadcast_message = create_broadcast_message(sample_template, stubbed=True) - current_event = create_broadcast_event(broadcast_message, message_type='alert') - - with pytest.raises(BroadcastIntegrityError) as exc: - send_broadcast_provider_message(current_event.id, 'ee') - - assert 'message is stubbed' in str(exc.value) - - -def test_send_broadcast_provider_message_does_nothing_if_cbc_proxy_disabled(mocker, notify_api, sample_template): - mock_proxy_client_getter = mocker.patch( - 'app.celery.broadcast_message_tasks.cbc_proxy_client', - ) - mock_client = Mock() - mock_proxy_client_getter.get_proxy.return_value = mock_client - - broadcast_message = create_broadcast_message(sample_template) - broadcast_event = create_broadcast_event(broadcast_message, message_type='alert') - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']), set_config(notify_api, 'CBC_PROXY_ENABLED', False): - send_broadcast_provider_message(broadcast_event.id, 'ee') - - assert mock_client.create_and_send_broadcast.called is False diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 13e831176..32b1adcbb 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -11,7 +11,6 @@ from notifications_utils.clients.zendesk.zendesk_client import ( from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( - auto_expire_broadcast_messages, check_for_missing_rows_in_completed_jobs, check_for_services_with_high_failure_rates_or_sending_to_tv_numbers, check_if_letters_still_in_created, @@ -19,11 +18,9 @@ from app.celery.scheduled_tasks import ( check_job_status, delete_invitations, delete_verify_codes, - remove_yesterdays_planned_tests_on_govuk_alerts, replay_created_notifications, run_scheduled_jobs, switch_current_sms_provider_on_slow_delivery, - trigger_link_tests, ) from app.config import QueueNames, TaskNames, Test from app.dao.jobs_dao import dao_get_job_by_id @@ -35,11 +32,9 @@ from app.models import ( JOB_STATUS_PENDING, NOTIFICATION_DELIVERED, NOTIFICATION_PENDING_VIRUS_CHECK, - BroadcastStatusType, ) from tests.app import load_example_csv from tests.app.db import ( - create_broadcast_message, create_job, create_notification, create_template, @@ -677,80 +672,3 @@ def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers( technical_ticket=True ) mock_send_ticket_to_zendesk.assert_called_once() - - -def test_trigger_link_tests_calls_for_all_providers( - mocker, notify_api -): - mock_trigger_link_test = mocker.patch( - 'app.celery.scheduled_tasks.trigger_link_test', - ) - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']): - trigger_link_tests() - - assert mock_trigger_link_test.apply_async.call_args_list == [ - call(kwargs={'provider': 'ee'}, queue='broadcast-tasks'), - call(kwargs={'provider': 'vodafone'}, queue='broadcast-tasks') - ] - - -def test_trigger_link_does_nothing_if_cbc_proxy_disabled( - mocker, notify_api -): - mock_trigger_link_test = mocker.patch( - 'app.celery.scheduled_tasks.trigger_link_test', - ) - - with set_config(notify_api, 'ENABLED_CBCS', ['ee', 'vodafone']), set_config(notify_api, 'CBC_PROXY_ENABLED', False): - trigger_link_tests() - - assert mock_trigger_link_test.called is False - - -@freeze_time('2021-07-19 15:50') -@pytest.mark.parametrize('status, finishes_at, final_status, should_call_publish_task', [ - (BroadcastStatusType.BROADCASTING, '2021-07-19 16:00', BroadcastStatusType.BROADCASTING, False), - (BroadcastStatusType.BROADCASTING, '2021-07-19 15:40', BroadcastStatusType.COMPLETED, True), - (BroadcastStatusType.BROADCASTING, None, BroadcastStatusType.BROADCASTING, False), - (BroadcastStatusType.PENDING_APPROVAL, None, BroadcastStatusType.PENDING_APPROVAL, False), - (BroadcastStatusType.CANCELLED, '2021-07-19 15:40', BroadcastStatusType.CANCELLED, False), -]) -def test_auto_expire_broadcast_messages( - mocker, - status, - finishes_at, - final_status, - sample_template, - should_call_publish_task, -): - message = create_broadcast_message( - status=status, - finishes_at=finishes_at, - template=sample_template, - ) - mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task') - - auto_expire_broadcast_messages() - assert message.status == final_status - - if should_call_publish_task: - mock_celery.assert_called_once_with( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) - else: - assert not mock_celery.called - - -def test_remove_yesterdays_planned_tests_on_govuk_alerts( - mocker -): - mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task') - - remove_yesterdays_planned_tests_on_govuk_alerts() - - mock_celery.assert_called_once_with( - name=TaskNames.PUBLISH_GOVUK_ALERTS, - queue=QueueNames.GOVUK_ALERTS - ) diff --git a/tests/app/clients/test_cbc_proxy.py b/tests/app/clients/test_cbc_proxy.py deleted file mode 100644 index 8a0659958..000000000 --- a/tests/app/clients/test_cbc_proxy.py +++ /dev/null @@ -1,637 +0,0 @@ -import json -import uuid -from collections import namedtuple -from datetime import datetime -from io import BytesIO -from unittest.mock import Mock, call - -import pytest -from botocore.exceptions import ClientError as BotoClientError - -from app import db -from app.clients.cbc_proxy import ( - CBCProxyClient, - CBCProxyEE, - CBCProxyO2, - CBCProxyRetryableException, - CBCProxyThree, - CBCProxyVodafone, -) -from app.utils import DATETIME_FORMAT - -EXAMPLE_AREAS = [{ - 'description': 'london', - 'polygon': [ - [51.12, -1.2], - [51.12, 1.2], - [51.74, 1.2], - [51.74, -1.2], - [51.12, -1.2], - ], -}] - - -@pytest.fixture(scope='function') -def cbc_proxy_client(client, mocker): - client = CBCProxyClient() - current_app = mocker.Mock(config={ - 'CBC_PROXY_AWS_ACCESS_KEY_ID': 'cbc-proxy-aws-access-key-id', - 'CBC_PROXY_AWS_SECRET_ACCESS_KEY': 'cbc-proxy-aws-secret-access-key', - 'CBC_PROXY_ENABLED': True, - }) - client.init_app(current_app) - return client - - -@pytest.fixture -def cbc_proxy_ee(cbc_proxy_client): - return cbc_proxy_client.get_proxy('ee') - - -@pytest.fixture -def cbc_proxy_vodafone(cbc_proxy_client): - return cbc_proxy_client.get_proxy('vodafone') - - -@pytest.mark.parametrize('provider_name, expected_provider_class', [ - ('ee', CBCProxyEE), - ('three', CBCProxyThree), - ('o2', CBCProxyO2), - ('vodafone', CBCProxyVodafone), -]) -def test_cbc_proxy_client_returns_correct_client(provider_name, expected_provider_class): - mock_lambda = Mock() - cbc_proxy_client = CBCProxyClient() - cbc_proxy_client._lambda_client = mock_lambda - - ret = cbc_proxy_client.get_proxy(provider_name) - - assert type(ret) == expected_provider_class - assert ret._lambda_client == mock_lambda - - -def test_cbc_proxy_lambda_client_has_correct_region(cbc_proxy_ee): - assert cbc_proxy_ee._lambda_client._client_config.region_name == 'us-west-2' - - -def test_cbc_proxy_lambda_client_has_correct_keys(cbc_proxy_ee): - key = cbc_proxy_ee._lambda_client._request_signer._credentials.access_key - secret = cbc_proxy_ee._lambda_client._request_signer._credentials.secret_key - - assert key == 'cbc-proxy-aws-access-key-id' - assert secret == 'cbc-proxy-aws-secret-access-key' - - -def test_cbc_proxy_send_link_test(mocker, cbc_proxy_ee): - mock_send_link_test = mocker.patch.object(cbc_proxy_ee, '_send_link_test') - cbc_proxy_ee.send_link_test() - - mock_send_link_test.assert_any_call(cbc_proxy_ee.lambda_name) - mock_send_link_test.assert_any_call(cbc_proxy_ee.failover_lambda_name) - - -@pytest.mark.parametrize('description, expected_language', ( - ('my-description', 'en-GB'), - ('mΕ·-description', 'cy-GB'), -)) -@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2']) -def test_cbc_proxy_one_2_many_create_and_send_invokes_function( - mocker, - cbc_proxy_client, - description, - cbc, - expected_language, -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - identifier = 'my-identifier' - headline = 'my-headline' - - sent = 'a-passed-through-sent-value' - expires = 'a-passed-through-expires-value' - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy.create_and_send_broadcast( - identifier=identifier, - message_number='0000007b', - headline=headline, - description=description, - areas=EXAMPLE_AREAS, - sent=sent, - expires=expires, - channel="severe", - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == identifier - assert 'message_number' not in payload - assert payload['message_format'] == 'cap' - assert payload['message_type'] == 'alert' - assert payload['headline'] == headline - assert payload['description'] == description - assert payload['areas'] == EXAMPLE_AREAS - assert payload['sent'] == sent - assert payload['expires'] == expires - assert payload['language'] == expected_language - assert payload['channel'] == 'severe' - - -@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2']) -def test_cbc_proxy_one_2_many_cancel_invokes_function(mocker, cbc_proxy_client, cbc): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - identifier = 'my-identifier' - MockProviderMessage = namedtuple( - 'BroadcastProviderMessage', ['id', 'message_number', 'created_at'] - ) - - provider_messages = [ - MockProviderMessage(uuid.uuid4(), '0000007b', datetime(2020, 12, 16)), - MockProviderMessage(uuid.uuid4(), '0000004e', datetime(2020, 12, 17)) - ] - sent = '2020-12-17 14:19:44.130585' - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy.cancel_broadcast( - identifier=identifier, - message_number='00000050', - previous_provider_messages=provider_messages, - sent=sent - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == identifier - assert 'message_number' not in payload - assert payload['message_format'] == 'cap' - assert payload['message_type'] == 'cancel' - assert payload['references'] == [ - { - "message_id": str(provider_messages[0].id), - "sent": provider_messages[0].created_at.strftime(DATETIME_FORMAT) - }, - { - "message_id": str(provider_messages[1].id), - "sent": provider_messages[1].created_at.strftime(DATETIME_FORMAT) - }, - ] - assert payload['sent'] == sent - - -@pytest.mark.parametrize('description, expected_language', ( - ('my-description', 'English'), - ('mΕ·-description', 'Welsh'), -)) -def test_cbc_proxy_vodafone_create_and_send_invokes_function( - mocker, - cbc_proxy_vodafone, - description, - expected_language, -): - identifier = 'my-identifier' - headline = 'my-headline' - - sent = 'a-passed-through-sent-value' - expires = 'a-passed-through-expires-value' - - ld_client_mock = mocker.patch.object( - cbc_proxy_vodafone, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy_vodafone.create_and_send_broadcast( - identifier=identifier, - message_number='0000007b', - headline=headline, - description=description, - areas=EXAMPLE_AREAS, - sent=sent, - expires=expires, - channel="test", - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName='vodafone-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == identifier - assert payload['message_number'] == '0000007b' - assert payload['message_format'] == 'ibag' - assert payload['message_type'] == 'alert' - assert payload['headline'] == headline - assert payload['description'] == description - assert payload['areas'] == EXAMPLE_AREAS - assert payload['sent'] == sent - assert payload['expires'] == expires - assert payload['language'] == expected_language - assert payload['channel'] == 'test' - - -def test_cbc_proxy_vodafone_cancel_invokes_function(mocker, cbc_proxy_vodafone): - identifier = 'my-identifier' - MockProviderMessage = namedtuple( - 'BroadcastProviderMessage', - ['id', 'message_number', 'created_at'] - ) - - provider_messages = [ - MockProviderMessage(uuid.uuid4(), 78, datetime(2020, 12, 16)), - MockProviderMessage(uuid.uuid4(), 123, datetime(2020, 12, 17)) - ] - sent = '2020-12-18 14:19:44.130585' - - ld_client_mock = mocker.patch.object( - cbc_proxy_vodafone, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy_vodafone.cancel_broadcast( - identifier=identifier, - message_number='00000050', - previous_provider_messages=provider_messages, - sent=sent - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName='vodafone-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == identifier - assert payload['message_number'] == '00000050' - assert payload['message_format'] == 'ibag' - assert payload['message_type'] == 'cancel' - assert payload['references'] == [ - { - "message_id": str(provider_messages[0].id), - "message_number": '0000004e', - "sent": provider_messages[0].created_at.strftime(DATETIME_FORMAT) - }, - { - "message_id": str(provider_messages[1].id), - "message_number": '0000007b', - "sent": provider_messages[1].created_at.strftime(DATETIME_FORMAT) - }, - ] - assert payload['sent'] == sent - - -@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2']) -def test_cbc_proxy_will_failover_to_second_lambda_if_boto_client_error( - mocker, - cbc_proxy_client, - cbc -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.side_effect = BotoClientError({}, 'error') - - with pytest.raises(CBCProxyRetryableException) as e: - cbc_proxy.create_and_send_broadcast( - identifier='my-identifier', - message_number='0000007b', - headline='my-headline', - description='test-description', - areas=EXAMPLE_AREAS, - sent='a-passed-through-sent-value', - expires='a-passed-through-expires-value', - channel="severe", - ) - - assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy') - - assert ld_client_mock.invoke.call_args_list == [ - call( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ), - call( - FunctionName=f'{cbc}-2-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - ] - - -@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2']) -def test_cbc_proxy_will_failover_to_second_lambda_if_function_error( - mocker, - cbc_proxy_client, - cbc -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.side_effect = [ - { - 'StatusCode': 200, - 'FunctionError': 'Handled', - 'Payload': BytesIO(json.dumps({"errorMessage": "", "errorType": "CBCNewConnectionError"}).encode('utf-8')), - }, - { - 'StatusCode': 200 - } - ] - - cbc_proxy.create_and_send_broadcast( - identifier='my-identifier', - message_number='0000007b', - headline='my-headline', - description='test-description', - areas=EXAMPLE_AREAS, - sent='a-passed-through-sent-value', - expires='a-passed-through-expires-value', - channel="severe", - ) - - assert ld_client_mock.invoke.call_args_list == [ - call( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ), - call( - FunctionName=f'{cbc}-2-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - ] - - -@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2']) -def test_cbc_proxy_will_failover_to_second_lambda_if_invoke_error( - mocker, - cbc_proxy_client, - cbc -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.side_effect = [ - { - 'StatusCode': 400 - }, - { - 'StatusCode': 200 - } - ] - - cbc_proxy.create_and_send_broadcast( - identifier='my-identifier', - message_number='0000007b', - headline='my-headline', - description='test-description', - areas=EXAMPLE_AREAS, - sent='a-passed-through-sent-value', - expires='a-passed-through-expires-value', - channel="test", - ) - - assert ld_client_mock.invoke.call_args_list == [ - call( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ), - call( - FunctionName=f'{cbc}-2-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - ] - - -@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2']) -def test_cbc_proxy_create_and_send_tries_failover_lambda_on_invoke_error_and_raises_if_both_invoke_error( - mocker, cbc_proxy_client, cbc -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 400, - } - - with pytest.raises(CBCProxyRetryableException) as e: - cbc_proxy.create_and_send_broadcast( - identifier='my-identifier', - message_number='0000007b', - headline='my-headline', - description='my-description', - areas=EXAMPLE_AREAS, - sent='a-passed-through-sent-value', - expires='a-passed-through-expires-value', - channel="test", - ) - - assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy') - - assert ld_client_mock.invoke.call_args_list == [ - call( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ), - call( - FunctionName=f'{cbc}-2-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - ] - - -@pytest.mark.parametrize('cbc', ['ee', 'vodafone', 'three', 'o2']) -def test_cbc_proxy_create_and_send_tries_failover_lambda_on_function_error_and_raises_if_both_function_error( - mocker, cbc_proxy_client, cbc -): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - 'FunctionError': 'something', - 'Payload': BytesIO(json.dumps({"errorMessage": "some message", "errorType": "SomeErrorType"}).encode('utf-8')), - } - - with pytest.raises(CBCProxyRetryableException) as e: - cbc_proxy.create_and_send_broadcast( - identifier='my-identifier', - message_number='0000007b', - headline='my-headline', - description='my-description', - areas=EXAMPLE_AREAS, - sent='a-passed-through-sent-value', - expires='a-passed-through-expires-value', - channel="severe", - ) - - assert e.match(f'Lambda failed for both {cbc}-1-proxy and {cbc}-2-proxy') - - assert ld_client_mock.invoke.call_args_list == [ - call( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ), - call( - FunctionName=f'{cbc}-2-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - ] - - -@pytest.mark.parametrize('cbc', ['ee', 'three', 'o2']) -def test_cbc_proxy_one_2_many_send_link_test_invokes_function(mocker, cbc_proxy_client, cbc): - cbc_proxy = cbc_proxy_client.get_proxy(cbc) - - mocker.patch('app.clients.cbc_proxy.uuid.uuid4', return_value=123) - - ld_client_mock = mocker.patch.object( - cbc_proxy, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy._send_link_test( - lambda_name=f'{cbc}-1-proxy' - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName=f'{cbc}-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == '123' - assert payload['message_type'] == 'test' - assert 'message_number' not in payload - assert payload['message_format'] == 'cap' - - -def test_cbc_proxy_vodafone_send_link_test_invokes_function(mocker, cbc_proxy_vodafone): - mocker.patch('app.clients.cbc_proxy.uuid.uuid4', return_value=123) - - db.session.connection().execute( - 'ALTER SEQUENCE broadcast_provider_message_number_seq RESTART WITH 1' - ) - - ld_client_mock = mocker.patch.object( - cbc_proxy_vodafone, - '_lambda_client', - create=True, - ) - - ld_client_mock.invoke.return_value = { - 'StatusCode': 200, - } - - cbc_proxy_vodafone._send_link_test( - lambda_name='vodafone-1-proxy' - ) - - ld_client_mock.invoke.assert_called_once_with( - FunctionName='vodafone-1-proxy', - InvocationType='RequestResponse', - Payload=mocker.ANY, - ) - - kwargs = ld_client_mock.invoke.mock_calls[0][-1] - payload_bytes = kwargs['Payload'] - payload = json.loads(payload_bytes) - - assert payload['identifier'] == '123' - assert payload['message_type'] == 'test' - assert payload['message_number'] == '00000001' - assert payload['message_format'] == 'ibag' diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 61b87ca8b..36229c59b 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -10,9 +10,6 @@ from sqlalchemy.orm.session import make_transient from app import db from app.dao.api_key_dao import save_model_api_key -from app.dao.broadcast_service_dao import ( - insert_or_update_service_broadcast_settings, -) from app.dao.invited_user_dao import save_invited_user from app.dao.jobs_dao import dao_create_job from app.dao.notifications_dao import dao_create_notification @@ -25,7 +22,6 @@ from app.dao.templates_dao import dao_create_template from app.dao.users_dao import create_secret_code, create_user_code from app.history_meta import create_history from app.models import ( - BROADCAST_TYPE, EMAIL_TYPE, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, @@ -155,60 +151,6 @@ def sample_service(sample_user): return service -@pytest.fixture(scope='function') -def sample_broadcast_service(broadcast_organisation, sample_user): - service_name = 'Sample broadcast service' - email_from = service_name.lower().replace(' ', '.') - - data = { - 'name': service_name, - 'message_limit': 1000, - 'restricted': False, - 'email_from': email_from, - 'created_by': sample_user, - 'crown': True, - 'count_as_live': False, - } - service = Service.query.filter_by(name=service_name).first() - if not service: - service = Service(**data) - dao_create_service(service, sample_user, service_permissions=[BROADCAST_TYPE]) - insert_or_update_service_broadcast_settings(service, channel="severe") - dao_add_service_to_organisation(service, current_app.config['BROADCAST_ORGANISATION_ID']) - else: - if sample_user not in service.users: - dao_add_user_to_service(service, sample_user) - - return service - - -@pytest.fixture(scope='function') -def sample_broadcast_service_2(broadcast_organisation, sample_user): - service_name = 'Sample broadcast service 2' - email_from = service_name.lower().replace(' ', '.') - - data = { - 'name': service_name, - 'message_limit': 1000, - 'restricted': False, - 'email_from': email_from, - 'created_by': sample_user, - 'crown': True, - 'count_as_live': False, - } - service = Service.query.filter_by(name=service_name).first() - if not service: - service = Service(**data) - dao_create_service(service, sample_user, service_permissions=[BROADCAST_TYPE]) - insert_or_update_service_broadcast_settings(service, channel="severe") - dao_add_service_to_organisation(service, current_app.config['BROADCAST_ORGANISATION_ID']) - else: - if sample_user not in service.users: - dao_add_user_to_service(service, sample_user) - - return service - - @pytest.fixture(scope='function', name='sample_service_full_permissions') def _sample_service_full_permissions(notify_db_session): service = create_service( @@ -662,19 +604,6 @@ def invitation_email_template(notify_service): ) -@pytest.fixture(scope='function') -def broadcast_invitation_email_template(notify_service): - content = '((user_name)) is invited to broadcast Notify by ((service_name)) ((url)) to complete registration', - return create_custom_template( - service=notify_service, - user=notify_service.users[0], - template_config_name='BROADCAST_INVITATION_EMAIL_TEMPLATE_ID', - content=content, - subject='Invitation to ((service_name))', - template_type='email' - ) - - @pytest.fixture(scope='function') def org_invite_email_template(notify_service): return create_custom_template( @@ -894,16 +823,6 @@ def sample_organisation(notify_db_session): return org -@pytest.fixture -def broadcast_organisation(notify_db_session): - org = Organisation.query.get(current_app.config['BROADCAST_ORGANISATION_ID']) - if not org: - org = Organisation(id=current_app.config['BROADCAST_ORGANISATION_ID'], name='broadcast organisation') - dao_create_organisation(org) - - return org - - @pytest.fixture def nhs_email_branding(notify_db_session): # we wipe email_branding table in test db between the tests, so we have to recreate this branding diff --git a/tests/app/dao/test_broadcast_message_dao.py b/tests/app/dao/test_broadcast_message_dao.py deleted file mode 100644 index fa52cb066..000000000 --- a/tests/app/dao/test_broadcast_message_dao.py +++ /dev/null @@ -1,131 +0,0 @@ -from datetime import datetime - -from app.dao.broadcast_message_dao import ( - create_broadcast_provider_message, - dao_get_all_broadcast_messages, - get_earlier_events_for_broadcast_event, -) -from app.dao.broadcast_service_dao import ( - insert_or_update_service_broadcast_settings, -) -from app.models import BROADCAST_TYPE, BroadcastEventMessageType -from tests.app.db import ( - create_broadcast_event, - create_broadcast_message, - create_service, - create_template, -) - - -def test_get_earlier_events_for_broadcast_event(sample_service): - t = create_template(sample_service, BROADCAST_TYPE) - bm = create_broadcast_message(t) - - events = [ - create_broadcast_event( - bm, - sent_at=datetime(2020, 1, 1, 12, 0, 0), - message_type=BroadcastEventMessageType.ALERT, - transmitted_content={'body': 'Initial content'} - ), - create_broadcast_event( - bm, - sent_at=datetime(2020, 1, 1, 13, 0, 0), - message_type=BroadcastEventMessageType.UPDATE, - transmitted_content={'body': 'Updated content'} - ), - create_broadcast_event( - bm, - sent_at=datetime(2020, 1, 1, 14, 0, 0), - message_type=BroadcastEventMessageType.UPDATE, - transmitted_content={'body': 'Updated content'}, - transmitted_areas=['wales'] - ), - create_broadcast_event( - bm, - sent_at=datetime(2020, 1, 1, 15, 0, 0), - message_type=BroadcastEventMessageType.CANCEL, - transmitted_finishes_at=datetime(2020, 1, 1, 15, 0, 0), - ) - ] - - # only fetches earlier events, and they're in time order - earlier_events = get_earlier_events_for_broadcast_event(events[2].id) - assert earlier_events == [events[0], events[1]] - - -def test_create_broadcast_provider_message_creates_in_correct_state(sample_broadcast_service): - t = create_template(sample_broadcast_service, BROADCAST_TYPE) - broadcast_message = create_broadcast_message(t) - broadcast_event = create_broadcast_event( - broadcast_message, - sent_at=datetime(2020, 1, 1, 12, 0, 0), - message_type=BroadcastEventMessageType.ALERT, - transmitted_content={'body': 'Initial content'} - ) - - broadcast_provider_message = create_broadcast_provider_message(broadcast_event, 'fake-provider') - - assert broadcast_provider_message.status == 'sending' - assert broadcast_provider_message.broadcast_event_id == broadcast_event.id - assert broadcast_provider_message.created_at is not None - assert broadcast_provider_message.updated_at is None - - -def test_dao_get_all_broadcast_messages(sample_broadcast_service): - template_1 = create_template(sample_broadcast_service, BROADCAST_TYPE) - # older message, should appear second in list - broadcast_message_1 = create_broadcast_message( - template_1, - starts_at=datetime(2021, 6, 15, 12, 0, 0), - status='cancelled') - - service_2 = create_service( - service_name="broadcast service 2", - service_permissions=[BROADCAST_TYPE] - ) - insert_or_update_service_broadcast_settings(service_2, channel="severe") - - template_2 = create_template(service_2, BROADCAST_TYPE) - # newer message, should appear first in list - broadcast_message_2 = create_broadcast_message( - template_2, - stubbed=False, - status='broadcasting', - starts_at=datetime(2021, 6, 20, 12, 0, 0), - ) - - # broadcast_message_stubbed - create_broadcast_message( - template_2, - stubbed=True, - status='broadcasting', - starts_at=datetime(2021, 6, 15, 12, 0, 0), - ) - # broadcast_message_old - create_broadcast_message( - template_2, - stubbed=False, - status='completed', - starts_at=datetime(2021, 5, 20, 12, 0, 0), - ) - # broadcast_message_rejected - create_broadcast_message( - template_2, - stubbed=False, - status='rejected', - starts_at=datetime(2021, 6, 15, 12, 0, 0), - ) - - broadcast_messages = dao_get_all_broadcast_messages() - assert len(broadcast_messages) == 2 - assert broadcast_messages == [ - ( - broadcast_message_2.id, None, 'severe', 'Dear Sir/Madam, Hello. Yours Truly, The Government.', - {'ids': [], 'simple_polygons': []}, 'broadcasting', datetime(2021, 6, 20, 12, 0), - None, None, None), - ( - broadcast_message_1.id, None, 'severe', 'Dear Sir/Madam, Hello. Yours Truly, The Government.', - {'ids': [], 'simple_polygons': []}, 'cancelled', datetime(2021, 6, 15, 12, 0), - None, None, None) - ] diff --git a/tests/app/db.py b/tests/app/db.py index 30864e876..541e96dcf 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -36,12 +36,6 @@ from app.models import ( SMS_TYPE, AnnualBilling, ApiKey, - BroadcastEvent, - BroadcastMessage, - BroadcastProvider, - BroadcastProviderMessage, - BroadcastProviderMessageNumber, - BroadcastStatusType, Complaint, DailySortedLetter, Domain, @@ -1134,105 +1128,6 @@ def create_service_contact_list( return contact_list -def create_broadcast_message( - template=None, - *, - service=None, # only used if template is not provided - created_by=None, - personalisation=None, - content=None, - status=BroadcastStatusType.DRAFT, - starts_at=None, - finishes_at=None, - areas=None, - stubbed=False, - cap_event=None, -): - if template: - service = template.service - template_id = template.id - template_version = template.version - personalisation = personalisation or {} - content = template._as_utils_template_with_personalisation( - personalisation - ).content_with_placeholders_filled_in - elif content: - template_id = None - template_version = None - personalisation = None - content = content - else: - pytest.fail('Provide template or content') - - broadcast_message = BroadcastMessage( - service_id=service.id, - template_id=template_id, - template_version=template_version, - personalisation=personalisation, - status=status, - starts_at=starts_at, - finishes_at=finishes_at, - created_by_id=created_by.id if created_by else service.created_by_id, - areas=areas or {'ids': [], 'simple_polygons': []}, - content=content, - stubbed=stubbed, - cap_event=cap_event, - ) - db.session.add(broadcast_message) - db.session.commit() - return broadcast_message - - -def create_broadcast_event( - broadcast_message, - sent_at=None, - message_type='alert', - transmitted_content=None, - transmitted_areas=None, - transmitted_sender=None, - transmitted_starts_at=None, - transmitted_finishes_at=None, -): - b_e = BroadcastEvent( - service=broadcast_message.service, - broadcast_message=broadcast_message, - sent_at=sent_at or datetime.utcnow(), - message_type=message_type, - transmitted_content=transmitted_content or {'body': 'this is an emergency broadcast message'}, - transmitted_areas=transmitted_areas or broadcast_message.areas, - transmitted_sender=transmitted_sender or 'www.notifications.service.gov.uk', - transmitted_starts_at=transmitted_starts_at, - transmitted_finishes_at=transmitted_finishes_at or datetime.utcnow() + timedelta(hours=24), - ) - db.session.add(b_e) - db.session.commit() - return b_e - - -def create_broadcast_provider_message( - broadcast_event, - provider, - status='sending' -): - broadcast_provider_message_id = uuid.uuid4() - provider_message = BroadcastProviderMessage( - id=broadcast_provider_message_id, - broadcast_event=broadcast_event, - provider=provider, - status=status, - ) - db.session.add(provider_message) - db.session.commit() - - provider_message_number = None - if provider == BroadcastProvider.VODAFONE: - provider_message_number = BroadcastProviderMessageNumber( - broadcast_provider_message_id=broadcast_provider_message_id) - db.session.add(provider_message_number) - db.session.commit() - return provider_message - - def create_webauthn_credential( user, name='my key', diff --git a/tests/app/govuk_alerts/__init__.py b/tests/app/govuk_alerts/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/app/govuk_alerts/test_get_broadcasts.py b/tests/app/govuk_alerts/test_get_broadcasts.py deleted file mode 100644 index a0a683c55..000000000 --- a/tests/app/govuk_alerts/test_get_broadcasts.py +++ /dev/null @@ -1,41 +0,0 @@ -from datetime import datetime - -import pytest -from flask import current_app, json - -from app.models import BROADCAST_TYPE -from tests import create_internal_authorization_header -from tests.app.db import create_broadcast_message, create_template - - -@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason") -def test_get_all_broadcasts_returns_list_of_broadcasts_and_200( - client, sample_broadcast_service -): - template_1 = create_template(sample_broadcast_service, BROADCAST_TYPE) - - broadcast_message_1 = create_broadcast_message( - template_1, - starts_at=datetime(2021, 6, 15, 12, 0, 0), - status='cancelled') - - broadcast_message_2 = create_broadcast_message( - template_1, - starts_at=datetime(2021, 6, 22, 12, 0, 0), - status='broadcasting') - - jwt_client_id = current_app.config['GOVUK_ALERTS_CLIENT_ID'] - header = create_internal_authorization_header(jwt_client_id) - - response = client.get('/govuk-alerts', headers=[header]) - - json_response = json.loads(response.get_data(as_text=True)) - - assert response.status_code == 200 - assert len(json_response['alerts']) == 2 - - assert json_response['alerts'][0]['id'] == str(broadcast_message_2.id) - assert json_response['alerts'][0]['starts_at'] == '2021-06-22T12:00:00.000000Z' - assert json_response['alerts'][0]['finishes_at'] is None - assert json_response['alerts'][1]['id'] == str(broadcast_message_1.id) - assert json_response['alerts'][1]['starts_at'] == '2021-06-15T12:00:00.000000Z' diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 6102da61a..9506047a2 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -18,7 +18,6 @@ from app.dao.services_dao import ( from app.dao.templates_dao import dao_redact_template from app.dao.users_dao import save_model_user from app.models import ( - BROADCAST_TYPE, EMAIL_AUTH_TYPE, EMAIL_TYPE, INBOUND_SMS_TYPE, @@ -38,7 +37,6 @@ from app.models import ( Notification, Permission, Service, - ServiceBroadcastSettings, ServiceEmailReplyTo, ServiceLetterContact, ServicePermission, @@ -250,16 +248,12 @@ def test_get_service_by_id(admin_request, sample_service): assert not json_resp['data']['research_mode'] assert json_resp['data']['email_branding'] is None assert json_resp['data']['prefix_sms'] is True - assert json_resp['data']['allowed_broadcast_provider'] is None - assert json_resp['data']['broadcast_channel'] is None assert set(json_resp['data'].keys()) == { 'active', - 'allowed_broadcast_provider', 'billing_contact_email_addresses', 'billing_contact_names', 'billing_reference', - 'broadcast_channel', 'consent_to_research', 'contact_link', 'count_as_live', @@ -289,28 +283,6 @@ def test_get_service_by_id(admin_request, sample_service): } -@pytest.mark.parametrize('broadcast_channel,allowed_broadcast_provider', ( - ('operator', 'all'), - ('test', 'all'), - ('severe', 'all'), - ('government', 'all'), - ('operator', 'o2'), - ('test', 'ee'), - ('severe', 'three'), - ('government', 'vodafone'), -)) -def test_get_service_by_id_for_broadcast_service_returns_broadcast_keys( - notify_db_session, admin_request, sample_broadcast_service, broadcast_channel, allowed_broadcast_provider -): - sample_broadcast_service.broadcast_channel = broadcast_channel - sample_broadcast_service.allowed_broadcast_provider = allowed_broadcast_provider - - json_resp = admin_request.get('service.get_service_by_id', service_id=sample_broadcast_service.id) - assert json_resp['data']['id'] == str(sample_broadcast_service.id) - assert json_resp['data']['allowed_broadcast_provider'] == allowed_broadcast_provider - assert json_resp['data']['broadcast_channel'] == broadcast_channel - - @pytest.mark.parametrize('detailed', [True, False]) def test_get_service_by_id_returns_organisation_type(admin_request, sample_service, detailed): json_resp = admin_request.get('service.get_service_by_id', service_id=sample_service.id, detailed=detailed) @@ -1000,7 +972,6 @@ def test_update_service_permissions_will_add_service_permissions(client, sample_ (LETTER_TYPE), (INBOUND_SMS_TYPE), (EMAIL_AUTH_TYPE), - (BROADCAST_TYPE), # TODO: remove this ability to set broadcast permission this way ] ) def test_add_service_permission_will_add_permission(client, service_with_no_permissions, permission_to_add): @@ -3728,496 +3699,3 @@ def test_get_returned_letter(admin_request, sample_letter_template): assert not response[4]['original_file_name'] assert not response[4]['job_row_number'] assert response[4]['uploaded_letter_file_name'] == 'filename.pdf' - - -@pytest.mark.parametrize('channel', ["operator", "test", "severe", "government"]) -def test_set_as_broadcast_service_sets_broadcast_channel( - admin_request, sample_service, broadcast_organisation, channel -): - assert sample_service.service_broadcast_settings is None - data = { - 'broadcast_channel': channel, - 'service_mode': 'live', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample service' - assert result['data']['broadcast_channel'] == channel - - records = ServiceBroadcastSettings.query.filter_by(service_id=sample_service.id).all() - assert len(records) == 1 - assert records[0].service_id == sample_service.id - assert records[0].channel == channel - - -def test_set_as_broadcast_service_updates_channel_for_broadcast_service( - admin_request, sample_broadcast_service -): - assert sample_broadcast_service.broadcast_channel == "severe" - - data = { - 'broadcast_channel': "test", - 'service_mode': 'training', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_broadcast_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample broadcast service' - assert result['data']['broadcast_channel'] == "test" - - records = ServiceBroadcastSettings.query.filter_by(service_id=sample_broadcast_service.id).all() - assert len(records) == 1 - assert records[0].service_id == sample_broadcast_service.id - assert records[0].channel == "test" - - -@pytest.mark.parametrize('channel', ["extreme", "exercise", "random", ""]) -def test_set_as_broadcast_service_rejects_unknown_channels( - admin_request, sample_service, broadcast_organisation, channel -): - data = { - 'broadcast_channel': channel, - 'service_mode': 'live', - 'provider_restriction': "all", - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -def test_set_as_broadcast_service_rejects_if_no_channel( - admin_request, notify_db_session, sample_service, broadcast_organisation -): - data = { - 'service_mode': 'training', - 'provider_restriction': "all", - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -@pytest.mark.parametrize('starting_permissions, ending_permissions', ( - ([], [BROADCAST_TYPE]), - ([EMAIL_AUTH_TYPE], [BROADCAST_TYPE, EMAIL_AUTH_TYPE]), - ([p for p in SERVICE_PERMISSION_TYPES if p != BROADCAST_TYPE], [BROADCAST_TYPE, EMAIL_AUTH_TYPE]), -)) -def test_set_as_broadcast_service_gives_broadcast_permission_and_removes_other_channel_permissions( - admin_request, broadcast_organisation, starting_permissions, ending_permissions -): - sample_service = create_service(service_permissions=starting_permissions) - data = { - 'broadcast_channel': "severe", - 'service_mode': 'training', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert set(result['data']['permissions']) == set(ending_permissions) - - permissions = ServicePermission.query.filter_by(service_id=sample_service.id).all() - assert set([p.permission for p in permissions]) == set(ending_permissions) - - -@pytest.mark.parametrize('has_email_auth, ending_permissions', ( - (False, [BROADCAST_TYPE]), - (True, [BROADCAST_TYPE, EMAIL_AUTH_TYPE]), -)) -def test_set_as_broadcast_service_maintains_broadcast_permission_for_existing_broadcast_service( - admin_request, sample_broadcast_service, has_email_auth, ending_permissions -): - if has_email_auth: - service_permission = ServicePermission(service_id=sample_broadcast_service.id, permission=EMAIL_AUTH_TYPE) - sample_broadcast_service.permissions.append(service_permission) - - current_permissions = [p.permission for p in sample_broadcast_service.permissions] - assert set(current_permissions) == set(ending_permissions) - - data = { - 'broadcast_channel': "severe", - 'service_mode': 'live', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_broadcast_service.id, - _data=data, - ) - assert set(result['data']['permissions']) == set(ending_permissions) - - permissions = ServicePermission.query.filter_by(service_id=sample_broadcast_service.id).all() - assert set([p.permission for p in permissions]) == set(ending_permissions) - - -def test_set_as_broadcast_service_sets_count_as_live_to_false( - admin_request, sample_service, broadcast_organisation -): - assert sample_service.count_as_live is True - - data = { - 'broadcast_channel': "severe", - 'service_mode': 'live', - 'provider_restriction': "all", - } - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert result['data']['count_as_live'] is False - - service_from_db = Service.query.filter_by(id=sample_service.id).all()[0] - assert service_from_db.count_as_live is False - - -def test_set_as_broadcast_service_sets_service_org_to_broadcast_org( - admin_request, sample_service, broadcast_organisation -): - assert sample_service.organisation_id != current_app.config['BROADCAST_ORGANISATION_ID'] - - data = { - 'broadcast_channel': "severe", - 'service_mode': 'training', - 'provider_restriction': "all", - } - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert result['data']['organisation'] == current_app.config['BROADCAST_ORGANISATION_ID'] - - service_from_db = Service.query.filter_by(id=sample_service.id).all()[0] - assert str(service_from_db.organisation_id) == current_app.config['BROADCAST_ORGANISATION_ID'] - - -def test_set_as_broadcast_service_does_not_error_if_run_on_a_service_that_is_already_a_broadcast_service( - admin_request, sample_service, broadcast_organisation -): - data = { - 'broadcast_channel': "severe", - 'service_mode': "live", - 'provider_restriction': "all", - } - for _ in range(2): - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - - -@freeze_time('2021-02-02') -def test_set_as_broadcast_service_sets_service_to_live_mode( - admin_request, notify_db_session, sample_service, broadcast_organisation -): - sample_service.restricted = True - notify_db_session.add(sample_service) - notify_db_session.commit() - assert sample_service.restricted is True - assert sample_service.go_live_at is None - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'live', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample service' - assert result['data']['restricted'] is False - assert result['data']['go_live_at'] == '2021-02-02 00:00:00.000000' - - -def test_set_as_broadcast_service_doesnt_override_existing_go_live_at( - admin_request, notify_db_session, sample_broadcast_service -): - sample_broadcast_service.restricted = False - sample_broadcast_service.go_live_at = datetime(2021, 1, 1) - notify_db_session.add(sample_broadcast_service) - notify_db_session.commit() - assert sample_broadcast_service.restricted is False - assert sample_broadcast_service.go_live_at is not None - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'live', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_broadcast_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample broadcast service' - assert result['data']['restricted'] is False - assert result['data']['go_live_at'] == '2021-01-01 00:00:00.000000' - - -def test_set_as_broadcast_service_sets_service_to_training_mode( - admin_request, notify_db_session, sample_broadcast_service -): - sample_broadcast_service.restricted = False - sample_broadcast_service.go_live_at = datetime(2021, 1, 1) - notify_db_session.add(sample_broadcast_service) - notify_db_session.commit() - assert sample_broadcast_service.restricted is False - assert sample_broadcast_service.go_live_at is not None - - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'training', - 'provider_restriction': "all", - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_broadcast_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample broadcast service' - assert result['data']['restricted'] is True - assert result['data']['go_live_at'] is None - - -@pytest.mark.parametrize('service_mode', ["testing", ""]) -def test_set_as_broadcast_service_rejects_unknown_service_mode( - admin_request, sample_service, broadcast_organisation, service_mode -): - data = { - 'broadcast_channel': 'severe', - 'service_mode': service_mode, - 'provider_restriction': "all", - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -def test_set_as_broadcast_service_rejects_if_no_service_mode( - admin_request, sample_service, broadcast_organisation -): - data = { - 'broadcast_channel': 'severe', - 'provider_restriction': "all", - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -@pytest.mark.parametrize('provider', ["all", "three", "ee", "vodafone", "o2"]) -def test_set_as_broadcast_service_sets_mobile_provider_restriction( - admin_request, sample_service, broadcast_organisation, provider -): - assert sample_service.service_broadcast_settings is None - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'live', - 'provider_restriction': provider - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - assert result['data']['name'] == 'Sample service' - assert result['data']['allowed_broadcast_provider'] == provider - - records = ServiceBroadcastSettings.query.filter_by(service_id=sample_service.id).all() - assert len(records) == 1 - assert records[0].service_id == sample_service.id - assert records[0].provider == provider - - -@pytest.mark.parametrize('provider', ["all", "vodafone"]) -def test_set_as_broadcast_service_updates_mobile_provider_restriction( - admin_request, notify_db_session, sample_broadcast_service, provider -): - sample_broadcast_service.service_broadcast_settings.provider = "o2" - notify_db_session.add(sample_broadcast_service) - notify_db_session.commit() - assert sample_broadcast_service.service_broadcast_settings.provider == "o2" - - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'live', - 'provider_restriction': provider - } - - result = admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_broadcast_service.id, - _data=data, - ) - - assert result['data']['name'] == 'Sample broadcast service' - assert result['data']['allowed_broadcast_provider'] == provider - - records = ServiceBroadcastSettings.query.filter_by(service_id=sample_broadcast_service.id).all() - assert len(records) == 1 - assert records[0].service_id == sample_broadcast_service.id - assert records[0].provider == provider - - -@pytest.mark.parametrize('provider', ["three, o2", "giffgaff", "", "None"]) -def test_set_as_broadcast_service_rejects_unknown_provider_restriction( - admin_request, sample_service, broadcast_organisation, provider -): - data = { - 'broadcast_channel': 'test', - 'service_mode': 'live', - 'provider_restriction': provider - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -def test_set_as_broadcast_service_errors_if_no_mobile_provider_restriction( - admin_request, sample_service, broadcast_organisation -): - data = { - 'broadcast_channel': 'severe', - 'service_mode': 'live', - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - _expected_status=400, - ) - - -def test_set_as_broadcast_service_updates_services_history( - admin_request, sample_service, broadcast_organisation -): - old_history_records = Service.get_history_model().query.filter_by(id=sample_service.id).all() - data = { - 'broadcast_channel': 'test', - 'service_mode': 'live', - 'provider_restriction': "all", - } - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data=data, - ) - - new_history_records = Service.get_history_model().query.filter_by(id=sample_service.id).all() - assert len(new_history_records) == len(old_history_records) + 1 - - -def test_set_as_broadcast_service_removes_user_permissions( - admin_request, - broadcast_organisation, - sample_service, - sample_service_full_permissions, - sample_invited_user, -): - service_user = sample_service.users[0] - - # make the user a member of a second service - dao_add_user_to_service( - sample_service_full_permissions, - service_user, - permissions=[ - Permission(service_id=sample_service_full_permissions.id, - user_id=service_user.id, - permission='send_emails') - ] - ) - assert len(service_user.get_permissions(service_id=sample_service.id)) == 8 - assert len(sample_invited_user.get_permissions()) == 3 - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data={ - 'broadcast_channel': 'test', - 'service_mode': 'live', - 'provider_restriction': 'ee' - } - ) - - # The user permissions for the broadcast service (apart from 'view_activity') get removed - assert service_user.get_permissions(service_id=sample_service.id) == ['view_activity'] - - # Permissions for users invited to the broadcast service (apart from 'view_activity') get removed - assert sample_invited_user.permissions == 'view_activity' - - # Permissions for other services remain - assert service_user.get_permissions(service_id=sample_service_full_permissions.id) == ['send_emails'] - - -@freeze_time('2021-12-21') -def test_set_as_broadcast_service_revokes_api_keys( - admin_request, - broadcast_organisation, - sample_service, - sample_service_full_permissions, -): - api_key_1 = create_api_key(service=sample_service) - api_key_2 = create_api_key(service=sample_service) - api_key_3 = create_api_key(service=sample_service_full_permissions) - - api_key_2.expiry_date = datetime.utcnow() - timedelta(days=365) - - admin_request.post( - 'service.set_as_broadcast_service', - service_id=sample_service.id, - _data={ - 'broadcast_channel': 'government', - 'service_mode': 'live', - 'provider_restriction': 'all', - } - ) - - # This key should have a new expiry date - assert api_key_1.expiry_date.isoformat().startswith('2021-12-21') - - # This key keeps its old expiry date - assert api_key_2.expiry_date.isoformat().startswith('2020-12-21') - - # This key is from a different service - assert api_key_3.expiry_date is None diff --git a/tests/app/service_invite/test_service_invite_rest.py b/tests/app/service_invite/test_service_invite_rest.py index 23528a9ef..b413fca07 100644 --- a/tests/app/service_invite/test_service_invite_rest.py +++ b/tests/app/service_invite/test_service_invite_rest.py @@ -73,60 +73,6 @@ def test_create_invited_user( mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks") -@pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason") -@pytest.mark.parametrize('extra_args, expected_start_of_invite_url', [ - ( - {}, - 'http://localhost:6012/invitation/' - ), - ( - {'invite_link_host': 'https://www.example.com'}, - 'https://www.example.com/invitation/' - ), -]) -def test_invited_user_for_broadcast_service_receives_broadcast_invite_email( - admin_request, - sample_broadcast_service, - mocker, - broadcast_invitation_email_template, - extra_args, - expected_start_of_invite_url, -): - mocked = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') - email_address = 'invited_user@service.gov.uk' - invite_from = sample_broadcast_service.users[0] - - data = dict( - service=str(sample_broadcast_service.id), - email_address=email_address, - from_user=str(invite_from.id), - permissions='send_messages,manage_service,manage_api_keys', - auth_type=EMAIL_AUTH_TYPE, - folder_permissions=['folder_1', 'folder_2', 'folder_3'], - **extra_args - ) - - admin_request.post( - 'service_invite.create_invited_user', - service_id=sample_broadcast_service.id, - _data=data, - _expected_status=201 - ) - - notification = Notification.query.first() - - assert notification.reply_to_text == invite_from.email_address - - assert len(notification.personalisation.keys()) == 3 - assert notification.personalisation['service_name'] == 'Sample broadcast service' - assert notification.personalisation['user_name'] == 'Test User' - assert notification.personalisation['url'].startswith(expected_start_of_invite_url) - assert len(notification.personalisation['url']) > len(expected_start_of_invite_url) - assert str(notification.template_id) == current_app.config['BROADCAST_INVITATION_EMAIL_TEMPLATE_ID'] - - mocked.assert_called_once_with([(str(notification.id))], queue="notify-internal-tasks") - - @pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason") def test_create_invited_user_without_auth_type(admin_request, sample_service, mocker, invitation_email_template): mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') diff --git a/tests/app/template/test_rest.py b/tests/app/template/test_rest.py index 1b2df883d..f47ca3527 100644 --- a/tests/app/template/test_rest.py +++ b/tests/app/template/test_rest.py @@ -19,7 +19,6 @@ from app.dao.templates_dao import ( dao_update_template, ) from app.models import ( - BROADCAST_TYPE, EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, @@ -38,7 +37,6 @@ from tests.conftest import set_config_values @pytest.mark.parametrize('template_type, subject', [ - (BROADCAST_TYPE, None), (SMS_TYPE, None), (EMAIL_TYPE, 'subject'), (LETTER_TYPE, 'subject'), @@ -218,7 +216,6 @@ def test_should_raise_error_if_service_does_not_exist_on_create(client, sample_u @pytest.mark.parametrize('permissions, template_type, subject, expected_error', [ - ([EMAIL_TYPE, SMS_TYPE, LETTER_TYPE], BROADCAST_TYPE, None, {'template_type': ['Creating broadcast message templates is not allowed']}), # noqa ([EMAIL_TYPE], SMS_TYPE, None, {'template_type': ['Creating text message templates is not allowed']}), ([SMS_TYPE], EMAIL_TYPE, 'subject', {'template_type': ['Creating email templates is not allowed']}), ([SMS_TYPE], LETTER_TYPE, 'subject', {'template_type': ['Creating letter templates is not allowed']}), @@ -557,7 +554,6 @@ def test_should_get_return_all_fields_by_default( ) assert json_response['data'][0].keys() == { 'archived', - 'broadcast_data', 'content', 'created_at', 'created_by', @@ -588,7 +584,6 @@ def test_should_get_return_all_fields_by_default( (EMAIL_TYPE, None), (SMS_TYPE, None), (LETTER_TYPE, None), - (BROADCAST_TYPE, 'This is a test'), )) def test_should_not_return_content_and_subject_if_requested( admin_request, @@ -753,7 +748,7 @@ def test_should_return_404_if_no_templates_for_service_with_id(client, sample_se @pytest.mark.parametrize('template_type', ( - SMS_TYPE, BROADCAST_TYPE, + SMS_TYPE, )) def test_create_400_for_over_limit_content( client, diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 16b21116a..d883371bb 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -2,7 +2,6 @@ import pytest from app.commands import ( insert_inbound_numbers_from_file, - local_dev_broadcast_permissions, populate_annual_billing_with_defaults, ) from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers @@ -22,26 +21,6 @@ def test_insert_inbound_numbers_from_file(notify_db_session, notify_api, tmpdir) assert set(x.number for x in inbound_numbers) == {'07700900373', '07700900473', '07700900375'} -def test_local_dev_broadcast_permissions( - sample_service, - sample_broadcast_service, - notify_api, -): - user = create_user() - dao_add_user_to_service(sample_service, user) - dao_add_user_to_service(sample_broadcast_service, user) - - assert len(user.get_permissions(sample_service.id)) == 0 - assert len(user.get_permissions(sample_broadcast_service.id)) == 0 - - notify_api.test_cli_runner().invoke( - local_dev_broadcast_permissions, ['-u', user.id] - ) - - assert len(user.get_permissions(sample_service.id)) == 0 - assert len(user.get_permissions(sample_broadcast_service.id)) > 0 - - @pytest.mark.parametrize("organisation_type, expected_allowance", [('central', 40000), ('local', 20000), diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 84fe1e549..8e8ba42e1 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -79,5 +79,4 @@ def test_queue_names_all_queues_correct(): QueueNames.SMS_CALLBACKS, QueueNames.SAVE_API_EMAIL, QueueNames.SAVE_API_SMS, - QueueNames.BROADCASTS, ]) == set(queues) diff --git a/tests/app/test_model.py b/tests/app/test_model.py index 5924f0a0b..b5bcdd5ef 100644 --- a/tests/app/test_model.py +++ b/tests/app/test_model.py @@ -359,9 +359,5 @@ def test_user_can_use_webauthn_if_they_login_with_it(sample_user, auth_type, can assert sample_user.can_use_webauthn == can_use_webauthn -def test_user_can_use_webauthn_if_in_broadcast_org(sample_broadcast_service): - assert sample_broadcast_service.users[0].can_use_webauthn - - def test_user_can_use_webauthn_if_in_notify_team(notify_service): assert notify_service.users[0].can_use_webauthn diff --git a/tests/app/v2/broadcast/__init__.py b/tests/app/v2/broadcast/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/app/v2/broadcast/sample_cap_xml_documents.py b/tests/app/v2/broadcast/sample_cap_xml_documents.py deleted file mode 100644 index 37acf006f..000000000 --- a/tests/app/v2/broadcast/sample_cap_xml_documents.py +++ /dev/null @@ -1,252 +0,0 @@ -import re - -WAINFLEET = """ - - 50385fcb0ab7aa447bbd46d848ce8466E - www.gov.uk/environment-agency - 2020-02-16T23:01:13-00:00 - Actual - Alert - Flood warning service - Public - - en-GB - Met - 053/055 Issue Severe Flood Warning EA - Immediate - Severe - Likely - 2020-02-26T23:01:14-00:00 - Environment Agency - A severe flood warning has been issued. Storm Dennis has resulted in significant rainfall in the Steeping River catchment with several bands of heavy rain passing through the area during today (Sunday 16 Feb). River levels along the Steeping River and the Wainfleet relief channel are expected to be similar to those in June 2019. This could result in flood embankments being overtopped. Should this happen, there is an increased risk that flood embankments could breach. It is expected that peak levels along the Steeping River in Wainfleet will be between midnight and 3am tonight. A multi-agency meeting is taking place this evening. Further messages will be issued should this be required. Do not walk on flood embankments and avoid riverside paths. Our staff are out in the area to check the flood defences and assist the emergency services and council. We will be closely monitoring the situation throughout the night. - # To check the latest information for your area - Visit [GOV.UK](https://flood-warning-information.service.gov.uk) to see the current flood warnings, view river and sea levels or check the 5-day flood risk forecast: https://flood-warning-information.service.gov.uk - Follow [@EnvAgency](https://twitter.com/EnvAgency) and [#floodaware](https://twitter.com/hashtag/floodaware) on Twitter. - Tune into weather, news and travel bulletins on local television and radio. - For access to flood warning information offline call Floodline on 0345 988 1188 using quickdial code: 307052. # What you should consider doing now - Call 999 if you are in immediate danger. - Co-operate with the emergency services and evacuate your property if told to do so. Most evacuation centres will let you bring your pets. - Act on your flood plan if you have one. - Move your family and pets to a safe place with a means of escape. - Use flood protection equipment (such as flood barriers, air brick covers and pumps) to protect your property. Unless you have proper equipment do not waste valuable time trying to keep the water out. - Move important items upstairs or to a safe place in your property, starting with cherished items of personal value that you will not be able to replace (such as family photographs). Next move valuables (such as computers), movable furniture and furnishings. - You may need to leave your property, so pack a bag with enough items for a few nights away. Include essential items including a torch with spare batteries, mobile phone and charger, warm clothes, home insurance information, water, food, first aid kit and any prescription medicines or baby care items you may need. - Turn off gas, electricity and water mains supplies before flood water starts to enter your property. Never touch an electrical switch if you are standing in water. - If it is safe to do so, make sure neighbours are aware of the situation and offer help to anyone who may need it. - Avoid walking, cycling or driving through flood water - 30 cm of fast-flowing water can move a car and 6 inches can knock an adult off their feet. - Flood water is dangerous and may be polluted. Wash your hands thoroughly if you’ve been in contact with it. ##### Businesses - Act on your business flood plan if you have one. - Move your staff and customers to a safe place with a means of escape. - Move stock and other valuable items upstairs or to a safe place in your building. For media enquiries please contact our media teams: https://www.gov.uk/government/organisations/environment-agency/about/media-enquiries - https://flood-warning-information.service.gov.uk - 0345 988 1188 - - River Steeping in Wainfleet All Saints - 53.10569,0.24453 53.10593,0.24430 53.10601,0.24375 53.10615,0.24349 53.10629,0.24356 53.10656,0.24336 53.10697,0.24354 53.10684,0.24298 53.10694,0.24264 53.10721,0.24302 53.10752,0.24310 53.10777,0.24308 53.10805,0.24320 53.10803,0.24187 53.10776,0.24085 53.10774,0.24062 53.10702,0.24056 53.10679,0.24088 53.10658,0.24071 53.10651,0.24049 53.10656,0.24022 53.10642,0.24022 53.10632,0.24052 53.10629,0.24082 53.10612,0.24093 53.10583,0.24133 53.10564,0.24178 53.10541,0.24282 53.10569,0.24453 - - TargetAreaCode - 053FWFSTEEP4 - - - - -""" - -WAINFLEET_CANCEL = """ - - 5fc99d720abb86020b233422a503af78E - www.gov.uk/environment-agency - 2020-02-16T23:02:26-00:00 - Actual - Cancel - Flood warning service - Public - - {} - - en-GB - Met - - Immediate - Severe - Likely - 2020-02-16T23:30:13-00:00 - Environment Agency - - https://flood-warning-information.service.gov.uk - 0345 988 1188 - - River Steeping in Wainfleet All Saints - 53.10569,0.24453 53.10593,0.24430 53.10601,0.24375 53.10615,0.24349 53.10629,0.24356 53.10656,0.24336 53.10697,0.24354 53.10684,0.24298 53.10694,0.24264 53.10721,0.24302 53.10752,0.24310 53.10777,0.24308 53.10805,0.24320 53.10803,0.24187 53.10776,0.24085 53.10774,0.24062 53.10702,0.24056 53.10679,0.24088 53.10658,0.24071 53.10651,0.24049 53.10656,0.24022 53.10642,0.24022 53.10632,0.24052 53.10629,0.24082 53.10612,0.24093 53.10583,0.24133 53.10564,0.24178 53.10541,0.24282 53.10569,0.24453 - - TargetAreaCode - 053FWFSTEEP4 - - - - -""" - -WAINFLEET_CANCEL_WITH_REFERENCES = WAINFLEET_CANCEL.format( - "www.gov.uk/environment-agency,50385fcb0ab7aa447bbd46d848ce8466E,2020-02-16T23:01:13-00:00" -) -WAINFLEET_CANCEL_WITH_MISSING_REFERENCES = WAINFLEET_CANCEL.format( - "" -) -WAINFLEET_CANCEL_WITH_EMPTY_REFERENCES = WAINFLEET_CANCEL.format( - "" -) - -WAINFLEET_CANCEL_WITH_WINDMERE_REFERENCES = WAINFLEET_CANCEL.format( - "" - # Wainfleet - "www.gov.uk/environment-agency,50385fcb0ab7aa447bbd46d848ce8466E,2020-02-16T23:01:13-00:00," - # Windemere - "www.gov.uk/environment-agency,4f6d28b10ab7aa447bbd46d85f1e9effE,2020-02-16T19:20:03+00:00" - "" -) - -UPDATE = """ - - PAAQ-4-mg5a94 - wcatwc@noaa.gov - 2013-01-05T10:58:23-00:00 - Actual - Update - WCATWC - Public - IPAWSv1.0 - wcatwc@noaa.gov,PAAQ-1-mg5a94,2013-01-05T09:01:16-00:00 wcatwc@noaa.gov,PAAQ-2-mg5a94,2013-01-05T09:30:16-00:00 wcatwc@noaa.gov,PAAQ-3-mg5a94,2013-01-05T10:17:31-00:00 - mg5a94 - - Geo - Tsunami Cancellation - None - Past - Unknown - Unlikely - 2013-01-05T10:58:23-00:00 - 2013-01-05T10:58:23-00:00 - NWS West Coast/Alaska Tsunami Warning Center Palmer AK - The tsunami Warning is canceled for the coastal areas of British Columbia and Alaska from the north tip of Vancouver Island, British Columbia to Cape Fairweather, Alaska (80 miles SE of Yakutat). - The tsunami Warning is canceled for the coastal areas of British Columbia and Alaska from the north tip of Vancouver Island, British Columbia to Cape Fairweather, Alaska (80 miles SE of Yakutat). - Event details: Preliminary magnitude 7.5 (Mw) earthquake / Lat: 55.300, Lon: -134.900 at 2013-01-05T08:58:20Z Tsunami cancellations indicate the end of the damaging tsunami threat. A cancellation is issued after an evaluation of sea level data confirms that a destructive tsunami will not impact the alerted region, or after tsunami levels have subsided to non-damaging levels. - Recommended Actions: Do not re-occupy hazard zones until local emergency officials indicate it is safe to do so. This will be the last West Coast/Alaska Tsunami Warning Center message issued for this event. Refer to the internet site ntwc.arh.noaa.gov for more information. - http://ntwc.arh.noaa.gov/events/PAAQ/2013/01/05/mg5a94/4/WEAK51/WEAK51.txt - - EventLocationName - 95 miles NW of Dixon Entrance, Alaska - - - EventPreliminaryMagnitude - 7.5 - - - EventPreliminaryMagnitudeType - Mw - - - EventOriginTime - 2013-01-05T08:58:20-00:00 - - - EventDepth - 5 kilometers - - - EventLatLon - 55.300,-134.900 0.000 - - - VTEC - /O.CAN.PAAQ.TS.W.0001.000000T0000Z-000000T0000Z/ - - - NWSUGC - BCZ220-210-922-912-921-911-110-AKZ026>029-023-024-019>022-025-051258- - - - ProductDefinition - Tsunami cancellations indicate the end of the damaging tsunami threat. A cancellation is issued after an evaluation of sea level data confirms that a destructive tsunami will not impact the alerted region, or after tsunami levels have subsided to non-damaging levels. - - - WEAK51 - Public Tsunami Warnings, Watches, and Advisories for AK, BC, and US West Coast - - - EAS-ORG - WXR - - - Event Data as a JSON document - application/json - http://ntwc.arh.noaa.gov/events/PAAQ/2013/01/05/mg5a94/4/WEAK51/PAAQ.json - - - 95 miles NW of Dixon Entrance, Alaska - 55.3,-134.9 0.0 - - - -""" - -WITH_PLACEHOLDER_FOR_CONTENT = """ - - 50385fcb0ab7aa447bbd46d848ce8466E - www.gov.uk/environment-agency - 2020-02-16T23:01:13-00:00 - Actual - Alert - Flood warning service - Public - www.gov.uk/environment-agency,4f6d28b10ab7aa447bbd46d85f1e9effE,2020-02-16T19:20:03+00:00 - - en-GB - Met - 053/055 Issue Severe Flood Warning EA - Immediate - Severe - Likely - 2020-02-26T23:01:14-00:00 - Environment Agency - {} - https://flood-warning-information.service.gov.uk - 0345 988 1188 - - River Steeping in Wainfleet All Saints - 53.10569,0.24453 53.10593,0.24430 53.10601,0.24375 53.10615,0.24349 53.10629,0.24356 53.10656,0.24336 53.10697,0.24354 53.10684,0.24298 53.10694,0.24264 53.10721,0.24302 53.10752,0.24310 53.10777,0.24308 53.10805,0.24320 53.10803,0.24187 53.10776,0.24085 53.10774,0.24062 53.10702,0.24056 53.10679,0.24088 53.10658,0.24071 53.10651,0.24049 53.10656,0.24022 53.10642,0.24022 53.10632,0.24052 53.10629,0.24082 53.10612,0.24093 53.10583,0.24133 53.10564,0.24178 53.10541,0.24282 53.10569,0.24453 - - TargetAreaCode - 053FWFSTEEP4 - - - - -""" - -WINDEMERE = """ - - 4f6d28b10ab7aa447bbd46d85f1e9effE - www.gov.uk/environment-agency - 2020-02-16T23:01:13-00:00 - Actual - Alert - Flood warning service - Public - www.gov.uk/environment-agency,4f6d28b10ab7aa447bbd46d85f1e9effE,2020-02-16T19:20:03+00:00 - - en-GB - Met - 053/055 Issue Severe Flood Warning EA - Immediate - Severe - Likely - 2020-02-26T23:01:14-00:00 - Environment Agency - This area has a lot of coordinates - https://flood-warning-information.service.gov.uk - 0345 988 1188 - - Windemere - - 54.377851258,-2.919733855 54.377769915,-2.919793618 54.377786831,-2.919932556 54.37784985,-2.91991857 54.377832816,-2.919795025 54.377851258,-2.919733855 - - - 54.423831395,-2.972442701 54.423742778,-2.972286474 54.423771844,-2.972025144 54.423880789,-2.971889007 54.42367635,-2.971606737 54.423474264,-2.971031684 54.42333879,-2.96999583 54.423865901,-2.969268464 54.424060196,-2.967454321 54.423890968,-2.966140238 54.423666078,-2.966165772 54.423494486,-2.966269624 54.423476638,-2.966253791 54.423305415,-2.966311413 54.423151671,-2.966431096 54.4230079,-2.966427712 54.422739437,-2.966282682 54.422561818,-2.966016492 54.422434539,-2.966198446 54.422353544,-2.966211952 54.42219365,-2.965977006 54.421955594,-2.965401156 54.421888265,-2.964829324 54.421865246,-2.964335595 54.421887523,-2.963796694 54.421844808,-2.963518273 54.421711374,-2.96334561 54.421657828,-2.963298117 54.42161462,-2.963081335 54.42149842,-2.96300155 54.42146346,-2.962877435 54.421365476,-2.962767254 54.421366091,-2.962690208 54.421296047,-2.962457388 54.42102979,-2.962035029 54.42088479,-2.962185749 54.420759849,-2.96207494 54.420779048,-2.961921273 54.420844403,-2.961614569 54.42076623,-2.961273681 54.420559067,-2.961330478 54.420100057,-2.961412202 54.419714406,-2.961310706 54.419347462,-2.961117183 54.418926358,-2.960953218 54.418610752,-2.961084531 54.417331183,-2.960376531 54.41670218,-2.960361823 54.416702792,-2.960284787 54.416523077,-2.960280585 54.41652369,-2.960203549 54.416164259,-2.960195146 54.416164872,-2.960118111 54.416119943,-2.96011706 54.416120555,-2.960040025 54.416030698,-2.960037925 54.41603131,-2.959960889 54.415986381,-2.959959839 54.415986994,-2.959882804 54.415897136,-2.959880704 54.415897748,-2.959803669 54.415762962,-2.959800519 54.415763574,-2.959723484 54.415628788,-2.959720335 54.4156294,-2.9596433 54.415449685,-2.959639102 54.415450297,-2.959562067 54.41531551,-2.959558919 54.415316123,-2.959481885 54.415136407,-2.959477687 54.415137019,-2.959400653 54.415047162,-2.959398554 54.415047774,-2.959321521 54.414957916,-2.959319422 54.414958528,-2.959242389 54.41486867,-2.95924029 54.414869282,-2.959163257 54.414824353,-2.959162208 54.414824965,-2.959085175 54.414780037,-2.959084126 54.414780648,-2.959007093 54.414735719,-2.959006044 54.414736331,-2.958929011 54.414691402,-2.958927962 54.414693849,-2.95861983 54.414738778,-2.958620879 54.414740001,-2.958466813 54.41478493,-2.958467861 54.414787375,-2.958159729 54.414832304,-2.958160777 54.414837803,-2.957467479 54.414792874,-2.957466432 54.414797149,-2.9569272 54.41475222,-2.956926153 54.414753441,-2.956772087 54.414708512,-2.956771041 54.414709122,-2.956694008 54.414754051,-2.956695054 54.414755882,-2.956463955 54.414710953,-2.956462909 54.414712173,-2.956308843 54.414577387,-2.956305705 54.414577997,-2.956228672 54.41444321,-2.956225534 54.41444382,-2.956148502 54.414398891,-2.956147456 54.414399501,-2.956070424 54.414354572,-2.956069378 54.414355182,-2.955992346 54.414265325,-2.955990254 54.414265934,-2.955913222 54.414221005,-2.955912177 54.414221615,-2.955835145 54.414176686,-2.955834099 54.414179125,-2.955525971 54.414089267,-2.955523881 54.414088048,-2.955677945 54.41399819,-2.955675854 54.4139988,-2.955598823 54.413864013,-2.955595687 54.413865232,-2.955441624 54.413820304,-2.955440579 54.413820913,-2.955363548 54.413865842,-2.955364593 54.413870107,-2.954825373 54.41373532,-2.95482224 54.413736538,-2.954668178 54.413781467,-2.954669222 54.413783294,-2.954438128 54.413648507,-2.954434996 54.413654594,-2.953664686 54.413699523,-2.953665729 54.413700739,-2.953511667 54.413745668,-2.95351271 54.413746276,-2.953435678 54.413925992,-2.95343985 54.413927209,-2.953285787 54.413972137,-2.953286829 54.413972746,-2.953209798 54.41419739,-2.953215011 54.414198606,-2.953060947 54.414153677,-2.953059904 54.414154893,-2.95290584 54.414199822,-2.952906882 54.41420043,-2.95282985 54.414245359,-2.952830893 54.414245967,-2.95275386 54.414290896,-2.952754902 54.414291503,-2.95267787 54.414336432,-2.952678912 54.41433704,-2.95260188 54.414292111,-2.952600838 54.414293327,-2.952446773 54.414248398,-2.952445732 54.414249005,-2.952368699 54.414114218,-2.952365574 54.414113611,-2.952442606 54.413799108,-2.952435314 54.413799716,-2.952358283 54.41362,-2.952354116 54.413620608,-2.952277085 54.41353075,-2.952275002 54.413531357,-2.952197971 54.413441499,-2.952195888 54.413442107,-2.952118858 54.413352249,-2.952116775 54.413352856,-2.952039744 54.413307927,-2.952038703 54.413308535,-2.951961673 54.413263606,-2.951960632 54.413264213,-2.951883601 54.413174355,-2.951881519 54.413174962,-2.951804489 54.413130033,-2.951803448 54.413130641,-2.951726418 54.413085712,-2.951725377 54.413086319,-2.951648347 54.412996461,-2.951646265 54.412997068,-2.951569235 54.412952139,-2.951568194 54.412952746,-2.951491165 54.412907817,-2.951490124 54.412908424,-2.951413094 54.412863495,-2.951412054 54.412864102,-2.951335024 54.412819173,-2.951333984 54.41281978,-2.951256954 54.412774851,-2.951255914 54.412775458,-2.951178884 54.4126856,-2.951176804 54.412686207,-2.951099774 54.412596349,-2.951097694 54.412596955,-2.951020665 54.412552026,-2.951019625 54.412552633,-2.950942596 54.412507704,-2.950941555 54.412508311,-2.950864527 54.412242376,-2.950396116 54.412197447,-2.950395077 54.412198659,-2.95024102 54.41215373,-2.950239981 54.412154336,-2.950162952 54.412109407,-2.950161913 54.41211062,-2.950007857 54.412065691,-2.950006818 54.412066903,-2.949852761 54.412021974,-2.949851722 54.412024397,-2.94954361 54.411979468,-2.949542572 54.411980074,-2.949465544 54.411935145,-2.949464505 54.41193575,-2.949387477 54.411845892,-2.949385401 54.411846498,-2.949308373 54.411891427,-2.949309411 54.411893243,-2.949078328 54.411848315,-2.94907729 54.41184892,-2.949000262 54.411803991,-2.948999224 54.411804596,-2.948922196 54.411669809,-2.948919083 54.411670415,-2.948842055 54.411580557,-2.94883998 54.411581162,-2.948762953 54.411401446,-2.948758802 54.411402051,-2.948681775 54.411222335,-2.948677625 54.41122294,-2.948600599 54.411133082,-2.948598524 54.411132477,-2.94867555 54.410862903,-2.948669325 54.410863508,-2.948592299 54.410593934,-2.948586075 54.410594539,-2.948509049 54.410459752,-2.948505937 54.410460357,-2.948428912 54.41032557,-2.9484258 54.410463503,-2.948028381 54.410358455,-2.947671578 54.410169874,-2.94765182 54.409518058,-2.948253084 54.409518663,-2.948176061 54.409563592,-2.948177098 54.409566011,-2.947869004 54.409287451,-2.947862577 54.409315255,-2.947755367 54.409252838,-2.947692297 54.409173054,-2.947551791 54.409092665,-2.947488307 54.40899092,-2.947855736 54.408937005,-2.947854492 54.408937609,-2.947777469 54.40889268,-2.947776433 54.408893889,-2.947622388 54.40884896,-2.947621352 54.408851378,-2.947313264 54.408806449,-2.947312228 54.408808866,-2.947004139 54.408763937,-2.947003104 54.408764541,-2.946926082 54.408629754,-2.946922975 54.40862915,-2.946999997 54.408179859,-2.946989641 54.408179255,-2.947066662 54.407999539,-2.947062519 54.40799833,-2.94721656 54.408088189,-2.947218632 54.408085167,-2.947603736 54.407995309,-2.947601663 54.4079941,-2.947755704 54.407949171,-2.947754668 54.407949775,-2.947677648 54.407904846,-2.947676611 54.407905451,-2.947599591 54.407860522,-2.947598555 54.407861126,-2.947521534 54.40768141,-2.94751739 54.407583727,-2.94622097 54.406513152,-2.94521031 54.405649723,-2.944142839 54.404954682,-2.944527419 54.404597002,-2.945443534 54.404597605,-2.94536652 54.404238172,-2.945358251 54.404237569,-2.945435265 54.404147711,-2.945433197 54.404148314,-2.945356184 54.404103385,-2.94535515 54.404102782,-2.945432163 54.404057853,-2.94543113 54.404059059,-2.945277103 54.40401413,-2.94527607 54.404014733,-2.945199057 54.403969803,-2.945198023 54.403970406,-2.94512101 54.403880548,-2.945118943 54.403881151,-2.94504193 54.403836222,-2.945040897 54.403836825,-2.944963884 54.403702037,-2.944960785 54.40370264,-2.944883772 54.403657711,-2.944882739 54.403657108,-2.944959751 54.403207817,-2.94494942 54.403207214,-2.945026432 54.403027498,-2.945022299 54.403026895,-2.94509931 54.402981966,-2.945098277 54.40298076,-2.945252299 54.402935831,-2.945251266 54.402936434,-2.945174254 54.403152507,-2.943977628 54.403036053,-2.943928738 54.402974476,-2.943757868 54.402893845,-2.943725206 54.402487746,-2.94508691 54.402442817,-2.945085877 54.40244342,-2.945008867 54.402398491,-2.945007834 54.402399696,-2.944853814 54.402354767,-2.94485278 54.40235537,-2.94477577 54.40231044,-2.944774738 54.402312851,-2.944466698 54.402178063,-2.9444636 54.402178666,-2.94438659 54.402313618,-2.943218929 54.402232986,-2.943186269 54.402160738,-2.943230825 54.402125276,-2.943168392 54.401865366,-2.944225344 54.401865969,-2.944148335 54.40177611,-2.944146271 54.401774906,-2.944300289 54.401729977,-2.944299256 54.401728169,-2.944530283 54.401773099,-2.944531316 54.401773701,-2.944454307 54.40181863,-2.944455339 54.401818028,-2.944532348 54.401907886,-2.944534414 54.401906078,-2.944765441 54.401861149,-2.944764408 54.401860547,-2.944841418 54.401770688,-2.944839352 54.401771291,-2.944762343 54.401726362,-2.94476131 54.401725157,-2.944915327 54.401635298,-2.944913261 54.401634696,-2.94499027 54.401511546,-2.94464854 54.40141234,-2.944692473 54.401365965,-2.94487626 54.401453773,-2.945140154 54.401453171,-2.945217163 54.401408241,-2.945216129 54.401407036,-2.945370146 54.401362106,-2.945369112 54.4013609,-2.945523129 54.401315971,-2.945522095 54.401314765,-2.945676111 54.401269836,-2.945675077 54.401269233,-2.945752085 54.401224304,-2.945751051 54.40115121,-2.945903413 54.401078479,-2.946009569 54.401032584,-2.946131747 54.400816925,-2.946126782 54.400817528,-2.946049775 54.400592883,-2.946044603 54.400593486,-2.945967597 54.400458699,-2.945964494 54.400459302,-2.945887488 54.400369444,-2.945885419 54.400370047,-2.945808413 54.400280189,-2.945806345 54.400280792,-2.945729339 54.400190934,-2.945727271 54.400191537,-2.945650265 54.400146608,-2.945649232 54.400147211,-2.945572226 54.400057353,-2.945570158 54.400057956,-2.945493152 54.400013027,-2.945492119 54.40001363,-2.945415113 54.399923772,-2.945413046 54.399924375,-2.945336041 54.399879446,-2.945335007 54.399880048,-2.945258002 54.39979019,-2.945255935 54.399790793,-2.94517893 54.399745864,-2.945177896 54.399746467,-2.945100891 54.399656609,-2.945098825 54.399657211,-2.94502182 54.399612282,-2.945020787 54.399612885,-2.944943782 54.399523027,-2.944941716 54.399523629,-2.944864711 54.3994787,-2.944863678 54.399480508,-2.944632664 54.39939065,-2.944630599 54.399391252,-2.944553594 54.399346323,-2.944552562 54.399346926,-2.944475557 54.399257067,-2.944473492 54.39925767,-2.944396488 54.399167811,-2.944394423 54.399168414,-2.944317419 54.399123485,-2.944316387 54.399124087,-2.944239383 54.399079158,-2.944238351 54.399081566,-2.943930335 54.399036637,-2.943929303 54.399038443,-2.943698292 54.398993514,-2.94369726 54.398994116,-2.943620256 54.398949187,-2.943619225 54.398949789,-2.943542221 54.398904859,-2.943541189 54.398905461,-2.943464186 54.398815603,-2.943462123 54.398816205,-2.94338512 54.398771275,-2.943384088 54.398771877,-2.943307085 54.398682019,-2.943305023 54.398683222,-2.943151016 54.398593364,-2.943148954 54.398593965,-2.943071951 54.398549036,-2.94307092 54.398549638,-2.942993917 54.398504708,-2.942992886 54.398505911,-2.942838881 54.398460982,-2.94283785 54.398461583,-2.942760847 54.398416654,-2.942759817 54.398417256,-2.942682814 54.398327397,-2.942680753 54.398327998,-2.94260375 54.398283069,-2.94260272 54.398284272,-2.942448715 54.398194413,-2.942446654 54.398195014,-2.942369652 54.398150085,-2.942368622 54.398150686,-2.94229162 54.397970969,-2.9422875 54.39797157,-2.942210498 54.397926641,-2.942209468 54.397927242,-2.942132466 54.397702596,-2.942127317 54.397703197,-2.942050315 54.39752348,-2.942046196 54.39752288,-2.942123197 54.397298234,-2.942118048 54.397297633,-2.942195049 54.397072987,-2.942189899 54.397072386,-2.942266899 54.396982527,-2.942264839 54.396981926,-2.942341839 54.396936997,-2.942340809 54.396935194,-2.942571809 54.396890265,-2.942570779 54.396889664,-2.942647779 54.396844734,-2.942646748 54.396844133,-2.942723748 54.396754275,-2.942721687 54.39675728,-2.942336689 54.39680221,-2.942337719 54.396803412,-2.94218372 54.396848341,-2.94218475 54.396848942,-2.94210775 54.396804013,-2.94210672 54.396805815,-2.941875721 54.396760886,-2.941874691 54.396763889,-2.941489693 54.39671896,-2.941488664 54.396720161,-2.941334665 54.39676509,-2.941335694 54.39676569,-2.941258694 54.396720761,-2.941257665 54.396721361,-2.941180665 54.39676629,-2.941181694 54.396768091,-2.940950695 54.39681302,-2.940951724 54.39681782,-2.940335726 54.396862749,-2.940336753 54.396867546,-2.939720754 54.396822616,-2.939719727 54.396824414,-2.939488728 54.396554839,-2.939482566 54.39655424,-2.939559565 54.396419452,-2.939556484 54.396418853,-2.939633483 54.396373924,-2.939632456 54.396374523,-2.939555457 54.396284664,-2.939553403 54.396285264,-2.939476405 54.396420051,-2.939479485 54.39642065,-2.939402486 54.396735155,-2.939409674 54.396735754,-2.939332675 54.396780683,-2.939333701 54.396781283,-2.939256702 54.396871141,-2.939258755 54.396872339,-2.939104755 54.396917268,-2.939105782 54.396919065,-2.938874782 54.396963994,-2.938875808 54.396964593,-2.938798808 54.397009522,-2.938799834 54.39701072,-2.938645834 54.397055649,-2.93864686 54.397056248,-2.938569859 54.397101177,-2.938570885 54.397102374,-2.938416885 54.397057445,-2.938415859 54.397058642,-2.938261858 54.397013712,-2.938260833 54.397014311,-2.938183832 54.396969382,-2.938182807 54.39696998,-2.938105807 54.396925051,-2.938104781 54.396926247,-2.937950781 54.396881318,-2.937949756 54.396881916,-2.937872756 54.396836987,-2.937871731 54.396837585,-2.937794731 54.396792656,-2.937793706 54.396793254,-2.937716706 54.396748325,-2.937715681 54.396748923,-2.937638681 54.396703994,-2.937637656 54.396704592,-2.937560657 54.396614733,-2.937558607 54.396615331,-2.937481608 54.396525473,-2.937479558 54.396527266,-2.93724856 54.396482337,-2.937247536 54.396481141,-2.937401534 54.396379187,-2.937799675 54.395739643,-2.939140486 54.395703939,-2.939108866 54.396334258,-2.93779865 54.396436332,-2.93738511 54.396621427,-2.936696213 54.396942979,-2.935794779 54.397303209,-2.934539944 54.397481923,-2.933512011 54.396740792,-2.931723872 54.396905627,-2.931327138 54.396445249,-2.930423359 54.396445487,-2.930392559 54.396262834,-2.929602899 54.39540588,-2.928844215 54.394060583,-2.929645506 54.393791353,-2.933089366 54.394044757,-2.936344867 54.394053146,-2.936422066 54.393972273,-2.936420224 54.393971676,-2.936497219 54.393657171,-2.936490054 54.393657768,-2.93641306 54.393478051,-2.936408966 54.393478648,-2.936331972 54.393388789,-2.936329926 54.393389387,-2.936252932 54.393344457,-2.936251909 54.393345054,-2.936174916 54.393300125,-2.936173892 54.393301319,-2.936019906 54.39325639,-2.936018883 54.393257584,-2.935864896 54.393212654,-2.935863874 54.393214445,-2.935632894 54.393169515,-2.935631872 54.393171305,-2.935400892 54.393126376,-2.93539987 54.393128762,-2.935091898 54.393083833,-2.935090876 54.393086814,-2.934705911 54.393041885,-2.93470489 54.393045461,-2.934242933 54.393000531,-2.934241912 54.393002914,-2.933933941 54.392957985,-2.93393292 54.392960367,-2.933624949 54.393005297,-2.933625969 54.393006487,-2.933471984 54.393051417,-2.933473004 54.393052012,-2.933396011 54.393007083,-2.933394991 54.393007678,-2.933317998 54.392962749,-2.933316978 54.392962153,-2.933393971 54.392917224,-2.933392951 54.392916629,-2.933469943 54.392871699,-2.933468923 54.392871104,-2.933545916 54.392736316,-2.933542855 54.392735721,-2.933619847 54.392690791,-2.933618827 54.392690196,-2.933695819 54.392600337,-2.933693778 54.392599742,-2.93377077 54.392509883,-2.933768729 54.392509287,-2.933845721 54.391835347,-2.933830412 54.391835943,-2.933753421 54.391746084,-2.93375138 54.39174668,-2.93367439 54.391656821,-2.933672349 54.391658012,-2.933518369 54.391613083,-2.933517348 54.391613678,-2.933440358 54.391568749,-2.933439338 54.391570534,-2.933208368 54.391525605,-2.933207348 54.391530365,-2.932591428 54.391575294,-2.932592447 54.391577078,-2.932361476 54.391622008,-2.932362495 54.391622602,-2.932285505 54.391667532,-2.932286524 54.391668721,-2.932132543 54.391758579,-2.932134581 54.391759174,-2.93205759 54.391804579,-2.931997016 54.391429906,-2.931634307 54.390476523,-2.929395058 54.389342076,-2.926150838 54.388976017,-2.925834597 54.388974245,-2.926065553 54.389019175,-2.926066565 54.389016221,-2.926451493 54.389061151,-2.926452505 54.389058196,-2.926837433 54.389103125,-2.926838446 54.388786254,-2.927139296 54.388703016,-2.927445412 54.389052874,-2.927530303 54.389097803,-2.927531316 54.389097212,-2.927608302 54.389052282,-2.927607288 54.389051099,-2.927761259 54.38900617,-2.927760245 54.389004395,-2.927991202 54.388959465,-2.927990188 54.388958873,-2.928067173 54.388913944,-2.928066159 54.388913352,-2.928143144 54.388823493,-2.928141116 54.388822901,-2.928218101 54.388777972,-2.928217086 54.38877738,-2.928294071 54.388687521,-2.928292043 54.388686337,-2.928446012 54.388731266,-2.928447027 54.388730674,-2.928524012 54.388685745,-2.928522997 54.388685152,-2.928599982 54.388595293,-2.928597952 54.388594701,-2.928674937 54.388549772,-2.928673922 54.38854918,-2.928750907 54.388414391,-2.928747862 54.388414984,-2.928670878 54.388325125,-2.928668848 54.388324532,-2.928745832 54.388279603,-2.928744818 54.388279011,-2.928821802 54.388189152,-2.928819772 54.388188559,-2.928896756 54.388098701,-2.928894725 54.388098108,-2.928971709 54.38791839,-2.928967649 54.387917798,-2.929044632 54.38778301,-2.929041587 54.387782417,-2.92911857 54.387737488,-2.929117554 54.387736895,-2.929194537 54.387691966,-2.929193522 54.387688409,-2.92965542 54.387733339,-2.929656436 54.387729781,-2.930118334 54.38777471,-2.93011935 54.38777293,-2.930350299 54.38781786,-2.930351316 54.387816673,-2.930505282 54.387861602,-2.930506299 54.387860415,-2.930660265 54.387905345,-2.930661282 54.38790297,-2.930969216 54.3879479,-2.930970233 54.387944337,-2.931432133 54.387989266,-2.931433151 54.387985701,-2.931895052 54.38803063,-2.93189607 54.388028847,-2.932127021 54.387668223,-2.932272838 54.387623294,-2.932271819 54.387624483,-2.932117854 54.387579553,-2.932116835 54.387580742,-2.93196287 54.387625672,-2.931963888 54.387627454,-2.93173294 54.387672384,-2.931733958 54.387677136,-2.931118095 54.387632206,-2.931117077 54.387633988,-2.930886129 54.387589058,-2.930885112 54.387590839,-2.930654164 54.38754591,-2.930653147 54.387546503,-2.930576164 54.387501574,-2.930575147 54.387503354,-2.9303442 54.387458424,-2.930343183 54.387459018,-2.9302662 54.387414088,-2.930265184 54.387415275,-2.930111219 54.387370345,-2.930110203 54.387372125,-2.929879256 54.387327195,-2.92987824 54.387328974,-2.929647293 54.387284045,-2.929646277 54.387284637,-2.929569295 54.387239708,-2.929568279 54.387240301,-2.929491297 54.387195371,-2.929490281 54.387195964,-2.929413299 54.387151035,-2.929412284 54.387151627,-2.929335302 54.387106698,-2.929334286 54.387107291,-2.929257305 54.387062361,-2.929256289 54.387064139,-2.929025344 54.387019209,-2.929024329 54.387019802,-2.928947347 54.386974872,-2.928946332 54.386975465,-2.92886935 54.386885606,-2.92886732 54.386886198,-2.928790339 54.386706481,-2.92878628 54.386707073,-2.928709298 54.386662143,-2.928708284 54.386662736,-2.928631303 54.386617806,-2.928630288 54.386618398,-2.928553307 54.38648361,-2.928550263 54.386484202,-2.928473282 54.386439273,-2.928472268 54.386440457,-2.928318307 54.386395528,-2.928317292 54.386396119,-2.928240312 54.38635119,-2.928239298 54.386352374,-2.928085337 54.386307444,-2.928084323 54.386308628,-2.927930362 54.386263699,-2.927929348 54.38626429,-2.927852368 54.386219361,-2.927851354 54.386219953,-2.927774373 54.38615859,-2.927572806 54.386140263,-2.927618589 54.386133643,-2.927310466 54.386088713,-2.927309452 54.386089305,-2.927232472 54.386044375,-2.927231459 54.38604674,-2.92692354 54.386001811,-2.926922527 54.386003584,-2.926691587 54.385958654,-2.926690575 54.385961018,-2.926382656 54.385916088,-2.926381643 54.38591727,-2.926227684 54.38587234,-2.926226672 54.385872931,-2.926149692 54.385828002,-2.92614868 54.385829183,-2.925994721 54.385784253,-2.92599371 54.385784844,-2.92591673 54.385739914,-2.925915718 54.385740505,-2.925838739 54.385695575,-2.925837727 54.385795483,-2.92570139 54.385752206,-2.925484836 54.385607487,-2.925604767 54.385608078,-2.925527788 54.385563148,-2.925526777 54.385563738,-2.925449797 54.385518809,-2.925448786 54.385703601,-2.924790809 54.385479189,-2.924754965 54.385480368,-2.924601007 54.385435439,-2.924599997 54.385436029,-2.924523018 54.385391099,-2.924522008 54.385391689,-2.924445029 54.385346759,-2.924444019 54.385347349,-2.924367041 54.385302419,-2.924366031 54.385303009,-2.924289052 54.385258079,-2.924288042 54.385258668,-2.924211064 54.385213739,-2.924210054 54.385214328,-2.924133075 54.385169399,-2.924132066 54.385169988,-2.924055087 54.38499027,-2.924051049 54.384989091,-2.924205005 54.384899232,-2.924202985 54.384900411,-2.924049029 54.384810552,-2.92404701 54.384811731,-2.923893055 54.384766801,-2.923892045 54.38476739,-2.923815068 54.384677531,-2.923813049 54.38467812,-2.923736072 54.384588261,-2.923734053 54.384589439,-2.923580099 54.38454451,-2.92357909 54.384545099,-2.923502112 54.384500169,-2.923501103 54.384500758,-2.923424126 54.384455829,-2.923423117 54.384456417,-2.92334614 54.384501347,-2.923347149 54.384502525,-2.923193195 54.384547454,-2.923194204 54.384548631,-2.923040249 54.38472835,-2.923044283 54.384729527,-2.922890328 54.384685186,-2.922812342 54.384640256,-2.922811334 54.384641433,-2.922657379 54.384596504,-2.922656371 54.384597092,-2.922579394 54.384552162,-2.922578386 54.384553339,-2.922424431 54.38450841,-2.922423424 54.384508998,-2.922346447 54.384464068,-2.922345439 54.384464656,-2.922268462 54.384329868,-2.922265439 54.384330456,-2.922188462 54.384195667,-2.922185439 54.384196255,-2.922108463 54.384106396,-2.922106448 54.384104632,-2.922337377 54.384149561,-2.922338385 54.384148973,-2.922415361 54.384059114,-2.922413345 54.384057938,-2.922567298 54.383968078,-2.922565282 54.383966902,-2.922719234 54.383921972,-2.922718226 54.383922561,-2.92264125 54.383877631,-2.922640242 54.383878219,-2.922563266 54.383743431,-2.922560242 54.383744607,-2.922406291 54.383654748,-2.922404275 54.383655925,-2.922250324 54.383521136,-2.922247301 54.3835229,-2.922016375 54.383433041,-2.922014361 54.383433629,-2.921937386 54.383388699,-2.921936378 54.383389287,-2.921859403 54.383299428,-2.921857389 54.383300016,-2.921780414 54.383255087,-2.921779407 54.383256262,-2.921625457 54.383166403,-2.921623444 54.383166991,-2.921546469 54.382987273,-2.921542442 54.38298786,-2.921465467 54.382853071,-2.921462447 54.382852484,-2.921539421 54.382717695,-2.921536401 54.382717107,-2.921613375 54.381683963,-2.921559428 54.38165959,-2.921220148 54.38156098,-2.921187145 54.381458492,-2.921662154 54.381323703,-2.921659134 54.381323116,-2.921736105 54.381188327,-2.921733084 54.381188915,-2.921656113 54.381054126,-2.921653092 54.381053538,-2.921730063 54.381008609,-2.921729056 54.381008021,-2.921806027 54.380783373,-2.921800992 54.380782785,-2.921877962 54.380692926,-2.921875948 54.380692338,-2.921952918 54.380602479,-2.921950903 54.380601891,-2.922027873 54.380512032,-2.922025859 54.380511444,-2.922102828 54.380107078,-2.922093762 54.38010649,-2.922170731 54.379971701,-2.922167709 54.379971113,-2.922244678 54.379836324,-2.922241655 54.379835736,-2.922318624 54.379611088,-2.922313586 54.3796105,-2.922390554 54.379071345,-2.922378463 54.379071933,-2.922301496 54.378982074,-2.922299481 54.378982662,-2.922222514 54.378802944,-2.922218484 54.378803532,-2.922141518 54.378668743,-2.922138496 54.378669331,-2.92206153 54.378579472,-2.922059515 54.37858006,-2.921982549 54.37853513,-2.921981542 54.378533954,-2.922135474 54.378489025,-2.922134467 54.378490789,-2.921903569 54.378266141,-2.921898534 54.378266729,-2.921821568 54.378042081,-2.921816534 54.378042668,-2.921739569 54.377997739,-2.921738562 54.377997151,-2.921815527 54.377952222,-2.92181452 54.377951634,-2.921891485 54.377906704,-2.921890478 54.377907292,-2.921813513 54.377547855,-2.921805458 54.377547267,-2.921882422 54.377457408,-2.921880408 54.377456232,-2.922034336 54.377411303,-2.922033329 54.377410715,-2.922110292 54.377365785,-2.922109285 54.377365197,-2.922186249 54.377320268,-2.922185242 54.37731968,-2.922262205 54.37722982,-2.922260191 54.377228644,-2.922414118 54.377138785,-2.922412103 54.377135843,-2.922796919 54.377180773,-2.922797927 54.377179007,-2.923028818 54.377089148,-2.923026801 54.377090325,-2.922872875 54.376955537,-2.92286985 54.37695789,-2.922561998 54.37691296,-2.922560991 54.376912372,-2.922637954 54.376867443,-2.922636946 54.376868031,-2.922559983 54.376823101,-2.922558975 54.376822513,-2.922635938 54.376687724,-2.922632915 54.376687136,-2.922709877 54.376597277,-2.922707862 54.3765961,-2.922861786 54.376506241,-2.92285977 54.376505652,-2.922936732 54.376460723,-2.922935724 54.376460134,-2.923012686 54.376549993,-2.923014703 54.376548816,-2.923168627 54.376638675,-2.923170644 54.376637498,-2.923324569 54.376727357,-2.923326586 54.376726179,-2.923480511 54.376816038,-2.923482528 54.376814861,-2.923636454 54.37685979,-2.923637463 54.376859201,-2.923714425 54.376769342,-2.923712407 54.37677052,-2.923558482 54.376680661,-2.923556465 54.376681839,-2.92340254 54.376591979,-2.923400522 54.376593157,-2.923246598 54.376503298,-2.923244581 54.376504475,-2.923090657 54.376414616,-2.92308864 54.376415793,-2.922934716 54.376370863,-2.922933708 54.376370275,-2.92301067 54.376190557,-2.923006637 54.376189379,-2.92316056 54.376054591,-2.923157535 54.376055179,-2.923080574 54.37592039,-2.923077549 54.375919802,-2.92315451 54.375829943,-2.923152494 54.375828765,-2.923306416 54.375783836,-2.923305407 54.375785013,-2.923151486 54.375695154,-2.923149469 54.375695742,-2.923072508 54.375560954,-2.923069484 54.375561542,-2.922992523 54.375471683,-2.922990507 54.375471095,-2.923067467 54.374752221,-2.923051336 54.37475281,-2.922974377 54.37470788,-2.922973369 54.374708468,-2.922896411 54.374663539,-2.922895403 54.37466295,-2.922972361 54.374258584,-2.922963289 54.374259172,-2.922886331 54.374124384,-2.922883307 54.374123795,-2.922960265 54.374033936,-2.922958249 54.374033347,-2.923035206 54.373988418,-2.923034198 54.373989595,-2.922880283 54.374034524,-2.922881291 54.374035113,-2.922804334 54.373900324,-2.92280131 54.373899147,-2.922955225 54.373854217,-2.922954216 54.373855394,-2.922800302 54.373810465,-2.922799294 54.373809876,-2.922876251 54.373764947,-2.922875243 54.373765535,-2.922798286 54.373675676,-2.922796271 54.373676264,-2.922719314 54.373541475,-2.922716291 54.373542064,-2.922639334 54.373407275,-2.922636311 54.373407863,-2.922559355 54.373362934,-2.922558347 54.373363522,-2.922481391 54.373318592,-2.922480383 54.37331918,-2.922403427 54.37340904,-2.922405442 54.373410451,-2.922220747 54.373383846,-2.922173968 54.373320945,-2.922172558 54.373321533,-2.922095602 54.373276603,-2.922094595 54.373277544,-2.921971465 54.373106811,-2.921967639 54.37311022,-2.921521295 54.373128192,-2.921521698 54.373129837,-2.921306221 54.373040448,-2.921242644 54.373042562,-2.920965604 54.373061473,-2.920842877 54.373097417,-2.920843681 54.373098356,-2.920720552 54.373009436,-2.920595412 54.372847924,-2.92056101 54.372686178,-2.920557391 54.372685708,-2.920618955 54.372578112,-2.92058576 54.372561079,-2.92046223 54.372524666,-2.92052299 54.37247075,-2.920521783 54.372453717,-2.920398254 54.372399802,-2.920397048 54.372382534,-2.9203043 54.372329088,-2.920241531 54.372328619,-2.920303094 54.37220305,-2.920269498 54.372059745,-2.920204719 54.372006533,-2.920111169 54.371917143,-2.920047596 54.371935936,-2.919940263 54.371287072,-2.920172044 54.370673447,-2.920496963 54.370187854,-2.920532277 54.370205591,-2.92056346 54.370203008,-2.920902042 54.370219805,-2.921056345 54.37030943,-2.921089137 54.370399994,-2.920998807 54.370471881,-2.921000417 54.370508295,-2.92093966 54.370579947,-2.92097205 54.370598389,-2.920910891 54.370670276,-2.9209125 54.37070575,-2.920974866 54.370849525,-2.920978084 54.370867027,-2.921040048 54.37093868,-2.921072438 54.37093821,-2.921134 54.370991656,-2.921196768 54.371045571,-2.921197975 54.371098547,-2.921322306 54.371260059,-2.921356709 54.371259589,-2.921418271 54.371439072,-2.921453077 54.371438602,-2.921514639 54.371582377,-2.921517859 54.371582612,-2.921487078 54.371726387,-2.921490299 54.371724977,-2.921674986 54.371689033,-2.921674181 54.371669886,-2.921827685 54.371633237,-2.921919223 54.371507434,-2.921916404 54.371507081,-2.921962576 54.370931982,-2.921949688 54.370932452,-2.921888127 54.370806766,-2.921869918 54.370789382,-2.921792564 54.370629751,-2.921511915 54.370575131,-2.921603049 54.370556453,-2.921694988 54.370707333,-2.921944654 54.370625755,-2.922035183 54.370625167,-2.922112135 54.370580237,-2.922111128 54.370579061,-2.92226503 54.370534131,-2.922264023 54.370532955,-2.922417925 54.370488026,-2.922416917 54.370487437,-2.922493868 54.370442508,-2.922492861 54.37044192,-2.922569812 54.37039699,-2.922568804 54.370396402,-2.922645755 54.370351472,-2.922644747 54.370350884,-2.922721698 54.370305954,-2.92272069 54.370305366,-2.922797641 54.370215506,-2.922795626 54.370214918,-2.922872576 54.370169988,-2.922871568 54.3701694,-2.922948518 54.37012447,-2.922947511 54.370123882,-2.923024461 54.370078952,-2.923023453 54.370078364,-2.923100403 54.370033434,-2.923099395 54.370032845,-2.923176345 54.369853127,-2.923172313 54.369852538,-2.923249262 54.369762679,-2.923247246 54.369761502,-2.923401145 54.369806431,-2.923402153 54.369805842,-2.923479103 54.369715983,-2.923477086 54.369715394,-2.923554036 54.369580605,-2.92355101 54.369580017,-2.923627959 54.369472185,-2.923625539 54.369447583,-2.923317138 54.369350739,-2.923053294 54.369317855,-2.922652354 54.369246909,-2.922527624 54.369121694,-2.922447855 54.368950608,-2.922490196 54.368788273,-2.922563518 54.368624644,-2.922806123 54.36855158,-2.922958405 54.368558329,-2.923251006 54.368358519,-2.923523579 54.368138145,-2.924134311 54.367891045,-2.924713652 54.367663803,-2.925047164 54.367517314,-2.92539788 54.367454177,-2.925427243 54.367454767,-2.925350298 54.367364907,-2.925348277 54.367365497,-2.925271332 54.367185779,-2.925267291 54.367186959,-2.925113402 54.367142029,-2.925112392 54.367142619,-2.925035447 54.367097689,-2.925034437 54.367098279,-2.924957492 54.366693912,-2.924948403 54.366694502,-2.924871459 54.366424924,-2.9248654 54.366424334,-2.924942343 54.366334475,-2.924940324 54.366333885,-2.925017267 54.366244026,-2.925015247 54.366242257,-2.925246075 54.366197327,-2.925245065 54.366197917,-2.925168122 54.366152987,-2.925167112 54.366152397,-2.925244055 54.366107468,-2.925243045 54.366108058,-2.925166102 54.365928339,-2.925162061 54.365929047,-2.925069731 54.366018788,-2.925087139 54.366110181,-2.924889108 54.366075888,-2.924672861 54.365943221,-2.92439284 54.365979165,-2.924393647 54.365980933,-2.92416282 54.365936003,-2.924161811 54.365937181,-2.924007926 54.365892252,-2.924006917 54.365892841,-2.923929975 54.365847911,-2.923928966 54.3658485,-2.923852024 54.365758641,-2.923850007 54.36575923,-2.923773065 54.3657143,-2.923772056 54.365714889,-2.923695114 54.36566996,-2.923694106 54.365670548,-2.923617164 54.365580689,-2.923615147 54.365581278,-2.923538205 54.365446489,-2.92353518 54.365447078,-2.923458239 54.365330261,-2.923455617 54.365205282,-2.923345077 54.365178677,-2.923298307 54.365043888,-2.923295283 54.365044477,-2.923218342 54.364999547,-2.923217334 54.365000136,-2.923140394 54.364910276,-2.923138378 54.36491263,-2.922830616 54.364867701,-2.922829608 54.364869465,-2.922598787 54.364824536,-2.92259778 54.364825712,-2.922443899 54.364780782,-2.922442892 54.364781371,-2.922365952 54.364736441,-2.922364945 54.364737029,-2.922288005 54.36464717,-2.922285991 54.364647758,-2.922209051 54.364423109,-2.922204016 54.364423697,-2.922127077 54.364100203,-2.922119828 54.364029021,-2.922025891 54.363939515,-2.921977714 54.363840199,-2.922037051 54.36375034,-2.922035037 54.363750927,-2.921958099 54.363705998,-2.921957092 54.36370541,-2.922034031 54.36366048,-2.922033024 54.363661068,-2.921956086 54.363211771,-2.92194602 54.363211183,-2.922022958 54.362986535,-2.922017925 54.362985947,-2.922094862 54.362851158,-2.922091842 54.36285057,-2.922168778 54.36276071,-2.922166765 54.362760122,-2.922243701 54.362606656,-2.922332601 54.362588214,-2.922393748 54.362444439,-2.922390525 54.362443733,-2.922482848 54.36214555,-2.922691621 54.362108547,-2.922829298 54.362036542,-2.922843073 54.362035954,-2.922920008 54.361901165,-2.922916986 54.361900694,-2.922978534 54.361801966,-2.92296093 54.361630998,-2.922987875 54.361631586,-2.922910941 54.361586657,-2.922909933 54.361587245,-2.922832999 54.361452456,-2.922829976 54.361451868,-2.922906911 54.361227219,-2.922901873 54.361224865,-2.923209607 54.361179935,-2.9232086 54.361178169,-2.9234394 54.361223099,-2.923440408 54.36122251,-2.923517342 54.36126744,-2.92351835 54.361266262,-2.923672217 54.361311192,-2.923673225 54.361310603,-2.923750159 54.361355533,-2.923751168 54.361354355,-2.923905035 54.361399285,-2.923906044 54.361398696,-2.923982978 54.361443625,-2.923983986 54.361442447,-2.924137854 54.361487377,-2.924138863 54.361486788,-2.924215797 54.361531717,-2.924216806 54.361531128,-2.92429374 54.361576058,-2.924294749 54.361574879,-2.924448618 54.361619809,-2.924449627 54.36161863,-2.924603496 54.36166356,-2.924604505 54.36166238,-2.924758374 54.36170731,-2.924759383 54.361705541,-2.924990187 54.361750471,-2.924991196 54.361746342,-2.925529738 54.361791271,-2.925530748 54.361788911,-2.925838487 54.361743981,-2.925837476 54.36174162,-2.926145214 54.361786549,-2.926146225 54.361785959,-2.92622316 54.361696099,-2.926221137 54.361694918,-2.926375006 54.361649989,-2.926373995 54.361648217,-2.926604798 54.361603287,-2.926603786 54.361600924,-2.926911523 54.361555994,-2.926910511 54.361554812,-2.927064379 54.361509882,-2.927063367 54.3615087,-2.927217235 54.361463771,-2.927216223 54.361463179,-2.927293157 54.36141825,-2.927292145 54.361417067,-2.927446013 54.361372138,-2.927445 54.361371546,-2.927521934 54.361326617,-2.927520921 54.361326025,-2.927597855 54.361236166,-2.92759583 54.361235575,-2.927672763 54.361190645,-2.927671751 54.361190054,-2.927748684 54.361145124,-2.927747671 54.361144532,-2.927824605 54.361099603,-2.927823592 54.361099011,-2.927900525 54.361054082,-2.927899512 54.36105349,-2.927976445 54.36100856,-2.927975432 54.361007969,-2.928052365 54.360918109,-2.928050339 54.360917518,-2.928127272 54.360827659,-2.928125246 54.360827067,-2.928202178 54.360782137,-2.928201165 54.360781545,-2.928278098 54.360691686,-2.928276071 54.360691094,-2.928353003 54.360601235,-2.928350977 54.360600643,-2.928427909 54.360510784,-2.928425882 54.360510192,-2.928502814 54.360375403,-2.928499773 54.360374811,-2.928576705 54.360329881,-2.928575692 54.360329289,-2.928652623 54.36023943,-2.928650596 54.360238838,-2.928727527 54.360193908,-2.928726514 54.360193316,-2.928803445 54.360148387,-2.928802431 54.360147202,-2.928956294 54.360102273,-2.92895528 54.360100496,-2.929186074 54.360055566,-2.92918506 54.360054974,-2.929261991 54.360010044,-2.929260977 54.360010636,-2.929184046 54.359920777,-2.929182017 54.359920185,-2.929258948 54.359875255,-2.929257934 54.359874663,-2.929334865 54.359829733,-2.92933385 54.359829141,-2.929410781 54.359784211,-2.929409766 54.359783618,-2.929486697 54.359828548,-2.929487712 54.35982677,-2.929718504 54.3598717,-2.929719519 54.359869328,-2.930027243 54.359913902,-2.930074417 54.359912598,-2.930243665 54.35986755,-2.930258035 54.359866956,-2.930334966 54.359777097,-2.930332935 54.359776504,-2.930409866 54.359731574,-2.93040885 54.359730981,-2.930485781 54.359686052,-2.930484765 54.359686645,-2.930407835 54.359327208,-2.93039971 54.359326615,-2.93047664 54.359281685,-2.930475624 54.359281092,-2.930552554 54.359200337,-2.930535339 54.359146896,-2.930472577 54.359147489,-2.930395647 54.35910256,-2.930394632 54.359103746,-2.930240773 54.359058816,-2.930239757 54.359059409,-2.930162828 54.35901448,-2.930161813 54.359015073,-2.930084883 54.358925213,-2.930082853 54.358925806,-2.930005924 54.358656229,-2.929999833 54.358655636,-2.930076762 54.35843158,-2.929994757 54.357876705,-2.929689847 54.35748488,-2.929219358 54.357360854,-2.928985739 54.357361564,-2.928893428 54.357307649,-2.928892211 54.357272179,-2.928829859 54.357272889,-2.928737548 54.35722796,-2.928736534 54.357165887,-2.928627419 54.356838497,-2.927958361 54.356431057,-2.927179798 54.35624448,-2.92689862 54.355631301,-2.925992352 54.355267126,-2.925430223 54.355178683,-2.92524359 54.355271491,-2.924860997 54.355201372,-2.924628615 54.355040214,-2.924548061 54.354927639,-2.9239916 54.354776409,-2.923788175 54.354373101,-2.923640644 54.354094537,-2.923634394 54.353786895,-2.923904451 54.353471209,-2.924051231 54.353192173,-2.924106513 54.352865497,-2.924514609 54.352740047,-2.924465633 54.352523559,-2.924568475 54.352242755,-2.924854503 54.35189893,-2.925154497 54.351608784,-2.925486464 54.351526849,-2.925623094 54.351570953,-2.925731786 54.351678548,-2.925764976 54.351758359,-2.925905243 54.351027664,-2.926258064 54.350789422,-2.926852738 54.350292477,-2.927195413 54.349904071,-2.927448213 54.348823747,-2.927685421 54.347914274,-2.927911081 54.345287643,-2.929374854 54.345531426,-2.930395667 54.345171988,-2.930387546 54.345171395,-2.93046445 54.345081535,-2.930462419 54.345080942,-2.930539323 54.345036013,-2.930538307 54.345034826,-2.930692114 54.344989896,-2.930691098 54.344986336,-2.931152517 54.345031266,-2.931153533 54.345030079,-2.931307339 54.345119938,-2.931309371 54.345119345,-2.931386275 54.345388923,-2.931392372 54.345388329,-2.931469275 54.345523118,-2.931472324 54.345522525,-2.931549228 54.345612384,-2.931551261 54.34561179,-2.931628165 54.34570165,-2.931630198 54.345700462,-2.931784007 54.345655532,-2.93178299 54.345654938,-2.931859895 54.345699868,-2.931860911 54.345699274,-2.931937816 54.345654344,-2.931936799 54.345653156,-2.932090608 54.345698086,-2.932091624 54.345695114,-2.932476146 54.345650185,-2.932475129 54.345648996,-2.932628938 54.345604066,-2.93262792 54.345602877,-2.932781728 54.345513018,-2.932779693 54.345512423,-2.932856597 54.345242845,-2.932850491 54.34524344,-2.932773587 54.345108651,-2.932770534 54.345109245,-2.932693631 54.345064315,-2.932692613 54.34506491,-2.93261571 54.344930121,-2.932612658 54.344930715,-2.932535755 54.344840856,-2.93253372 54.34484145,-2.932456817 54.344616802,-2.932451731 54.344616207,-2.932528633 54.344481418,-2.932525581 54.344480824,-2.932602483 54.344435894,-2.932601465 54.344436488,-2.932524563 54.344301699,-2.932521511 54.344302294,-2.93244461 54.344257364,-2.932443592 54.344257958,-2.932366691 54.344213029,-2.932365673 54.344214811,-2.932134969 54.344124952,-2.932132935 54.34412614,-2.931979132 54.34408121,-2.931978115 54.344082992,-2.931747411 54.344038062,-2.931746394 54.344040438,-2.931438789 54.344085367,-2.931439805 54.344087148,-2.931209101 54.344132078,-2.931210117 54.344133265,-2.931056314 54.344178195,-2.93105733 54.344179382,-2.930903527 54.344224311,-2.930904542 54.344225498,-2.930750739 54.344405217,-2.930754801 54.344404624,-2.930831703 54.344494483,-2.930833734 54.344496263,-2.930603027 54.344451333,-2.930602012 54.344452519,-2.930448208 54.344407589,-2.930447193 54.344409368,-2.930216487 54.34409486,-2.930209383 54.344094267,-2.930286284 54.344004408,-2.930284254 54.344003815,-2.930361156 54.343913955,-2.930359126 54.343913362,-2.930436027 54.343823503,-2.930433997 54.34382291,-2.930510897 54.34377798,-2.930509882 54.343777387,-2.930586783 54.343732457,-2.930585768 54.343731864,-2.930662669 54.343686934,-2.930661653 54.343686341,-2.930738554 54.343641411,-2.930737539 54.343640818,-2.930814439 54.343595888,-2.930813424 54.343595295,-2.930890324 54.343550365,-2.930889309 54.343549772,-2.930966209 54.343504842,-2.930965193 54.343504249,-2.931042094 54.343459319,-2.931041078 54.343458726,-2.931117978 54.343368866,-2.931115947 54.343368273,-2.931192847 54.343233483,-2.931189799 54.342151964,-2.931580667 54.340609137,-2.933514626 54.340519278,-2.933512589 54.340519873,-2.933435694 54.340430013,-2.933433658 54.340430608,-2.933356763 54.340340749,-2.933354727 54.340341343,-2.933277833 54.340296414,-2.933276815 54.340297009,-2.933199921 54.340162219,-2.933196867 54.340162814,-2.933119973 54.340072955,-2.933117937 54.340073549,-2.933041043 54.34002862,-2.933040025 54.340029809,-2.932886238 54.339984879,-2.93288522 54.339986068,-2.932731433 54.339896209,-2.932729398 54.339896803,-2.932652504 54.339762014,-2.932649452 54.339762608,-2.932572559 54.339582889,-2.93256849 54.339583484,-2.932491597 54.339313905,-2.932485494 54.339313311,-2.932562387 54.339268381,-2.93256137 54.339267787,-2.932638262 54.339177927,-2.932636227 54.339177333,-2.93271312 54.339087473,-2.932711085 54.339086879,-2.932787977 54.339041949,-2.932786959 54.339041355,-2.932863851 54.338996425,-2.932862834 54.33899583,-2.932939726 54.338950901,-2.932938708 54.338949711,-2.933092492 54.338904782,-2.933091474 54.338904187,-2.933168366 54.338859257,-2.933167348 54.338858662,-2.933244239 54.338768803,-2.933242203 54.338768208,-2.933319095 54.338723278,-2.933318077 54.338722683,-2.933394968 54.338453105,-2.93338886 54.33845251,-2.93346575 54.338317721,-2.933462696 54.338318316,-2.933385805 54.338273386,-2.933384787 54.338272791,-2.933461678 54.338138002,-2.933458623 54.338137407,-2.933535514 54.338092477,-2.933534495 54.338091882,-2.933611385 54.338046952,-2.933610367 54.338046357,-2.933687257 54.337956498,-2.93368522 54.337955903,-2.93376211 54.337866043,-2.933760073 54.337865448,-2.933836963 54.337820518,-2.933835945 54.337821709,-2.933682165 54.337776779,-2.933681147 54.337774993,-2.933911816 54.337685134,-2.933909778 54.337683943,-2.934063557 54.337639014,-2.934062538 54.337638418,-2.934139428 54.337593489,-2.934138409 54.337594084,-2.93406152 54.337549154,-2.934060501 54.337547963,-2.934214279 54.337503034,-2.93421326 54.337503629,-2.934136371 54.33741377,-2.934134333 54.33741496,-2.933980556 54.33737003,-2.933979537 54.337371816,-2.93374887 54.337237027,-2.933745815 54.337236432,-2.933822704 54.337011783,-2.933817611 54.337011188,-2.933894499 54.336921328,-2.933892462 54.336920733,-2.93396935 54.336785944,-2.933966294 54.336785349,-2.934043182 54.336695489,-2.934041144 54.336694894,-2.934118032 54.336605034,-2.934115994 54.336604439,-2.934192882 54.336559509,-2.934191863 54.336558914,-2.93426875 54.336513984,-2.934267731 54.336513388,-2.934344618 54.336423529,-2.93434258 54.336422933,-2.934419467 54.336198285,-2.934414372 54.33619888,-2.934337485 54.335884372,-2.934330352 54.335884967,-2.934253466 54.335660319,-2.934248371 54.335659723,-2.934325257 54.335030707,-2.934310991 54.335030111,-2.934387876 54.334805463,-2.93438278 54.333897762,-2.934377573 54.333314628,-2.934241316 54.332107119,-2.933491147 54.331841527,-2.934131022 54.3315607,-2.934416843 54.331253152,-2.934671297 54.330417577,-2.934636965 54.329623954,-2.934988025 54.328836572,-2.935692891 54.328547113,-2.935932353 54.328269622,-2.935787657 54.327827044,-2.936069768 54.327266334,-2.936518332 54.326631825,-2.937211229 54.326153538,-2.937461749 54.325368892,-2.937812924 54.324844479,-2.938216137 54.324673267,-2.938273742 54.324320782,-2.938527103 54.323634499,-2.938941985 54.322190401,-2.939723951 54.321604637,-2.939925841 54.321504474,-2.940092677 54.321297557,-2.940118706 54.321187689,-2.940377566 54.320029573,-2.941365829 54.319162121,-2.941960966 54.318390388,-2.942957975 54.318822523,-2.944013262 54.318821922,-2.944090116 54.318687132,-2.944087029 54.318686531,-2.944163883 54.318641601,-2.944162854 54.318640999,-2.944239707 54.31855114,-2.944237649 54.318550538,-2.944314502 54.318505609,-2.944313473 54.318505007,-2.944390327 54.318460077,-2.944389297 54.318459475,-2.944466151 54.318414546,-2.944465121 54.318413944,-2.944541974 54.318324084,-2.944539915 54.318323482,-2.944616769 54.318233623,-2.944614709 54.318232419,-2.944768415 54.31809763,-2.944765326 54.318097028,-2.944842179 54.318007168,-2.944840119 54.318005964,-2.944993824 54.317961034,-2.944992794 54.317960432,-2.945069646 54.317870573,-2.945067586 54.317871175,-2.944990734 54.317691456,-2.944986614 54.317690854,-2.945063466 54.317645924,-2.945062436 54.317644719,-2.94521614 54.31755486,-2.945214079 54.317553655,-2.945367783 54.317463796,-2.945365722 54.317465,-2.945212019 54.317375141,-2.945209959 54.317374539,-2.94528681 54.317284679,-2.945284749 54.317284077,-2.9453616 54.317239147,-2.94536057 54.317237942,-2.945514272 54.317193012,-2.945513242 54.317120161,-2.945634554 54.316830562,-2.945889249 54.316785632,-2.945888218 54.31678744,-2.945657668 54.31669758,-2.945655606 54.316696978,-2.945732456 54.316517259,-2.945728334 54.316516656,-2.945805183 54.316381867,-2.945802091 54.316381264,-2.94587894 54.316291405,-2.945876878 54.316290802,-2.945953728 54.316245872,-2.945952697 54.316245269,-2.946029546 54.31620034,-2.946028515 54.316197928,-2.946335911 54.316152998,-2.94633488 54.316152395,-2.946411729 54.316197325,-2.94641276 54.316196722,-2.946489609 54.316241652,-2.946490641 54.316241049,-2.94656749 54.316194309,-2.946797005 54.316194913,-2.946720156 54.316105053,-2.946718093 54.31610626,-2.946564395 54.31606133,-2.946563364 54.316061933,-2.946486515 54.316017003,-2.946485483 54.3160164,-2.946562332 54.315926541,-2.946560269 54.315925937,-2.946637117 54.315836078,-2.946635054 54.315835475,-2.946711902 54.315655756,-2.946707775 54.315655153,-2.946784623 54.315475434,-2.946780496 54.31547483,-2.946857344 54.315384971,-2.94685528 54.315383764,-2.947008975 54.315338834,-2.947007943 54.315337024,-2.947238485 54.314887727,-2.947228162 54.314888934,-2.947074469 54.314754144,-2.947071373 54.314754748,-2.946994527 54.314709818,-2.946993494 54.314710421,-2.946916648 54.314665492,-2.946915616 54.314666095,-2.94683877 54.314621165,-2.946837738 54.314621769,-2.946760892 54.314576839,-2.94675986 54.314577442,-2.946683014 54.314532512,-2.946681983 54.314533115,-2.946605137 54.314398326,-2.946602042 54.314398929,-2.946525196 54.313570831,-2.945537775 54.313427176,-2.945519109 54.313363672,-2.94559451 54.31372438,-2.946586569 54.313589591,-2.946583474 54.313588987,-2.946660318 54.313499128,-2.946658255 54.313498525,-2.946735099 54.313408665,-2.946733036 54.313407459,-2.946886723 54.313362529,-2.946885692 54.313361926,-2.946962535 54.313316996,-2.946961503 54.313315789,-2.947115191 54.313270859,-2.947114159 54.313269049,-2.947344689 54.313358908,-2.947346754 54.313357701,-2.947500442 54.313222912,-2.947497344 54.313222308,-2.947574187 54.31277301,-2.947563862 54.312773614,-2.947487019 54.312638825,-2.947483922 54.312638221,-2.947560764 54.312548362,-2.947558699 54.312548965,-2.947481857 54.312369246,-2.947477727 54.31236985,-2.947400885 54.312235061,-2.947397788 54.312235664,-2.947320946 54.312145805,-2.947318882 54.312146408,-2.94724204 54.312101479,-2.947241008 54.312099668,-2.947471532 54.312054738,-2.9474705 54.312053531,-2.947624182 54.311918741,-2.947621085 54.311921156,-2.94731372 54.311876226,-2.947312688 54.311877433,-2.947159006 54.311787574,-2.947156942 54.31178697,-2.947233783 54.311742041,-2.947232751 54.311741437,-2.947309591 54.311696507,-2.947308559 54.311695904,-2.9473854 54.311516185,-2.94738127 54.311515581,-2.947458111 54.311425721,-2.947456046 54.311425118,-2.947532886 54.311380188,-2.947531854 54.311379584,-2.947608694 54.311334655,-2.947607661 54.311334051,-2.947684501 54.311199261,-2.947681403 54.311198658,-2.947758243 54.311108798,-2.947756177 54.311108194,-2.947833017 54.310973405,-2.947829919 54.310974009,-2.947753079 54.31079429,-2.947748948 54.310794894,-2.94767211 54.310705034,-2.947670044 54.310705638,-2.947593206 54.310480989,-2.947588043 54.310481593,-2.947511205 54.310301874,-2.947507075 54.310300062,-2.947737589 54.310255133,-2.947736556 54.310254529,-2.947813394 54.310209599,-2.947812362 54.310208391,-2.947966037 54.310163461,-2.947965004 54.310162857,-2.948041842 54.310117928,-2.948040809 54.310117324,-2.948117647 54.310162253,-2.94811868 54.310161649,-2.948195518 54.31002686,-2.948192418 54.310026256,-2.948269256 54.309981326,-2.948268223 54.309980722,-2.94834506 54.309935792,-2.948344027 54.309934584,-2.948497701 54.309889654,-2.948496668 54.30988905,-2.948573505 54.30984412,-2.948572472 54.309841702,-2.94887982 54.309751842,-2.948877753 54.30894492,-2.948628637 54.308945525,-2.948551801 54.308990454,-2.948552835 54.308991663,-2.948399164 54.308497435,-2.948387797 54.308496227,-2.948541466 54.308406367,-2.948539399 54.308405763,-2.948616234 54.308360833,-2.9486152 54.308359624,-2.948768869 54.308269765,-2.948766802 54.30826916,-2.948843636 54.308179301,-2.948841568 54.308178092,-2.948995237 54.308133162,-2.948994203 54.308133766,-2.948917369 54.308088837,-2.948916335 54.308088232,-2.948993169 54.308043302,-2.948992135 54.308042698,-2.949068969 54.307907908,-2.949065866 54.307907304,-2.9491427 54.307592795,-2.949135461 54.307592191,-2.949212294 54.307502331,-2.949210226 54.307501726,-2.949287059 54.307366937,-2.949283956 54.307366332,-2.949360789 54.307276473,-2.94935872 54.307275868,-2.949435552 54.307230938,-2.949434518 54.307230333,-2.94951135 54.307140474,-2.949509281 54.307139869,-2.949586113 54.307050009,-2.949584044 54.307049404,-2.949660876 54.306824755,-2.949655703 54.30682415,-2.949732535 54.30677922,-2.9497315 54.30677801,-2.949885163 54.30673308,-2.949884128 54.306732475,-2.949960959 54.306687545,-2.949959924 54.306688151,-2.949883093 54.306643221,-2.949882058 54.306643826,-2.949805227 54.306509037,-2.949802123 54.306509642,-2.949725292 54.306464712,-2.949724257 54.306464107,-2.949801088 54.306329318,-2.949797984 54.306328713,-2.949874814 54.306283783,-2.94987378 54.306283178,-2.94995061 54.306238248,-2.949949575 54.306238853,-2.949872745 54.306059134,-2.949868605 54.306059739,-2.949791775 54.306014809,-2.949790741 54.306014204,-2.949867571 54.305879415,-2.949864466 54.30587881,-2.949941296 54.305797452,-2.950000897 54.305474079,-2.949978079 54.305403643,-2.949792034 54.305224892,-2.94966497 54.305135154,-2.949647535 54.305080391,-2.949753853 54.304755565,-2.949915425 54.30475496,-2.949992252 54.304530311,-2.949987078 54.304529706,-2.950063905 54.304439846,-2.950061835 54.304438636,-2.950215489 54.304348776,-2.950213419 54.304169662,-2.950132452 54.304170268,-2.950055625 54.304080408,-2.950053555 54.304081013,-2.949976729 54.304036084,-2.949975694 54.304036689,-2.949898868 54.303991759,-2.949897833 54.303992364,-2.949821007 54.303947434,-2.949819972 54.303948039,-2.949743146 54.30376832,-2.949739008 54.303767715,-2.949815833 54.303632926,-2.949812729 54.303630505,-2.950120032 54.30358618,-2.950042171 54.303588601,-2.949734869 54.303543671,-2.949733835 54.303544276,-2.949657009 54.303499347,-2.949655975 54.303499952,-2.949579149 54.303455022,-2.949578115 54.303455627,-2.94950129 54.303410697,-2.949500255 54.303411302,-2.94942343 54.303366372,-2.949422396 54.303366977,-2.949345571 54.303322047,-2.949344537 54.303322652,-2.949267712 54.303232792,-2.949265644 54.303232187,-2.949342468 54.303187258,-2.949341434 54.303187862,-2.94926461 54.303008143,-2.949260473 54.303007538,-2.949337298 54.302962609,-2.949336264 54.302962004,-2.949413088 54.302917074,-2.949412054 54.302915864,-2.949565702 54.302960794,-2.949566737 54.302959584,-2.949720385 54.303004514,-2.94972142 54.302931053,-2.949919508 54.3029568,-2.950073777 54.302947209,-2.950150394 54.302821769,-2.950101402 54.302822979,-2.949947754 54.302778049,-2.949946719 54.302778655,-2.949869895 54.302733725,-2.94986886 54.302736145,-2.949561565 54.302556426,-2.949557427 54.302560054,-2.949096486 54.302649914,-2.949098554 54.302651123,-2.948944906 54.302381544,-2.948938704 54.302382149,-2.948861881 54.3021575,-2.948856713 54.302156895,-2.948933536 54.301797457,-2.948925267 54.301798061,-2.948848444 54.301708202,-2.948846377 54.301707597,-2.948923199 54.301617738,-2.948921132 54.301618342,-2.94884431 54.301528482,-2.948842243 54.301529087,-2.948765422 54.301439227,-2.948763355 54.301439832,-2.948686533 54.301349972,-2.948684466 54.301350576,-2.948607645 54.301215787,-2.948604545 54.301216391,-2.948527724 54.301081602,-2.948524625 54.301080997,-2.948601446 54.300946208,-2.948598346 54.300944395,-2.948828808 54.300854535,-2.948826741 54.300852722,-2.949057202 54.300807792,-2.949056168 54.300807187,-2.949132988 54.300717328,-2.949130921 54.300717932,-2.949054101 54.300673002,-2.949053067 54.300673607,-2.948976247 54.300628677,-2.948975213 54.300629282,-2.948898393 54.300539422,-2.948896326 54.300538213,-2.949049966 54.300493283,-2.949048932 54.300494492,-2.948895293 54.300000264,-2.948883924 54.299999055,-2.949037561 54.299954125,-2.949036528 54.299955334,-2.94888289 54.299865475,-2.948880823 54.299866079,-2.948804005 54.299237061,-2.948789537 54.299237666,-2.948712719 54.299192736,-2.948711686 54.299192131,-2.948788504 54.298742833,-2.94877817 54.298742229,-2.948854986 54.298697299,-2.948853953 54.298697903,-2.948777136 54.298652973,-2.948776103 54.298652369,-2.948852919 54.298562509,-2.948850853 54.2985613,-2.949004485 54.298471441,-2.949002418 54.298469022,-2.949309682 54.298153304,-2.949456075 54.298154513,-2.949302444 54.297974794,-2.949298309 54.297974189,-2.949375124 54.29770461,-2.94936892 54.297705215,-2.949292105 54.297660285,-2.949291072 54.29766089,-2.949214257 54.29757103,-2.949212189 54.297571635,-2.949135375 54.297481775,-2.949133308 54.29748238,-2.949056494 54.29734759,-2.949053393 54.297346985,-2.949130207 54.297302056,-2.949129173 54.297301451,-2.949205987 54.297256521,-2.949204953 54.297257126,-2.949128139 54.297212196,-2.949127106 54.2972128,-2.949050292 54.297167871,-2.949049258 54.297168475,-2.948972445 54.297123545,-2.948971411 54.29712415,-2.948894598 54.29707922,-2.948893564 54.297079824,-2.948816751 54.297034895,-2.948815718 54.297036103,-2.948662091 54.296946244,-2.948660025 54.296947452,-2.948506399 54.296857592,-2.948504333 54.296858197,-2.94842752 54.296813267,-2.948426487 54.296813871,-2.948349674 54.296768941,-2.948348641 54.296769545,-2.948271828 54.296634756,-2.94826873 54.29663536,-2.948191917 54.29659043,-2.948190885 54.296591034,-2.948114072 54.296456244,-2.948110975 54.296455036,-2.948264599 54.296320247,-2.948261501 54.296320851,-2.948184689 54.296230991,-2.948182624 54.296231595,-2.948105812 54.296141735,-2.948103747 54.296139319,-2.948410993 54.295959599,-2.948406862 54.295889645,-2.948159414 54.295809375,-2.948080744 54.295675068,-2.948016198 54.295547936,-2.94818229 54.295513925,-2.947935671 54.295379135,-2.947932574 54.295378531,-2.948009384 54.295108952,-2.94800319 54.295108348,-2.94808 54.294928629,-2.94807587 54.294928025,-2.948152679 54.294658445,-2.948146484 54.294657841,-2.948223293 54.294433192,-2.94821813 54.294432588,-2.948294939 54.294252868,-2.948290808 54.294252264,-2.948367616 54.294207334,-2.948366583 54.294207938,-2.948289775 54.294163009,-2.948288743 54.294161196,-2.948519167 54.294071336,-2.948517101 54.294072545,-2.948363485 54.294027615,-2.948362452 54.294028219,-2.948285645 54.29375864,-2.948279449 54.293758036,-2.948356256 54.293668176,-2.948354191 54.293666726,-2.948538528 54.293577108,-2.948505739 54.293577712,-2.948428932 54.293397992,-2.948424801 54.293398597,-2.948347995 54.293308737,-2.948345929 54.293309341,-2.948269123 54.293264411,-2.94826809 54.293265015,-2.948191284 54.293175155,-2.948189219 54.293175759,-2.948112413 54.293130829,-2.94811138 54.293131433,-2.948034574 54.293086503,-2.948033542 54.293087107,-2.947956736 54.292997247,-2.947954672 54.292997851,-2.947877866 54.292952921,-2.947876834 54.292953525,-2.947800028 54.292908595,-2.947798996 54.292909199,-2.94772219 54.292864269,-2.947721158 54.292864873,-2.947644353 54.292819943,-2.947643321 54.292820546,-2.947566515 54.292730687,-2.947564452 54.292731894,-2.947410841 54.292686964,-2.947409809 54.292687567,-2.947333004 54.292642637,-2.947331972 54.292643241,-2.947255167 54.292418591,-2.94725001 54.292419195,-2.947173205 54.292374265,-2.947172174 54.292373661,-2.947248978 54.292238872,-2.947245884 54.292238268,-2.947322689 54.292148408,-2.947320626 54.292147805,-2.94739743 54.292057945,-2.947395367 54.292056738,-2.947548975 54.292011808,-2.947547943 54.292012412,-2.947471139 54.291877622,-2.947468044 54.291877018,-2.947544848 54.291787159,-2.947542784 54.291786555,-2.947619588 54.291741625,-2.947618556 54.291741021,-2.947695359 54.291696092,-2.947694328 54.291694884,-2.947847934 54.291515164,-2.947843806 54.291514561,-2.947920609 54.291469631,-2.947919577 54.291469027,-2.94799638 54.291424097,-2.947995348 54.291423493,-2.948072151 54.291378563,-2.948071118 54.291377959,-2.948147921 54.2912881,-2.948145856 54.291287496,-2.948222659 54.291242566,-2.948221627 54.291241962,-2.948298429 54.291152102,-2.948296364 54.291151498,-2.948373166 54.290971778,-2.948369036 54.290971174,-2.948445838 54.290881314,-2.948443773 54.29088071,-2.948520574 54.29079085,-2.948518509 54.290791454,-2.948441707 54.290746525,-2.948440675 54.29074592,-2.948517476 54.290476341,-2.94851128 54.290476945,-2.948434478 54.290342155,-2.94843138 54.290342759,-2.94835458 54.29029783,-2.948353547 54.290298434,-2.948276746 54.290163644,-2.948273649 54.29016304,-2.948350449 54.28989346,-2.948344254 54.289666998,-2.94856949 54.289666394,-2.94864629 54.289531604,-2.948643191 54.289531,-2.948719991 54.28948607,-2.948718958 54.289485466,-2.948795757 54.289350676,-2.948792658 54.289350072,-2.948869457 54.289260212,-2.948867391 54.289259608,-2.94894419 54.289214678,-2.948943156 54.289215282,-2.948866358 54.289170352,-2.948865324 54.289169748,-2.948942123 54.288810309,-2.948933858 54.288809704,-2.949010656 54.288719844,-2.949008589 54.28871924,-2.949085387 54.28862938,-2.949083321 54.288628775,-2.949160118 54.288583845,-2.949159085 54.288583241,-2.949235883 54.288493381,-2.949233816 54.288492776,-2.949310613 54.288447846,-2.94930958 54.288448451,-2.949232782 54.288403521,-2.949231749 54.288401707,-2.94946214 54.288356777,-2.949461107 54.288357987,-2.949307512 54.288088407,-2.949301311 54.288085988,-2.949608498 54.288041058,-2.949607464 54.288040453,-2.949684261 54.287950594,-2.949682193 54.2879489,-2.949897223 54.287632454,-2.95013573 54.287634269,-2.949905343 54.287544409,-2.949903274 54.287543804,-2.94998007 54.287498875,-2.949979036 54.287498269,-2.950055831 54.28740841,-2.950053763 54.287407804,-2.950130558 54.287362875,-2.950129524 54.287396761,-2.950391455 54.287324752,-2.950405159 54.287261123,-2.950495865 54.287135804,-2.950431531 54.287045944,-2.950429462 54.287045339,-2.950506256 54.286955479,-2.950504187 54.286954873,-2.950580981 54.286730224,-2.950575807 54.286548082,-2.950878843 54.286440129,-2.950891717 54.286188643,-2.950870561 54.28619167,-2.950486596 54.286011951,-2.950482457 54.286011345,-2.95055925 54.285921486,-2.95055718 54.28592088,-2.950633973 54.28587595,-2.950632938 54.285865269,-2.95084775 54.285712022,-2.950905664 54.285461626,-2.950746284 54.285272436,-2.95080337 54.284933027,-2.950534415 54.28461876,-2.950496456 54.284441825,-2.950139084 54.284217175,-2.950133913 54.28421657,-2.950210702 54.28417164,-2.950209668 54.284171035,-2.950286457 54.284126105,-2.950285423 54.284122472,-2.950746158 54.284077542,-2.950745123 54.284076937,-2.950821912 54.284032007,-2.950820877 54.28403564,-2.950360143 54.28390085,-2.950357039 54.283900244,-2.950433828 54.283855315,-2.950432794 54.283854709,-2.950509582 54.283809779,-2.950508548 54.283810385,-2.950431759 54.283720525,-2.95042969 54.28372113,-2.950352901 54.2836762,-2.950351867 54.283674384,-2.950582232 54.283449734,-2.950577058 54.28345034,-2.95050027 54.28336048,-2.950498201 54.283359875,-2.950574989 54.283270015,-2.950572919 54.283269409,-2.950649707 54.283134619,-2.950646603 54.283134014,-2.95072339 54.283089084,-2.950722355 54.283088478,-2.950799143 54.282998619,-2.950797073 54.282999224,-2.950720286 54.282864434,-2.950717181 54.282863829,-2.950793968 54.282773969,-2.950791898 54.282773363,-2.950868685 54.282503784,-2.950862475 54.282503178,-2.950939261 54.282413318,-2.950937191 54.282412713,-2.951013977 54.282322853,-2.951011906 54.282322247,-2.951088692 54.282007737,-2.951081446 54.282007132,-2.951158231 54.281872342,-2.951155125 54.281872948,-2.95107834 54.281828018,-2.951077305 54.281828623,-2.95100052 54.281693834,-2.950997414 54.281694439,-2.95092063 54.281649509,-2.950919595 54.281650115,-2.95084281 54.281560255,-2.95084074 54.281559649,-2.950917525 54.281514719,-2.95091649 54.281515325,-2.950839705 54.281425465,-2.950837635 54.28142486,-2.95091442 54.28137993,-2.950913385 54.281379324,-2.950990169 54.281109744,-2.950983958 54.281109139,-2.951060742 54.280884489,-2.951055566 54.280885095,-2.950978783 54.280615515,-2.950972572 54.280616726,-2.950819007 54.280481936,-2.950815902 54.280482542,-2.95073912 54.280392682,-2.95073705 54.280393287,-2.950660268 54.280258498,-2.950657164 54.280259103,-2.950580382 54.280214173,-2.950579347 54.280214778,-2.950502565 54.280169849,-2.950501531 54.280170454,-2.950424749 54.279981869,-2.950405048 54.279977873,-2.950911806 54.279931005,-2.951156472 54.279849646,-2.951216033 54.27974036,-2.951397823 54.279676004,-2.951580648 54.279612738,-2.951625266 54.279487541,-2.951545586 54.279235812,-2.951555142 54.279146194,-2.951522359 54.279038362,-2.951519874 54.27884164,-2.95139247 54.278696774,-2.951527359 54.278624037,-2.951633193 54.278551664,-2.951692959 54.278426103,-2.951659347 54.278399509,-2.951612659 54.278400963,-2.95142839 54.278429375,-2.951244743 54.278492762,-2.951184769 54.278475274,-2.951122932 54.278503565,-2.95095464 54.278467984,-2.950907744 54.278252078,-2.950933488 54.278127001,-2.950838457 54.278019532,-2.950789906 54.277786139,-2.950753815 54.277569627,-2.950856336 54.277479283,-2.950915688 54.277351541,-2.951158476 54.277242982,-2.951248123 54.277081234,-2.951244396 54.277008982,-2.951288806 54.276865206,-2.951285493 54.276702853,-2.951358541 54.276576322,-2.951447773 54.276485735,-2.951537832 54.276376448,-2.951719606 54.276304197,-2.951764014 54.276178393,-2.951761114 54.276105777,-2.951851586 54.276069833,-2.951850757 54.275870686,-2.952030457 54.275834742,-2.952029628 54.275734805,-2.952165541 54.275635474,-2.952224681 54.275526308,-2.952391095 54.275399412,-2.952526384 54.275217751,-2.952767911 54.275144892,-2.952889088 54.27499966,-2.953070023 54.274854912,-2.953189539 54.274548903,-2.953243903 54.274458557,-2.953303246 54.274313567,-2.953453468 54.274277138,-2.953514055 54.274113932,-2.953694569 54.274005371,-2.953784203 54.273896568,-2.953904544 54.273840707,-2.954148963 54.273750361,-2.954208302 54.273715025,-2.954130702 54.273624558,-2.954205395 54.273587884,-2.954296688 54.273443379,-2.954385488 54.273317575,-2.95438258 54.273173313,-2.954440672 54.273064752,-2.954530302 54.272919274,-2.95474193 54.272828684,-2.954831974 54.272666207,-2.954920355 54.272558253,-2.954933215 54.272305794,-2.95503487 54.27208867,-2.955214123 54.271889275,-2.955424495 54.271891465,-2.955148136 54.271792619,-2.95514585 54.271766148,-2.955083813 54.270561485,-2.957390003 54.269675247,-2.9580758 54.269392532,-2.958591317 54.269142948,-2.959460764 54.269179503,-2.959384837 54.269314293,-2.959387969 54.269314904,-2.959311207 54.269584483,-2.959317469 54.269585094,-2.959240707 54.269674954,-2.959242795 54.269675565,-2.959166032 54.269720495,-2.959167076 54.269721106,-2.959090314 54.269766036,-2.959091357 54.269767257,-2.958937832 54.269857117,-2.958939919 54.269857728,-2.958863156 54.269902657,-2.9588642 54.269903268,-2.958787437 54.269948198,-2.95878848 54.269948808,-2.958711717 54.269993738,-2.95871276 54.269994959,-2.958559235 54.270084819,-2.958561321 54.27008543,-2.958484558 54.27013036,-2.958485601 54.27013097,-2.958408837 54.2701759,-2.95840988 54.27017651,-2.958333117 54.2703113,-2.958336245 54.27031191,-2.958259482 54.270716279,-2.958268865 54.27071689,-2.958192101 54.27080675,-2.958194186 54.270806139,-2.95827095 54.270851069,-2.958271993 54.270849849,-2.958425522 54.270894779,-2.958426564 54.270894168,-2.958503329 54.270984028,-2.958505415 54.270983418,-2.958582179 54.271028348,-2.958583222 54.271026516,-2.958813517 54.271071446,-2.95881456 54.271070835,-2.958891325 54.271115765,-2.958892368 54.271114544,-2.959045898 54.271159474,-2.959046941 54.271158863,-2.959123706 54.271203793,-2.95912475 54.271202571,-2.95927828 54.271247501,-2.959279323 54.27124689,-2.959356089 54.27129182,-2.959357132 54.271290598,-2.959510663 54.271335528,-2.959511707 54.271334306,-2.959665237 54.271379236,-2.959666281 54.271378625,-2.959743047 54.271423555,-2.959744091 54.271422943,-2.959820856 54.271467873,-2.959821901 54.271466651,-2.959975432 54.271511581,-2.959976476 54.271510969,-2.960053242 54.271555899,-2.960054287 54.271554065,-2.960284584 54.271598995,-2.960285628 54.27187322,-2.959708474 54.271980563,-2.959772393 54.272214321,-2.95976247 54.272312433,-2.959856888 54.272400826,-2.960043218 54.272417575,-2.960197171 54.272388905,-2.960411492 54.272099885,-2.960589044 54.271974204,-2.960570764 54.271947858,-2.960493371 54.271966319,-2.960432376 54.271922123,-2.960339211 54.271903172,-2.960461619 54.271947613,-2.960524077 54.2719459,-2.960739023 54.27201534,-2.961047762 54.271995777,-2.961246936 54.272207889,-2.961697205 54.272720619,-2.962768739 54.27274819,-2.962692599 54.27283805,-2.962694694 54.272839276,-2.962541158 54.272884206,-2.962542206 54.272885432,-2.96238867 54.272930361,-2.962389717 54.272931587,-2.96223618 54.272976517,-2.962237227 54.27297713,-2.962160459 54.27302206,-2.962161506 54.273022672,-2.962084738 54.273067602,-2.962085785 54.273068215,-2.962009016 54.273158074,-2.96201111 54.273016872,-2.961685333 54.273135159,-2.961503809 54.273171348,-2.961473939 54.273298743,-2.961277269 54.27334404,-2.961232253 54.273480666,-2.961005083 54.27353446,-2.961021692 54.273533603,-2.961129169 54.273577676,-2.961237692 54.273658549,-2.961239575 54.273903129,-2.960999559 54.273975751,-2.960909108 54.274130593,-2.960651644 54.274283722,-2.960609135 54.274338861,-2.960456848 54.27434907,-2.960303515 54.274458247,-2.960137127 54.274548352,-2.960108508 54.274620729,-2.960048763 54.27469176,-2.960157914 54.27461816,-2.960371202 54.274527077,-2.960522655 54.274499141,-2.960644861 54.274506903,-2.960798613 54.274541745,-2.960937637 54.274648475,-2.961078335 54.274846412,-2.961052228 54.27493725,-2.960931485 54.274903142,-2.960700333 54.274940432,-2.960532271 54.275022529,-2.960380608 54.275015133,-2.960180791 54.275052545,-2.959997374 54.275333677,-2.959681406 54.275397312,-2.95959074 54.275443709,-2.95940753 54.275399756,-2.959283649 54.275328602,-2.959189851 54.275239231,-2.959126346 54.275222358,-2.958987738 54.275304331,-2.958851425 54.275296322,-2.958728381 54.275261599,-2.958574001 54.27526282,-2.958420456 54.275228829,-2.958173949 54.275241231,-2.957744231 54.275233099,-2.957636541 54.275279371,-2.957468683 54.275246109,-2.95713005 54.275274164,-2.956992485 54.275474537,-2.956659265 54.275555289,-2.956676494 54.275590502,-2.956769455 54.275589527,-2.956892292 54.275650966,-2.957078006 54.27579401,-2.957173467 54.275838209,-2.957266637 54.275846219,-2.957389683 54.27589855,-2.957590545 54.275997029,-2.957638902 54.276024353,-2.957593463 54.276132551,-2.9575499 54.276195209,-2.957582068 54.276268072,-2.957460896 54.276241602,-2.957398852 54.276313734,-2.957369809 54.276348702,-2.957493482 54.276384768,-2.95747896 54.276385622,-2.957371476 54.276448768,-2.957342224 54.276530739,-2.957205905 54.276639424,-2.95710092 54.27675612,-2.957118983 54.276756973,-2.957011497 54.276784297,-2.956966057 54.276794623,-2.956797359 54.276777139,-2.956735522 54.276787343,-2.956582179 54.276815032,-2.956490673 54.276770346,-2.956458922 54.276762578,-2.956305163 54.276816738,-2.956275702 54.276737569,-2.956058858 54.276774731,-2.955906139 54.276882806,-2.955877925 54.276972544,-2.955895361 54.277080497,-2.955882502 54.277206058,-2.955916126 54.277385655,-2.955935642 54.2774573,-2.955968018 54.277636776,-2.95600289 54.277744121,-2.956066809 54.277905503,-2.956116621 54.278030576,-2.956211668 54.278119583,-2.956321238 54.278226927,-2.956385159 54.278353097,-2.956342006 54.278397539,-2.95640447 54.278496263,-2.956422115 54.278514479,-2.95639182 54.278622189,-2.956409674 54.278747505,-2.956474012 54.278809676,-2.956567604 54.2788529,-2.956783628 54.278907303,-2.956723454 54.278978582,-2.956801899 54.279041362,-2.956818713 54.279031157,-2.956972064 54.278975778,-2.957155085 54.279028231,-2.957340606 54.279035997,-2.957494374 54.279062711,-2.957525711 54.279206121,-2.957575114 54.279241455,-2.957652727 54.279286629,-2.957623058 54.27930582,-2.957469914 54.27926211,-2.957315312 54.279226044,-2.957329835 54.279164483,-2.95715946 54.279210145,-2.957068366 54.279335949,-2.957071283 54.279381122,-2.957041612 54.279418529,-2.956858172 54.279348103,-2.956672234 54.279375792,-2.956580722 54.279519324,-2.956614766 54.279555512,-2.956584887 54.279590969,-2.956647144 54.279689814,-2.956649435 54.279646103,-2.956494832 54.279565716,-2.956431533 54.279611743,-2.956294369 54.279647321,-2.95634127 54.279756736,-2.956144138 54.279828624,-2.956145803 54.279927104,-2.956194161 54.279963292,-2.956164281 54.279962561,-2.956256419 54.280124309,-2.956260165 54.280124918,-2.956183384 54.280169848,-2.956184424 54.280169239,-2.956261206 54.280393888,-2.95626641 54.280394497,-2.956189628 54.280529287,-2.95619275 54.280529896,-2.956115968 54.280619755,-2.956118049 54.280620973,-2.955964484 54.280665903,-2.955965524 54.280665294,-2.956042307 54.280710224,-2.956043348 54.280709615,-2.95612013 54.280844405,-2.956123252 54.280843796,-2.956200035 54.281158305,-2.956207321 54.281158914,-2.956130537 54.281473423,-2.956137822 54.281474032,-2.956061038 54.281518962,-2.956062078 54.281519571,-2.955985294 54.281564501,-2.955986335 54.281563892,-2.956063119 54.281608822,-2.95606416 54.281608213,-2.956140944 54.281698072,-2.956143025 54.281697463,-2.95621981 54.281832253,-2.956222933 54.281831644,-2.956299717 54.282011363,-2.956303881 54.282010754,-2.956380666 54.282145544,-2.956383789 54.282144935,-2.956460575 54.282189865,-2.956461616 54.282189255,-2.956538401 54.282279115,-2.956540484 54.282278506,-2.956617269 54.282323436,-2.95661831 54.282322826,-2.956695096 54.282367756,-2.956696138 54.282367147,-2.956772923 54.282412077,-2.956773965 54.282410858,-2.956927537 54.282455788,-2.956928578 54.282455178,-2.957005364 54.282500108,-2.957006406 54.282498889,-2.957159978 54.282543819,-2.95716102 54.28254138,-2.957468165 54.28258631,-2.957469207 54.2825857,-2.957545993 54.28254077,-2.957544951 54.282534059,-2.958389598 54.282578988,-2.958390642 54.282577157,-2.958621 54.282936596,-2.958629348 54.282935985,-2.958706135 54.282980915,-2.958707178 54.282980304,-2.958783965 54.283025234,-2.958785009 54.283021569,-2.959245731 54.282976639,-2.959244687 54.282975417,-2.959398261 54.282930488,-2.959397216 54.282929265,-2.95955079 54.282884336,-2.959549746 54.282884947,-2.959472959 54.282660298,-2.959467737 54.282660909,-2.959390951 54.282615979,-2.959389906 54.28261659,-2.95931312 54.28257166,-2.959312076 54.282572882,-2.959158504 54.282527952,-2.95915746 54.282532227,-2.958619957 54.282307578,-2.95861474 54.282308189,-2.958537954 54.282263259,-2.958536911 54.282263869,-2.958460125 54.282218939,-2.958459082 54.28221955,-2.958382296 54.28217462,-2.958381253 54.28217523,-2.958304467 54.2821303,-2.958303424 54.282131521,-2.958149854 54.282176451,-2.958150897 54.282177671,-2.957997326 54.282132741,-2.957996283 54.282133962,-2.957842712 54.282089032,-2.95784167 54.282088422,-2.957918455 54.281998562,-2.95791637 54.281997952,-2.957993155 54.281908092,-2.957991069 54.281907482,-2.958067854 54.281862552,-2.958066811 54.281861332,-2.958220381 54.281816402,-2.958219338 54.281815792,-2.958296123 54.281770862,-2.95829508 54.281770251,-2.958371865 54.281680392,-2.958369778 54.281679171,-2.958523347 54.281589311,-2.958521261 54.281588701,-2.958598045 54.281498841,-2.958595958 54.28149823,-2.958672742 54.281408371,-2.958670656 54.28140715,-2.958824224 54.28113757,-2.958817962 54.28113696,-2.958894745 54.2810471,-2.958892658 54.281045879,-2.959046225 54.281000949,-2.959045181 54.280999727,-2.959198747 54.280954797,-2.959197703 54.280954186,-2.959274487 54.280909256,-2.959273442 54.280908034,-2.959427009 54.280952964,-2.959428053 54.280952353,-2.959504836 54.280997283,-2.95950588 54.280996672,-2.959582664 54.281311181,-2.959589975 54.281311792,-2.959513191 54.28167123,-2.959521546 54.281671842,-2.959444761 54.281851561,-2.959448939 54.28185095,-2.959525724 54.28189588,-2.959526768 54.281895268,-2.959603553 54.281940198,-2.959604597 54.281937142,-2.959988522 54.281892212,-2.959987477 54.2818916,-2.960064262 54.28193653,-2.960065307 54.281935919,-2.960142092 54.281980849,-2.960143137 54.281979626,-2.960296707 54.282024555,-2.960297753 54.282025167,-2.960220968 54.282070097,-2.960222013 54.282070708,-2.960145227 54.282115638,-2.960146273 54.282115027,-2.960223058 54.282249816,-2.960226193 54.282249204,-2.960302979 54.282294134,-2.960304024 54.282292911,-2.960457595 54.282337841,-2.960458641 54.282335393,-2.960765784 54.282290464,-2.960764738 54.28228924,-2.960918309 54.28224431,-2.960917263 54.282243086,-2.961070834 54.282198156,-2.961069788 54.282197544,-2.961146574 54.282152614,-2.961145527 54.28215139,-2.961299098 54.28210646,-2.961298052 54.28210401,-2.961605193 54.28214894,-2.96160624 54.282147715,-2.96175981 54.282237575,-2.961761904 54.282236962,-2.961838689 54.282326822,-2.961840783 54.282326209,-2.961917569 54.282416069,-2.961919663 54.282416681,-2.961842877 54.282461611,-2.961843924 54.282462224,-2.961767138 54.282552083,-2.961769232 54.282552696,-2.961692446 54.282642555,-2.961694539 54.282643168,-2.961617753 54.282733027,-2.961619847 54.28273364,-2.96154306 54.28277857,-2.961544107 54.282779182,-2.96146732 54.282824112,-2.961468367 54.282824724,-2.96139158 54.282869654,-2.961392627 54.282870266,-2.96131584 54.282960126,-2.961317933 54.282960738,-2.961241146 54.283005668,-2.961242192 54.28300628,-2.961165405 54.28305121,-2.961166452 54.283051822,-2.961089665 54.283186612,-2.961092803 54.283187224,-2.961016016 54.283366943,-2.9610202 54.283367555,-2.960943413 54.283547274,-2.960947597 54.283548498,-2.960794021 54.283503568,-2.960792975 54.283506016,-2.960485823 54.283550945,-2.960486869 54.283549722,-2.960640445 54.28390916,-2.96064881 54.283909772,-2.960572022 54.283999632,-2.960574113 54.284000243,-2.960497324 54.284045173,-2.96049837 54.284045785,-2.960421581 54.284090715,-2.960422626 54.284091326,-2.960345837 54.284136256,-2.960346883 54.284137479,-2.960193305 54.284182409,-2.96019435 54.284183021,-2.960117561 54.284227951,-2.960118606 54.284229174,-2.959965027 54.284274103,-2.959966072 54.284274715,-2.959889283 54.284319645,-2.959890328 54.284320256,-2.959813538 54.284365186,-2.959814583 54.284366408,-2.959661004 54.284411338,-2.959662049 54.28441256,-2.95950847 54.28445749,-2.959509514 54.284458712,-2.959355935 54.284503642,-2.959356979 54.284504253,-2.959280189 54.284549183,-2.959281234 54.284549794,-2.959204444 54.284594724,-2.959205488 54.284595335,-2.959128698 54.284685194,-2.959130786 54.284686416,-2.958977206 54.284731346,-2.95897825 54.284731957,-2.95890146 54.284776886,-2.958902504 54.284777497,-2.958825713 54.284822427,-2.958826757 54.284823038,-2.958749967 54.284867967,-2.95875101 54.284869189,-2.958597429 54.285003978,-2.95860056 54.285005199,-2.958446979 54.285095059,-2.958449065 54.285095669,-2.958372274 54.285140599,-2.958373318 54.285141209,-2.958296527 54.285231069,-2.958298613 54.285231679,-2.958221822 54.285276609,-2.958222865 54.28527783,-2.958069283 54.285367689,-2.958071368 54.285367079,-2.95814816 54.285501869,-2.958151289 54.285500648,-2.958304872 54.285545578,-2.958305916 54.285544357,-2.958459499 54.285589287,-2.958460542 54.285588676,-2.958537334 54.285543746,-2.958536291 54.285542525,-2.958689874 54.285497595,-2.95868883 54.285496985,-2.958765622 54.285317266,-2.958761447 54.285316044,-2.95891503 54.285360974,-2.958916074 54.285360363,-2.958992865 54.285405293,-2.958993909 54.285404682,-2.9590707 54.285449612,-2.959071745 54.285447779,-2.959302119 54.285537639,-2.959304208 54.285539471,-2.959073833 54.285584401,-2.959074877 54.285585012,-2.958998085 54.285629942,-2.958999129 54.285631163,-2.958845545 54.285676093,-2.958846589 54.285676704,-2.958769797 54.285721634,-2.958770841 54.285722855,-2.958617257 54.285812715,-2.958619344 54.285813325,-2.958542552 54.285858255,-2.958543595 54.285858866,-2.958466803 54.285993655,-2.958469933 54.285994265,-2.95839314 54.286039195,-2.958394184 54.286039806,-2.958317391 54.286084735,-2.958318434 54.286085346,-2.958241642 54.286130276,-2.958242685 54.286130886,-2.958165892 54.286175816,-2.958166935 54.286177036,-2.958013349 54.286221966,-2.958014392 54.286222576,-2.957937599 54.286312436,-2.957939685 54.286313046,-2.957862892 54.286357976,-2.957863935 54.286358586,-2.957787141 54.286448445,-2.957789227 54.286449055,-2.957712433 54.286808494,-2.957720774 54.286807884,-2.957797568 54.286852814,-2.957798611 54.286850983,-2.958028994 54.286895913,-2.958030036 54.286895303,-2.958106831 54.286940233,-2.958107874 54.286938402,-2.958338257 54.286983332,-2.9583393 54.28698089,-2.958646478 54.287025819,-2.958647521 54.287023377,-2.958954699 54.287068306,-2.958955743 54.287067085,-2.959109332 54.287112014,-2.959110376 54.287110793,-2.959263966 54.287155722,-2.95926501 54.287155111,-2.959341805 54.287200041,-2.959342849 54.28719943,-2.959419644 54.28724436,-2.959420688 54.287243749,-2.959497483 54.287288678,-2.959498528 54.287288067,-2.959575323 54.287332997,-2.959576367 54.287332386,-2.959653162 54.287377316,-2.959654207 54.287376093,-2.959807797 54.287421023,-2.959808842 54.287417354,-2.960269613 54.287462284,-2.960270659 54.287461672,-2.960347454 54.287506602,-2.960348499 54.287505378,-2.96050209 54.287550308,-2.960503136 54.287549696,-2.960579931 54.287594626,-2.960580977 54.287593402,-2.960734568 54.287638332,-2.960735614 54.287634659,-2.961196387 54.28758973,-2.961195341 54.287588505,-2.961348932 54.287543575,-2.961347885 54.287541738,-2.961578271 54.287496808,-2.961577224 54.287496196,-2.96165402 54.287541126,-2.961655067 54.287538062,-2.962039043 54.287672852,-2.962042185 54.287673464,-2.96196539 54.288032902,-2.961973768 54.288033515,-2.961896972 54.288123375,-2.961899066 54.288125212,-2.961668677 54.288170142,-2.961669724 54.288171367,-2.961516131 54.288261227,-2.961518225 54.288265513,-2.960980648 54.288220583,-2.960979601 54.288221807,-2.960826008 54.288131947,-2.960823916 54.288132559,-2.960747119 54.28808763,-2.960746073 54.288088242,-2.960669277 54.288043312,-2.960668231 54.288043924,-2.960591435 54.287998994,-2.960590389 54.288000217,-2.960436797 54.287955288,-2.960435751 54.287955899,-2.960358955 54.28791097,-2.960357909 54.287911581,-2.960281113 54.287866652,-2.960280068 54.287867263,-2.960203272 54.287822333,-2.960202226 54.287822945,-2.96012543 54.287778015,-2.960124385 54.287781072,-2.959740406 54.287826002,-2.959741451 54.287827225,-2.959587859 54.287872154,-2.959588904 54.287873377,-2.959435312 54.287963236,-2.959437401 54.287963847,-2.959360604 54.288053707,-2.959362693 54.288054318,-2.959285897 54.288278967,-2.959291119 54.288279578,-2.959214322 54.288459297,-2.959218499 54.288459908,-2.959141702 54.288639627,-2.959145879 54.288640238,-2.959069082 54.288730098,-2.95907117 54.288730708,-2.958994373 54.288820568,-2.958996461 54.288821179,-2.958919663 54.288911038,-2.958921751 54.288911649,-2.958844953 54.289226158,-2.958852261 54.289226768,-2.958775462 54.289451417,-2.958780681 54.289450806,-2.95885748 54.289540666,-2.958859568 54.289539444,-2.959013167 54.289584374,-2.959014211 54.289578875,-2.959705404 54.289623804,-2.959706449 54.289622582,-2.959860047 54.289667511,-2.959861092 54.2896669,-2.959937892 54.28971183,-2.959938937 54.289711218,-2.960015736 54.289801078,-2.960017827 54.289800466,-2.960094626 54.289980185,-2.960098808 54.289979574,-2.960175608 54.290114363,-2.960178744 54.290114975,-2.960101944 54.290204834,-2.960104034 54.290205446,-2.960027234 54.290250375,-2.960028279 54.290250987,-2.959951479 54.290295917,-2.959952524 54.290296528,-2.959875724 54.290341458,-2.959876769 54.29034268,-2.959723167 54.29038761,-2.959724212 54.290388833,-2.959570611 54.290433763,-2.959571656 54.290434374,-2.959494855 54.290479303,-2.9594959 54.290479915,-2.959419099 54.290524844,-2.959420143 54.290525455,-2.959343342 54.290570385,-2.959344387 54.290570996,-2.959267586 54.290526067,-2.959266541 54.290526678,-2.95918974 54.290571607,-2.959190785 54.29057344,-2.958960382 54.29061837,-2.958961426 54.290619591,-2.958807823 54.290664521,-2.958808867 54.290665132,-2.958732066 54.290710062,-2.95873311 54.290711283,-2.958579507 54.290756213,-2.958580551 54.290756823,-2.95850375 54.290801753,-2.958504793 54.290802363,-2.958427992 54.290847293,-2.958429035 54.290847904,-2.958352234 54.290892834,-2.958353277 54.290893444,-2.958276476 54.290983303,-2.958278562 54.290983914,-2.958201761 54.291028844,-2.958202804 54.291029454,-2.958126002 54.291119313,-2.958128088 54.291119924,-2.958051286 54.291209783,-2.958053373 54.291210394,-2.95797657 54.291300253,-2.957978657 54.291300863,-2.957901854 54.291390723,-2.95790394 54.291391333,-2.957827138 54.291481193,-2.957829223 54.291481803,-2.957752421 54.291571662,-2.957754506 54.291572272,-2.957677703 54.291707062,-2.957680832 54.291707672,-2.957604028 54.291752601,-2.957605071 54.291753211,-2.957528268 54.291798141,-2.957529311 54.291799361,-2.957375704 54.291844291,-2.957376746 54.29184551,-2.957223139 54.29189044,-2.957224182 54.291891659,-2.957070574 54.291936589,-2.957071617 54.291937199,-2.956994813 54.291982128,-2.956995855 54.291983348,-2.956842247 54.292028277,-2.956843289 54.292029496,-2.956689682 54.292074426,-2.956690723 54.292075035,-2.95661392 54.292119965,-2.956614961 54.292120574,-2.956538157 54.292165504,-2.956539199 54.292166114,-2.956462395 54.292255973,-2.956464478 54.292256582,-2.956387673 54.292301512,-2.956388715 54.292302121,-2.95631191 54.292391981,-2.956313993 54.29239259,-2.956237188 54.292446871,-2.956192355 54.292617848,-2.95616559 54.292618457,-2.956088785 54.292708317,-2.956090867 54.292708926,-2.956014062 54.292753855,-2.956015103 54.292754464,-2.955938298 54.292844324,-2.95594038 54.292844933,-2.955863574 54.292934792,-2.955865656 54.292934184,-2.955942461 54.293024043,-2.955944543 54.293023434,-2.956021349 54.293113294,-2.956023431 54.293113903,-2.955946625 54.293158833,-2.955947666 54.293158224,-2.956024472 54.293382872,-2.956029677 54.293382263,-2.956106484 54.293517053,-2.956109607 54.293516444,-2.956186414 54.293561374,-2.956187455 54.293560764,-2.956264261 54.293740484,-2.956268427 54.293739874,-2.956345234 54.293874664,-2.956348358 54.293873445,-2.956501972 54.293918375,-2.956503014 54.293917766,-2.956579821 54.294007625,-2.956581905 54.294007016,-2.956658712 54.294051946,-2.956659754 54.294051336,-2.956736562 54.294096266,-2.956737603 54.294095657,-2.956814411 54.294140586,-2.956815453 54.294139977,-2.956892261 54.294184907,-2.956893303 54.294184297,-2.956970111 54.294139367,-2.956969068 54.294138148,-2.957122684 54.294183078,-2.957123726 54.294181858,-2.957277342 54.294226788,-2.957278384 54.294226178,-2.957355192 54.294181248,-2.95735415 54.294176978,-2.957891804 54.294132049,-2.957890761 54.294128387,-2.958351607 54.294083457,-2.958350564 54.294082236,-2.958504179 54.294037306,-2.958503135 54.294034864,-2.958810365 54.294079793,-2.958811409 54.294079183,-2.958888217 54.294124112,-2.958889261 54.294123501,-2.958966069 54.29434815,-2.95897129 54.294348761,-2.958894482 54.29452848,-2.958898659 54.294527869,-2.958975467 54.294572799,-2.958976512 54.294572188,-2.95905332 54.294662047,-2.959055409 54.294661436,-2.959132218 54.294706366,-2.959133262 54.294705755,-2.959210071 54.294750685,-2.959211115 54.294751296,-2.959134307 54.294841155,-2.959136396 54.294841766,-2.959059587 54.295066415,-2.959064809 54.295065804,-2.959141618 54.295155664,-2.959143707 54.295155053,-2.959220517 54.295199982,-2.959221561 54.295199371,-2.959298371 54.295693598,-2.959309862 54.295692987,-2.959386673 54.295782847,-2.959388763 54.295781624,-2.959542384 54.295826554,-2.959543429 54.295825943,-2.95962024 54.295781013,-2.959619195 54.295779179,-2.959849627 54.296048757,-2.959855899 54.296049368,-2.959779088 54.296094298,-2.959780133 54.296095521,-2.95962651 54.296140451,-2.959627555 54.296141062,-2.959550744 54.29645557,-2.959558059 54.296456181,-2.959481247 54.296546041,-2.959483337 54.296547263,-2.959329712 54.296637122,-2.959331802 54.296638344,-2.959178177 54.296683274,-2.959179222 54.296684496,-2.959025597 54.296729426,-2.959026641 54.29673248,-2.958642579 54.29677741,-2.958643623 54.296778631,-2.958489997 54.29682356,-2.958491041 54.296824782,-2.958337416 54.296869711,-2.95833846 54.296870322,-2.958261647 54.296915251,-2.95826269 54.296915862,-2.958185877 54.296960792,-2.958186921 54.296961402,-2.958110108 54.297006332,-2.958111151 54.297006942,-2.958034338 54.297051872,-2.958035382 54.297053092,-2.957881755 54.297142952,-2.957883842 54.297143562,-2.957807029 54.297592859,-2.95781746 54.297592249,-2.957894274 54.297682108,-2.957896361 54.297682719,-2.957819546 54.297817508,-2.957822676 54.297896184,-2.958101086 54.298030119,-2.958211758 54.298263143,-2.958294 54.298407285,-2.95825125 54.29844103,-2.958528622 54.29853089,-2.95853071 54.298529669,-2.958684342 54.298574598,-2.958685386 54.298573987,-2.958762202 54.298663847,-2.95876429 54.298800468,-2.958536974 54.298890328,-2.958539062 54.298889106,-2.958692695 54.298978966,-2.958694783 54.298977744,-2.958848416 54.299022674,-2.958849461 54.299022063,-2.958926278 54.299066993,-2.958927322 54.299066382,-2.959004139 54.299111311,-2.959005183 54.2991107,-2.959082 54.299195059,-2.959775443 54.299150129,-2.959774398 54.299148295,-2.960004849 54.299193225,-2.960005895 54.299192613,-2.960082712 54.299237543,-2.960083757 54.299236931,-2.960160575 54.299281861,-2.96016162 54.299282472,-2.960084803 54.299327402,-2.960085849 54.299326791,-2.960162666 54.299514884,-2.960243876 54.29950345,-2.960550937 54.29954838,-2.960551984 54.299547768,-2.960628801 54.299592698,-2.960629848 54.299592086,-2.960706665 54.299637015,-2.960707712 54.299636403,-2.96078453 54.299681333,-2.960785576 54.299677047,-2.961323302 54.299766907,-2.961325396 54.299765682,-2.961479032 54.299720752,-2.961477985 54.299719527,-2.961631621 54.299674597,-2.961630574 54.299672759,-2.961861028 54.299762619,-2.961863123 54.299761393,-2.962016759 54.299806323,-2.962017807 54.29980571,-2.962094625 54.29985064,-2.962095673 54.299830216,-2.962402527 54.29989373,-2.962327176 54.300073449,-2.962331368 54.300072836,-2.962408187 54.300252555,-2.96241238 54.300251941,-2.962489199 54.300341801,-2.962491296 54.300341188,-2.962568115 54.300475976,-2.96257126 54.30047659,-2.962494441 54.300521519,-2.962495489 54.300520906,-2.962572309 54.300565836,-2.962573357 54.300563996,-2.962803816 54.300608926,-2.962804865 54.300607699,-2.962958504 54.300652628,-2.962959553 54.300651401,-2.963113192 54.300696331,-2.963114241 54.300701851,-2.962422863 54.300746781,-2.962423911 54.300749846,-2.962039811 54.300920578,-2.962043793 54.301021875,-2.961738817 54.301201593,-2.961743007 54.301202206,-2.961666186 54.301247136,-2.961667234 54.301247748,-2.961590413 54.301337607,-2.961592508 54.30133822,-2.961515687 54.301428079,-2.961517781 54.301428692,-2.96144096 54.301518551,-2.961443055 54.301564706,-2.961290459 54.301654565,-2.961292553 54.301655177,-2.961215732 54.301745037,-2.961217826 54.301745649,-2.961141004 54.301790579,-2.961142051 54.301791803,-2.960988407 54.302246608,-2.96030747 54.302245996,-2.960384293 54.302380785,-2.960387431 54.302381397,-2.960310608 54.302426326,-2.960311654 54.30242755,-2.960158008 54.302472479,-2.960159054 54.302471868,-2.960235877 54.302561727,-2.960237969 54.302561115,-2.960314792 54.302875623,-2.960322115 54.302875011,-2.960398939 54.302919941,-2.960399985 54.302919329,-2.960476809 54.302964259,-2.960477855 54.302963647,-2.960554679 54.303008576,-2.960555726 54.303007964,-2.96063255 54.303052894,-2.960633596 54.30305167,-2.960787245 54.3030966,-2.960788291 54.303095987,-2.960865116 54.303140917,-2.960866162 54.303140305,-2.960942987 54.303230164,-2.96094508 54.303230776,-2.960868256 54.303320636,-2.960870349 54.303321248,-2.960793524 54.303680685,-2.960801897 54.303995805,-2.960732398 54.304130593,-2.960735538 54.304129981,-2.960812364 54.304174911,-2.96081341 54.304173687,-2.960967063 54.304218616,-2.96096811 54.304216779,-2.961198589 54.304261709,-2.961199636 54.304262934,-2.961045983 54.304307863,-2.96104703 54.304309088,-2.960893377 54.304354017,-2.960894424 54.30435463,-2.960817597 54.304444489,-2.960819691 54.304443877,-2.960896517 54.304533736,-2.960898611 54.304533124,-2.960975438 54.304622983,-2.960977532 54.304622371,-2.961054359 54.304847019,-2.961059594 54.304847631,-2.960982766 54.30493749,-2.96098486 54.305256282,-2.96045439 54.305301212,-2.960455436 54.3053006,-2.960532264 54.30534553,-2.960533311 54.305346142,-2.960456482 54.305436001,-2.960458575 54.305436613,-2.960381746 54.305571402,-2.960384885 54.305572014,-2.960308056 54.305616943,-2.960309102 54.305617555,-2.960232273 54.305662485,-2.960233319 54.305663096,-2.96015649 54.305618167,-2.960155444 54.30561939,-2.960001786 54.30566432,-2.960002832 54.305664931,-2.959926003 54.30579972,-2.95992914 54.305800332,-2.959852311 54.305845261,-2.959853356 54.305845873,-2.959776527 54.306025591,-2.959780709 54.30602498,-2.959857539 54.306069909,-2.959858585 54.306068075,-2.960089074 54.306113004,-2.96009012 54.306112393,-2.96016695 54.306337041,-2.96017218 54.306336429,-2.960249011 54.306650937,-2.960256334 54.306650325,-2.960333165 54.306785114,-2.960336304 54.306784502,-2.960413135 54.306919291,-2.960416274 54.306918679,-2.960493105 54.307008538,-2.960495198 54.307007926,-2.96057203 54.307052856,-2.960573076 54.307052244,-2.960649908 54.307142103,-2.960652001 54.307141491,-2.960728833 54.30718642,-2.96072988 54.307185808,-2.960806712 54.307140879,-2.960805665 54.307139654,-2.960959329 54.307094725,-2.960958282 54.307094112,-2.961035113 54.307139042,-2.96103616 54.30713843,-2.961112992 54.307183359,-2.961114039 54.307182747,-2.961190871 54.307452325,-2.961197154 54.307452937,-2.961120322 54.307497867,-2.961121369 54.307498479,-2.961044537 54.307588338,-2.961046631 54.307590175,-2.960816133 54.307635104,-2.960817179 54.307636329,-2.960663514 54.307681258,-2.960664561 54.30768187,-2.960587728 54.3077268,-2.960588774 54.307727412,-2.960511941 54.307817271,-2.960514034 54.307817883,-2.960437201 54.307952672,-2.96044034 54.307953284,-2.960363507 54.308177932,-2.960368739 54.30817732,-2.960445572 54.30822225,-2.960446619 54.308221638,-2.960523453 54.308266567,-2.960524499 54.308265955,-2.960601333 54.308310885,-2.96060238 54.308310273,-2.960679214 54.308355203,-2.96068026 54.30835459,-2.960757095 54.30844445,-2.960759188 54.308443838,-2.960836022 54.308578626,-2.960839163 54.308578014,-2.960915998 54.308712803,-2.960919138 54.308713415,-2.960842304 54.309072852,-2.960850679 54.30907224,-2.960927514 54.309207029,-2.960930655 54.309207641,-2.960853819 54.3092975,-2.960855913 54.309298112,-2.960779077 54.309432901,-2.960782218 54.309432289,-2.960859054 54.309567078,-2.960862195 54.309566465,-2.960939031 54.309611395,-2.960940078 54.309610783,-2.961016915 54.309655712,-2.961017962 54.309656325,-2.960941125 54.309701254,-2.960942172 54.309701866,-2.960865335 54.309836655,-2.960868476 54.309837267,-2.960791639 54.310016986,-2.960795827 54.310017598,-2.96071899 54.310062527,-2.960720036 54.310061915,-2.960796874 54.310151775,-2.960798968 54.310152387,-2.96072213 54.310287175,-2.96072527 54.310287788,-2.960648433 54.310377647,-2.960650526 54.310377035,-2.960727364 54.310421964,-2.960728411 54.310422576,-2.960651573 54.310512436,-2.960653666 54.310665563,-2.960611122 54.310774006,-2.960536796 54.310873096,-2.960508363 54.310927624,-2.96043278 54.311053427,-2.96043571 54.311054039,-2.960358871 54.311143898,-2.960360964 54.31114451,-2.960284124 54.311189439,-2.960285171 54.311190051,-2.960208331 54.31127991,-2.960210424 54.311280522,-2.960133584 54.311325452,-2.96013463 54.311326063,-2.960057791 54.311415922,-2.960059883 54.311416534,-2.959983043 54.311551323,-2.959986181 54.311551935,-2.959909341 54.311596864,-2.959910387 54.311597476,-2.959833547 54.311732264,-2.959836685 54.311732876,-2.959759844 54.311777806,-2.95976089 54.311778417,-2.95968405 54.311733487,-2.959683004 54.311734099,-2.959606163 54.311823958,-2.959608255 54.311823347,-2.959685095 54.311958135,-2.959688232 54.311957524,-2.959765073 54.312092313,-2.959768211 54.312091701,-2.959845052 54.312540997,-2.959855511 54.312676275,-2.959797175 54.312784106,-2.959799685 54.312893527,-2.959602405 54.313082598,-2.959560691 54.31308321,-2.959483848 54.313128139,-2.959484894 54.31312875,-2.95940805 54.31321861,-2.959410141 54.313219832,-2.959256455 54.313264762,-2.9592575 54.313265373,-2.959180657 54.313310302,-2.959181702 54.313286278,-2.958812227 54.313407492,-2.95826167 54.313452421,-2.958262714 54.313453642,-2.958109027 54.313498572,-2.958110071 54.313499182,-2.958033227 54.313544112,-2.958034271 54.313544722,-2.957957427 54.313589652,-2.957958471 54.313590873,-2.957804783 54.313635802,-2.957805827 54.313637023,-2.957652139 54.313681952,-2.957653183 54.313682563,-2.957576338 54.313817351,-2.957579469 54.313816741,-2.957656313 54.314041389,-2.957661531 54.314040779,-2.957738376 54.314265427,-2.957743595 54.314264817,-2.95782044 54.314309746,-2.957821484 54.314309136,-2.957898329 54.314398995,-2.957900417 54.314398385,-2.957977262 54.314443315,-2.957978306 54.314442704,-2.958055152 54.314487634,-2.958056196 54.314487023,-2.958133041 54.314531953,-2.958134085 54.314531342,-2.958210931 54.314576272,-2.958211975 54.314575661,-2.958288821 54.314620591,-2.958289865 54.31461998,-2.958366711 54.31466491,-2.958367756 54.314664299,-2.958444602 54.314754159,-2.958446691 54.314753548,-2.958523537 54.314798477,-2.958524581 54.314797867,-2.958601427 54.314887726,-2.958603517 54.314887115,-2.958680363 54.314932045,-2.958681408 54.314931434,-2.958758254 54.315111152,-2.958762434 54.315111763,-2.958685587 54.315156693,-2.958686632 54.315157303,-2.958609785 54.315247163,-2.958611874 54.315247773,-2.958535027 54.315292703,-2.958536072 54.315293314,-2.958459225 54.315383173,-2.958461314 54.315383783,-2.958384467 54.315518572,-2.9583876 54.315519183,-2.958310752 54.31578876,-2.958317019 54.315789371,-2.958240171 54.316103878,-2.958247481 54.316104489,-2.958170632 54.316194348,-2.958172721 54.316194958,-2.958095872 54.316239888,-2.958096916 54.316240498,-2.958020067 54.316285428,-2.958021111 54.316286038,-2.957944262 54.316330968,-2.957945306 54.316331578,-2.957868457 54.316466367,-2.957871589 54.316465757,-2.957948438 54.316959982,-2.957959923 54.316959372,-2.958036773 54.31709416,-2.958039905 54.31709355,-2.958116756 54.31713848,-2.9581178 54.31713909,-2.95804095 54.317273879,-2.958044082 54.317274489,-2.957967231 54.317319419,-2.957968275 54.317320029,-2.957891424 54.317409888,-2.957893512 54.317410499,-2.957816661 54.317500358,-2.957818749 54.317500968,-2.957741898 54.317635757,-2.957745029 54.317636367,-2.957668178 54.317771156,-2.957671309 54.317771766,-2.957594458 54.317906555,-2.957597589 54.317907165,-2.957520737 54.317997024,-2.957522824 54.317997634,-2.957445972 54.318042564,-2.957447015 54.318043174,-2.957370163 54.318088103,-2.957371206 54.318088713,-2.957294354 54.318133643,-2.957295397 54.318134253,-2.957218545 54.318179182,-2.957219588 54.318179792,-2.957142736 54.318224722,-2.957143779 54.318225332,-2.957066926 54.318270261,-2.95706797 54.318270871,-2.956991117 54.318315801,-2.95699216 54.31831702,-2.956838454 54.31836195,-2.956839497 54.31836256,-2.956762644 54.318407489,-2.956763687 54.318408708,-2.956609981 54.318453638,-2.956611024 54.318454247,-2.956534171 54.318499177,-2.956535213 54.318500396,-2.956381507 54.318545325,-2.95638255 54.318545935,-2.956305696 54.318590864,-2.956306739 54.318591474,-2.956229885 54.31895091,-2.956238223 54.318950301,-2.956315077 54.31908509,-2.956318204 54.319084481,-2.956395059 54.319219269,-2.956398186 54.31921866,-2.95647504 54.319353449,-2.956478168 54.319352839,-2.956555023 54.319532557,-2.956559193 54.319531948,-2.956636048 54.319666737,-2.956639176 54.319666127,-2.956716031 54.320385,-2.956732717 54.32038561,-2.95665586 54.320475469,-2.956657945 54.320476079,-2.956581089 54.320565938,-2.956583174 54.320566547,-2.956506317 54.320656406,-2.956508402 54.320657016,-2.956431545 54.320701945,-2.956432588 54.320702555,-2.95635573 54.320882273,-2.9563599 54.320883492,-2.956206185 54.320928421,-2.956207227 54.320929031,-2.95613037 54.32097396,-2.956131412 54.32097457,-2.956054554 54.321019499,-2.956055596 54.321020108,-2.955978739 54.321065038,-2.955979781 54.321065647,-2.955902923 54.321110577,-2.955903965 54.321111186,-2.955827107 54.321156115,-2.955828149 54.321156724,-2.95575129 54.321246583,-2.955753374 54.321247192,-2.955676516 54.321337052,-2.955678599 54.321337661,-2.955601741 54.32138259,-2.955602782 54.321381981,-2.955679641 54.321426911,-2.955680683 54.32142752,-2.955603824 54.321472449,-2.955604866 54.321473058,-2.955528007 54.321562917,-2.95553009 54.321563526,-2.955453231 54.321653385,-2.955455314 54.321653994,-2.955378455 54.321698924,-2.955379497 54.321699533,-2.955302638 54.321789392,-2.95530472 54.32179,-2.955227861 54.32187986,-2.955229944 54.321880468,-2.955153084 54.321925398,-2.955154125 54.321926006,-2.955077266 54.321970936,-2.955078307 54.321971545,-2.955001447 54.322016474,-2.955002488 54.322017083,-2.954925629 54.322062012,-2.95492667 54.322062621,-2.95484981 54.32210755,-2.954850851 54.322108159,-2.954773991 54.322153088,-2.954775031 54.322153697,-2.954698172 54.322198626,-2.954699212 54.322199235,-2.954622352 54.322244164,-2.954623393 54.322245381,-2.954469672 54.32233524,-2.954471753 54.322335848,-2.954394893 54.322470637,-2.954398014 54.322471245,-2.954321154 54.322516175,-2.954322194 54.322515566,-2.954399055 54.322560496,-2.954400095 54.322559888,-2.954476956 54.322604817,-2.954477996 54.322605426,-2.954401135 54.322695285,-2.954403216 54.322694677,-2.954480077 54.322964254,-2.95448632 54.322964862,-2.954409459 54.323009792,-2.954410499 54.3230104,-2.954333637 54.323100259,-2.954335718 54.323102083,-2.954105133 54.323147013,-2.954106173 54.323147621,-2.954029311 54.323192551,-2.954030351 54.323193766,-2.953876627 54.323283625,-2.953878707 54.323284233,-2.953801845 54.323329163,-2.953802885 54.323329771,-2.953726022 54.3233747,-2.953727062 54.323375308,-2.9536502 54.323420238,-2.953651239 54.323420845,-2.953574377 54.323555634,-2.953577496 54.323556242,-2.953500633 54.323646101,-2.953502712 54.323646708,-2.953425849 54.323781497,-2.953428967 54.323782105,-2.953352104 54.324051682,-2.95335834 54.324051075,-2.953435203 54.324185863,-2.953438322 54.324185256,-2.953515185 54.324275115,-2.953517264 54.324274507,-2.953594128 54.324364367,-2.953596208 54.324363759,-2.953673072 54.324453618,-2.953675151 54.32445301,-2.953752015 54.324542869,-2.953754095 54.324542262,-2.953830959 54.324632121,-2.953833039 54.324631513,-2.953909904 54.324721372,-2.953911983 54.324720764,-2.953988848 54.324810623,-2.953990928 54.324810015,-2.954067793 54.324944804,-2.954070913 54.32494298,-2.954301509 54.32489805,-2.954300469 54.324897442,-2.954377334 54.324852513,-2.954376293 54.324851905,-2.954453158 54.324806975,-2.954452118 54.324806367,-2.954528983 54.324761437,-2.954527942 54.324760829,-2.954604807 54.324715899,-2.954603766 54.324715291,-2.954680631 54.324625432,-2.954678549 54.324624823,-2.954755414 54.324579894,-2.954754373 54.324579285,-2.954831238 54.324444497,-2.954828115 54.324443888,-2.954904979 54.324354029,-2.954902897 54.324353421,-2.954979761 54.324308491,-2.95497872 54.324307882,-2.955055584 54.324262953,-2.955054543 54.324261736,-2.955208271 54.324306665,-2.955209312 54.324307274,-2.955132448 54.324397133,-2.955134531 54.324397742,-2.955057666 54.324487601,-2.955059749 54.324488209,-2.954982884 54.324578068,-2.954984967 54.324578677,-2.954908102 54.324668536,-2.954910184 54.324669145,-2.95483332 54.324759004,-2.954835401 54.324759612,-2.954758537 54.324849471,-2.954760618 54.32485008,-2.954683753 54.324939939,-2.954685835 54.324940547,-2.95460897 54.325030406,-2.954611051 54.325031015,-2.954534186 54.325120874,-2.954536267 54.325123914,-2.954151939 54.325168844,-2.95415298 54.32517006,-2.953999248 54.32521499,-2.954000289 54.325215597,-2.953923423 54.325260527,-2.953924463 54.325261135,-2.953847597 54.325306064,-2.953848637 54.325306672,-2.953771771 54.325351602,-2.953772811 54.32535221,-2.953695945 54.325397139,-2.953696985 54.325397747,-2.953620119 54.325442676,-2.953621158 54.325443284,-2.953544292 54.325488214,-2.953545332 54.325489429,-2.953391599 54.325534358,-2.953392639 54.325535573,-2.953238906 54.325580503,-2.953239945 54.32558111,-2.953163079 54.325760829,-2.953167236 54.325760221,-2.953244102 54.326074728,-2.953251377 54.326075336,-2.95317451 54.326120265,-2.953175549 54.326120873,-2.953098682 54.326165802,-2.953099721 54.32616641,-2.953022853 54.326211339,-2.953023892 54.326213161,-2.952793289 54.326258091,-2.952794328 54.326259305,-2.952640593 54.326304234,-2.952641631 54.326304842,-2.952564764 54.326349771,-2.952565802 54.326350985,-2.952412067 54.326395915,-2.952413105 54.326396522,-2.952336237 54.326486381,-2.952338314 54.326487594,-2.952184577 54.326532524,-2.952185615 54.326533131,-2.952108747 54.32662299,-2.952110823 54.326623597,-2.952033955 54.326668526,-2.952034993 54.326669133,-2.951958125 54.326714062,-2.951959162 54.326714669,-2.951882294 54.326759599,-2.951883332 54.326760205,-2.951806463 54.326805135,-2.951807501 54.326805741,-2.951730632 54.326850671,-2.95173167 54.326851277,-2.951654801 54.326941137,-2.951656876 54.326941743,-2.951580007 54.326986673,-2.951581045 54.326987279,-2.951504175 54.327032209,-2.951505213 54.327032815,-2.951428344 54.327077745,-2.951429381 54.327078351,-2.951352512 54.32716821,-2.951354586 54.327168816,-2.951277717 54.327393464,-2.951282903 54.327392858,-2.951359773 54.327976943,-2.951373257 54.327976336,-2.951450128 54.328156055,-2.951454278 54.328156661,-2.951377406 54.328201591,-2.951378444 54.328202197,-2.951301572 54.328381915,-2.951305721 54.328382521,-2.95122885 54.32847238,-2.951230924 54.328472987,-2.951154052 54.328562846,-2.951156126 54.328563452,-2.951079254 54.328608382,-2.951080291 54.328608988,-2.951003419 54.328698847,-2.951005493 54.328699453,-2.950928621 54.328744382,-2.950929658 54.328744988,-2.950852785 54.328834848,-2.950854859 54.328835453,-2.950777986 54.328880383,-2.950779023 54.328880989,-2.95070215 54.329105637,-2.950707333 54.329106243,-2.95063046 54.329151172,-2.950631497 54.329150566,-2.95070837 54.329330285,-2.950712516 54.329329679,-2.95078939 54.329419538,-2.950791463 54.329418932,-2.950868337 54.329463862,-2.950869374 54.329463256,-2.950946247 54.329508185,-2.950947284 54.329507579,-2.951024158 54.329552509,-2.951025195 54.329551903,-2.951102069 54.329596832,-2.951103106 54.32959562,-2.951256854 54.32964055,-2.951257891 54.329639943,-2.951334765 54.329684873,-2.951335802 54.32968366,-2.95148955 54.32972859,-2.951490588 54.329727983,-2.951567462 54.329772913,-2.9515685 54.329772307,-2.951645374 54.329817236,-2.951646412 54.329816023,-2.95180016 54.329860953,-2.951801198 54.329859739,-2.951954947 54.329904669,-2.951955985 54.329902849,-2.952186608 54.329947778,-2.952187647 54.329946565,-2.952341396 54.329991494,-2.952342434 54.32999028,-2.952496183 54.33003521,-2.952497222 54.330034603,-2.952574097 54.330079532,-2.952575135 54.330078925,-2.95265201 54.330033996,-2.952650971 54.330032781,-2.952804721 54.329987852,-2.952803682 54.329987245,-2.952880557 54.329942315,-2.952879517 54.329941708,-2.952956392 54.329851849,-2.952954314 54.329851241,-2.953031188 54.329806312,-2.953030149 54.329803274,-2.95341452 54.329758345,-2.953413481 54.329757737,-2.953490355 54.329802667,-2.953491395 54.329798412,-2.954029514 54.329843342,-2.954030555 54.32984395,-2.95395368 54.330023668,-2.953957841 54.330022452,-2.95411159 54.33015724,-2.954114711 54.330157848,-2.954037837 54.330337567,-2.954041998 54.33033635,-2.954195748 54.33038128,-2.954196789 54.330380064,-2.954350539 54.330424993,-2.95435158 54.330423168,-2.954582206 54.330468098,-2.954583247 54.330468706,-2.954506372 54.330513636,-2.954507412 54.330514244,-2.954430537 54.330559174,-2.954431577 54.330559782,-2.954354702 54.330604711,-2.954355742 54.330605928,-2.954201991 54.330650857,-2.954203031 54.330651465,-2.954126156 54.330696395,-2.954127196 54.330697611,-2.953973444 54.33074254,-2.953974484 54.330743148,-2.953897608 54.330788078,-2.953898649 54.33079294,-2.95328364 54.33074801,-2.9532826 54.330749832,-2.953051972 54.330794762,-2.953053011 54.330797191,-2.952745507 54.330752261,-2.952744468 54.330753476,-2.952590716 54.330798405,-2.952591755 54.330799012,-2.952514879 54.330843942,-2.952515917 54.330844549,-2.952439041 54.330889478,-2.95244008 54.330891906,-2.952132574 54.330936835,-2.952133613 54.330937442,-2.952056736 54.330982372,-2.952057774 54.330982978,-2.951980898 54.331027908,-2.951981936 54.331029121,-2.951828183 54.331074051,-2.951829221 54.331074657,-2.951752344 54.331299305,-2.951757533 54.331298699,-2.95183441 54.331343628,-2.951835448 54.331342415,-2.951989202 54.331297485,-2.951988164 54.331295058,-2.952295673 54.331339988,-2.952296711 54.331338774,-2.952450465 54.331293844,-2.952449427 54.33129263,-2.952603181 54.331247701,-2.952602142 54.331245272,-2.95290965 54.331290201,-2.952910689 54.331289594,-2.952987566 54.331424383,-2.952990684 54.331423775,-2.953067561 54.331513635,-2.95306964 54.331514242,-2.952992762 54.331604101,-2.95299484 54.331604708,-2.952917963 54.331694567,-2.952920041 54.331695175,-2.952843163 54.331740104,-2.952844202 54.331740711,-2.952767324 54.33192043,-2.95277148 54.331921644,-2.952617724 54.331966573,-2.952618763 54.33196718,-2.952541884 54.33205704,-2.952543962 54.332057647,-2.952467083 54.332102576,-2.952468122 54.332103183,-2.952391243 54.332193042,-2.95239332 54.332193649,-2.952316442 54.332238579,-2.95231748 54.332239186,-2.952240601 54.332284115,-2.95224164 54.332284722,-2.952164761 54.332329651,-2.952165799 54.332331472,-2.951935162 54.332376401,-2.9519362 54.332377008,-2.951859321 54.332421937,-2.951860359 54.332426789,-2.951245326 54.332471718,-2.951246363 54.332472931,-2.951092604 54.33251786,-2.951093641 54.332536627,-2.948710379 54.332716345,-2.948714518 54.332715741,-2.948791398 54.33276067,-2.948792432 54.332760066,-2.948869312 54.332894854,-2.948872416 54.33289425,-2.948949296 54.332984109,-2.948951366 54.332983504,-2.949028246 54.333118293,-2.949031351 54.333117688,-2.949108232 54.333162617,-2.949109267 54.333162012,-2.949186147 54.333251872,-2.949188218 54.333251267,-2.949265098 54.333296196,-2.949266134 54.333295591,-2.949343014 54.33338545,-2.949345085 54.333384845,-2.949421966 54.333519634,-2.949425072 54.333519029,-2.949501953 54.333698747,-2.949506095 54.333698142,-2.949582977 54.33387786,-2.949587119 54.333877255,-2.949664001 54.333967114,-2.949666073 54.333966509,-2.949742955 54.334056368,-2.949745026 54.334055763,-2.949821908 54.334145622,-2.94982398 54.334145017,-2.949900863 54.334234876,-2.949902934 54.33423427,-2.949979817 54.3342792,-2.949980853 54.334278594,-2.950057736 54.334368453,-2.950059808 54.334367848,-2.950136691 54.334457707,-2.950138763 54.334457101,-2.950215646 54.334546961,-2.950217719 54.334546355,-2.950294602 54.334591285,-2.950295638 54.334590679,-2.950372522 54.334635608,-2.950373558 54.334635003,-2.950450441 54.334724862,-2.950452514 54.33472365,-2.950606281 54.33476858,-2.950607318 54.334767368,-2.950761085 54.334812298,-2.950762122 54.334810479,-2.950992773 54.33476555,-2.950991736 54.334763731,-2.951222387 54.334718802,-2.951221349 54.334718196,-2.951298233 54.334673266,-2.951297195 54.33467266,-2.951374079 54.334582801,-2.951372004 54.334582194,-2.951448887 54.334492335,-2.951446812 54.334492942,-2.951369929 54.334313224,-2.951365778 54.33431383,-2.951288896 54.334134112,-2.951284746 54.334134718,-2.951207864 54.334089788,-2.951206826 54.334089182,-2.951283709 54.333909464,-2.951279559 54.333908251,-2.951433323 54.333863322,-2.951432285 54.333862715,-2.951509167 54.333772856,-2.951507092 54.333771037,-2.951737737 54.333726107,-2.951736699 54.333724894,-2.951890462 54.333635035,-2.951888386 54.333634428,-2.951965267 54.333589499,-2.951964229 54.333586465,-2.952348636 54.333541535,-2.952347597 54.333537285,-2.952885766 54.333492356,-2.952884727 54.333488104,-2.953422895 54.333443174,-2.953421855 54.333441351,-2.953652498 54.333486281,-2.953653538 54.333485673,-2.95373042 54.333440743,-2.95372938 54.333437704,-2.954113785 54.333482633,-2.954114825 54.333478375,-2.954652993 54.333523305,-2.954654034 54.333520262,-2.95503844 54.333475333,-2.955037398 54.333464369,-2.956421257 54.333509299,-2.9564223 54.33350686,-2.956729825 54.333551789,-2.956730868 54.33354935,-2.957038393 54.33359428,-2.957039436 54.33359306,-2.957193199 54.333503201,-2.957191111 54.333502591,-2.957267992 54.33359245,-2.95727008 54.333591229,-2.957423843 54.333726018,-2.957426975 54.333725408,-2.957503856 54.333770337,-2.957504901 54.333769727,-2.957581782 54.333859586,-2.957583871 54.333858975,-2.957660752 54.333903905,-2.957661797 54.333902684,-2.95781556 54.333992543,-2.95781765 54.333991933,-2.957894531 54.334036862,-2.957895576 54.334036251,-2.957972458 54.33412611,-2.957974548 54.3341255,-2.95805143 54.334170429,-2.958052475 54.334169819,-2.958129357 54.334214748,-2.958130402 54.334214137,-2.958207284 54.334259067,-2.958208329 54.334258456,-2.958285212 54.334303386,-2.958286257 54.334302775,-2.958363139 54.334482493,-2.95836732 54.334481271,-2.958521086 54.334526201,-2.958522131 54.33452559,-2.958599014 54.334570519,-2.958600059 54.334569908,-2.958676942 54.334614838,-2.958677988 54.334613615,-2.958831754 54.334658545,-2.9588328 54.334657934,-2.958909683 54.334702863,-2.958910729 54.334702252,-2.958987612 54.334747182,-2.958988658 54.33474657,-2.959065541 54.3347915,-2.959066587 54.334790277,-2.959220354 54.334835207,-2.9592214 54.334833984,-2.959375167 54.334878913,-2.959376214 54.334878302,-2.959453097 54.334969261,-2.959316799 54.334969995,-2.959224539 54.334925065,-2.959223492 54.334926288,-2.959069725 54.334881359,-2.959068679 54.334882581,-2.958914912 54.334837652,-2.958913866 54.334838263,-2.958836983 54.334793333,-2.958835937 54.334794555,-2.95868217 54.334749626,-2.958681124 54.334750237,-2.958604241 54.334705307,-2.958603196 54.334706529,-2.958449429 54.3346616,-2.958448384 54.334662821,-2.958294617 54.334617892,-2.958293572 54.334618503,-2.958216689 54.334573573,-2.958215644 54.334574794,-2.958061878 54.334440006,-2.958058744 54.334440617,-2.957981861 54.334395687,-2.957980816 54.334396298,-2.957903933 54.334351368,-2.957902889 54.334351979,-2.957826006 54.334307049,-2.957824961 54.33430827,-2.957671196 54.334263341,-2.957670152 54.334263951,-2.957593269 54.334219021,-2.957592225 54.334220852,-2.957361578 54.334130993,-2.95735949 54.334131603,-2.957282607 54.333951885,-2.957278432 54.333953105,-2.957124668 54.333818317,-2.957121536 54.333819537,-2.956967773 54.333864466,-2.956968817 54.333865076,-2.956891935 54.333775217,-2.956889848 54.333776437,-2.956736085 54.333731507,-2.956735041 54.333733946,-2.956427515 54.333689016,-2.956426472 54.333690845,-2.956195828 54.333645915,-2.956194785 54.333649571,-2.955733497 54.333604641,-2.955732455 54.333609513,-2.955117404 54.333654442,-2.955118446 54.333655051,-2.955041564 54.333610121,-2.955040523 54.33361073,-2.954963641 54.333655659,-2.954964683 54.333658702,-2.954580276 54.333703631,-2.954581317 54.33370424,-2.954504435 54.33365931,-2.954503394 54.333662351,-2.954118987 54.333617422,-2.954117947 54.333621677,-2.953579777 54.333666606,-2.953580817 54.333668429,-2.953350173 54.333713359,-2.953351212 54.333714574,-2.953197449 54.333759503,-2.953198489 54.333763755,-2.952660317 54.333718825,-2.952659278 54.333721253,-2.952351752 54.333766183,-2.95235279 54.33376679,-2.952275909 54.334216085,-2.952286294 54.334216692,-2.952209411 54.334306551,-2.952211488 54.334307158,-2.952134605 54.334397017,-2.952136682 54.334397624,-2.952059799 54.334442553,-2.952060838 54.33444316,-2.951983955 54.334488089,-2.951984993 54.334488696,-2.95190811 54.334533626,-2.951909148 54.334534232,-2.951832265 54.334579162,-2.951833303 54.334579768,-2.95175642 54.334624698,-2.951757458 54.334625305,-2.951680574 54.334670234,-2.951681612 54.334670841,-2.951604729 54.33471577,-2.951605767 54.334716377,-2.951528883 54.334761306,-2.951529921 54.334761913,-2.951453037 54.334806842,-2.951454075 54.334807448,-2.951377191 54.334852378,-2.951378229 54.334853591,-2.951224462 54.33489852,-2.951225499 54.334899126,-2.951148615 54.334988985,-2.95115069 54.334989592,-2.951073806 54.335034521,-2.951074843 54.335035127,-2.950997959 54.335080057,-2.950998996 54.335080663,-2.950922112 54.335035733,-2.950921075 54.335038157,-2.950613538 54.334993227,-2.950612502 54.334994439,-2.950458734 54.33494951,-2.950457697 54.334950115,-2.950380813 54.334905186,-2.950379777 54.334907608,-2.950072241 54.334862679,-2.950071205 54.334863889,-2.949917438 54.33481896,-2.949916402 54.334820171,-2.949762634 54.334954959,-2.949765742 54.334953749,-2.94991951 54.334998678,-2.949920546 54.334998073,-2.94999743 54.335177791,-2.950001574 54.335177185,-2.950078458 54.335267044,-2.950080531 54.335266439,-2.950157415 54.335311369,-2.950158451 54.335310763,-2.950235336 54.335355692,-2.950236372 54.335355087,-2.950313257 54.335400016,-2.950314293 54.335398199,-2.950544948 54.335443129,-2.950545984 54.335441917,-2.950699754 54.335486847,-2.950700791 54.335485029,-2.950931446 54.335754606,-2.950937668 54.335755212,-2.950860783 54.335845071,-2.950862857 54.335845677,-2.950785971 54.335935536,-2.950788045 54.335936142,-2.95071116 54.336026001,-2.950713233 54.336027213,-2.950559461 54.336072142,-2.950560498 54.336072748,-2.950483612 54.336162607,-2.950485685 54.336163213,-2.950408799 54.336253072,-2.950410872 54.336253678,-2.950333986 54.336343537,-2.950336059 54.336342931,-2.950412945 54.336612508,-2.950419165 54.336611902,-2.950496052 54.336746691,-2.950499162 54.336746085,-2.950576049 54.336835944,-2.950578123 54.336835338,-2.95065501 54.337104916,-2.950661231 54.33710431,-2.950738119 54.337284028,-2.950742267 54.33728524,-2.950588491 54.337330169,-2.950589527 54.337332592,-2.950281974 54.337377522,-2.95028301 54.337376916,-2.950359899 54.337556634,-2.950364045 54.337556028,-2.950440934 54.337600958,-2.95044197 54.337600352,-2.950518859 54.337645282,-2.950519896 54.337645887,-2.950443007 54.337825606,-2.950447154 54.337826211,-2.950370264 54.337871141,-2.950371301 54.337870535,-2.95044819 54.337915465,-2.950449227 54.337914253,-2.950603006 54.338004112,-2.95060508 54.3380029,-2.950758859 54.33804783,-2.950759896 54.338047224,-2.950836786 54.338226942,-2.950840934 54.338226336,-2.950917824 54.338450983,-2.95092301 54.338449165,-2.951153681 54.338494094,-2.951154719 54.338493488,-2.951231609 54.338538418,-2.951232647 54.338537811,-2.951309537 54.3386726,-2.95131265 54.338671387,-2.951466432 54.338716317,-2.95146747 54.338715104,-2.951621252 54.338804963,-2.951623328 54.338804356,-2.951700219 54.338759427,-2.951699181 54.33875882,-2.951776072 54.33880375,-2.95177711 54.338803143,-2.951854001 54.338848072,-2.951855039 54.338847466,-2.95193193 54.338937325,-2.951934007 54.338936111,-2.95208779 54.338981041,-2.952088828 54.338980434,-2.952165719 54.339025363,-2.952166758 54.33902415,-2.952320541 54.339114008,-2.952322619 54.339112794,-2.952476402 54.339202653,-2.95247848 54.339202046,-2.952555372 54.339246976,-2.952556411 54.339246369,-2.952633302 54.339381157,-2.95263642 54.33938055,-2.952713312 54.339560268,-2.952717469 54.339559661,-2.952794361 54.339829238,-2.952800597 54.33982863,-2.95287749 54.33987356,-2.952878529 54.339872952,-2.952955422 54.340054371,-2.952744279 54.339919704,-2.952725782 54.339920311,-2.952648889 54.339875381,-2.95264785 54.339875989,-2.952570957 54.339831059,-2.952569918 54.339831666,-2.952493025 54.339741807,-2.952490947 54.339742414,-2.952414054 54.339652555,-2.952411976 54.339653162,-2.952335084 54.339608233,-2.952334045 54.33960884,-2.952257152 54.339518981,-2.952255075 54.339519588,-2.952178183 54.339429729,-2.952176105 54.339430942,-2.952022321 54.339296154,-2.952019206 54.339297367,-2.951865422 54.339252438,-2.951864383 54.339253651,-2.951710599 54.339208722,-2.951709561 54.339209328,-2.95163267 54.339164399,-2.951631632 54.339165005,-2.95155474 54.339120076,-2.951553702 54.339120682,-2.95147681 54.339030823,-2.951474734 54.33903143,-2.951397843 54.338941571,-2.951395767 54.338942177,-2.951318876 54.338897247,-2.951317838 54.338897854,-2.951240947 54.338852924,-2.95123991 54.338854137,-2.951086127 54.338809207,-2.95108509 54.338809813,-2.951008199 54.338764884,-2.951007161 54.338766096,-2.950853379 54.338676237,-2.950851305 54.338677449,-2.950697523 54.33858759,-2.950695449 54.338590619,-2.950310996 54.33850076,-2.950308923 54.338502577,-2.950078251 54.338457647,-2.950077215 54.338458253,-2.950000324 54.338503182,-2.950001361 54.338503788,-2.94992447 54.338458858,-2.949923434 54.338461279,-2.949615872 54.33841635,-2.949614836 54.33841756,-2.949461055 54.338552349,-2.949464162 54.338552954,-2.949387271 54.338597884,-2.949388307 54.338596673,-2.949542088 54.338686532,-2.94954416 54.338685927,-2.949621051 54.338955504,-2.949627266 54.338956109,-2.949550374 54.339001039,-2.94955141 54.339001644,-2.949474518 54.339046574,-2.949475554 54.339047179,-2.949398662 54.339226897,-2.949402805 54.339227502,-2.949325913 54.339497079,-2.949332126 54.339494659,-2.949639695 54.339539588,-2.949640731 54.339538983,-2.949717624 54.339583912,-2.94971866 54.339584518,-2.949641767 54.339629447,-2.949642803 54.339631263,-2.949412125 54.339721122,-2.949414196 54.339721727,-2.949337303 54.339766656,-2.949338339 54.339767261,-2.949261446 54.339812191,-2.949262481 54.339812796,-2.949185588 54.339857725,-2.949186624 54.339858935,-2.949032837 54.339903865,-2.949033873 54.33990447,-2.948956979 54.339949399,-2.948958015 54.339950609,-2.948804228 54.339995538,-2.948805263 54.339996143,-2.94872837 54.340041073,-2.948729404 54.340041677,-2.948652511 54.340086607,-2.948653546 54.340087211,-2.948576652 54.340132141,-2.948577687 54.340132745,-2.948500793 54.340177675,-2.948501828 54.340178884,-2.94834804 54.340223813,-2.948349075 54.340224418,-2.948272181 54.340314277,-2.94827425 54.340314881,-2.948197356 54.340359811,-2.94819839 54.340360415,-2.948121496 54.340405345,-2.94812253 54.340405949,-2.948045636 54.340450879,-2.94804667 54.340451483,-2.947969776 54.340496412,-2.94797081 54.340497016,-2.947893916 54.340541946,-2.94789495 54.34054255,-2.947818055 54.340632409,-2.947820123 54.340633013,-2.947743229 54.340677943,-2.947744262 54.340678547,-2.947667368 54.340723476,-2.947668401 54.34072408,-2.947591507 54.34076901,-2.94759254 54.340769614,-2.947515645 54.340814543,-2.947516679 54.340815751,-2.947362889 54.340860681,-2.947363923 54.340861285,-2.947287028 54.341130862,-2.947293228 54.341130258,-2.947370123 54.341220117,-2.94737219 54.341219513,-2.947449086 54.341309372,-2.947451153 54.341308768,-2.947528049 54.341353698,-2.947529082 54.341353094,-2.947605978 54.341442953,-2.947608046 54.341442349,-2.947684942 54.341487279,-2.947685975 54.341486674,-2.947762872 54.341531604,-2.947763905 54.341531,-2.947840802 54.341620859,-2.94784287 54.341620255,-2.947919766 54.34206955,-2.947930107 54.342068946,-2.948007004 54.342293593,-2.948012175 54.342292989,-2.948089073 54.342337919,-2.948090107 54.342338523,-2.948013209 54.342383453,-2.948014243 54.342385265,-2.94778355 54.342430195,-2.947784584 54.342428986,-2.947938379 54.342608704,-2.947942516 54.342609309,-2.947865618 54.342654238,-2.947866652 54.342654842,-2.947789753 54.342699772,-2.947790787 54.342700376,-2.947713889 54.342745305,-2.947714923 54.342865142,-2.947333117 54.343391153,-2.946729907 54.343606936,-2.946719485 54.343859386,-2.946617608 54.344410422,-2.946261079 54.344680723,-2.94617499 54.344862732,-2.94588689 54.345177601,-2.945847972 54.345473291,-2.945962448 54.345771273,-2.945784689 54.345771876,-2.945707785 54.345816806,-2.945708817 54.345817408,-2.945631912 54.345907267,-2.945633976 54.34590787,-2.945557071 54.3459528,-2.945558103 54.345954005,-2.945404294 54.345998935,-2.945405325 54.345999537,-2.945328421 54.346044467,-2.945329452 54.346045672,-2.945175643 54.346315249,-2.94518183 54.346316454,-2.945028019 54.346361383,-2.945029051 54.346361986,-2.944952145 54.346406915,-2.944953176 54.346407517,-2.944876271 54.346452447,-2.944877302 54.346453651,-2.944723491 54.346635176,-2.944496896 54.346814894,-2.944501018 54.346816098,-2.944347206 54.346861028,-2.944348236 54.346861629,-2.94427133 54.346951489,-2.94427339 54.34695209,-2.944196484 54.34699702,-2.944197514 54.346998825,-2.943966794 54.347043755,-2.943967824 54.347047966,-2.943429478 54.347182754,-2.943432566 54.347452933,-2.943361835 54.347587721,-2.943364923 54.34758712,-2.94344183 54.347721909,-2.943444918 54.347721307,-2.943521826 54.347811166,-2.943523885 54.347810565,-2.943600793 54.347990283,-2.943604912 54.347989682,-2.94368182 54.348034611,-2.94368285 54.34803401,-2.943759758 54.348078939,-2.943760788 54.348078337,-2.943837697 54.348168197,-2.943839756 54.348167595,-2.943916665 54.348392243,-2.943921815 54.348392844,-2.943844906 54.348527633,-2.943847995 54.348528234,-2.943771086 54.348573164,-2.943772116 54.348573766,-2.943695206 54.348483906,-2.943693147 54.348710118,-2.94349833 54.348889475,-2.943548594 54.349320558,-2.943589242 54.349608348,-2.943565067 54.349633982,-2.94373489 54.349939503,-2.943741893 54.350093105,-2.943637717 54.350131214,-2.943361656 54.350065218,-2.942606268 54.350120695,-2.94240753 54.350221222,-2.942194437 54.350626909,-2.942034481 54.350754513,-2.941806619 54.351079686,-2.941598659 54.351493398,-2.941561963 54.351482865,-2.940607806 54.351721654,-2.939951671 54.351891427,-2.940078634 54.351964626,-2.941065004 54.352274812,-2.941625992 54.352409601,-2.941629074 54.352409,-2.941705991 54.35245393,-2.941707019 54.352999188,-2.942088762 54.354246307,-2.942363489 54.354245706,-2.94244041 54.354335565,-2.942442467 54.354334964,-2.942519387 54.354559611,-2.94252453 54.354559011,-2.942601451 54.35460394,-2.94260248 54.354603339,-2.9426794 54.354783057,-2.942683516 54.355061139,-2.942751432 54.355241338,-2.94269401 54.355368342,-2.942543046 54.355368943,-2.942466124 54.355413873,-2.942467153 54.355414474,-2.942390231 54.355459403,-2.942391259 54.355460004,-2.942314337 54.355504933,-2.942315365 54.355505534,-2.942238443 54.355550463,-2.942239471 54.355551064,-2.942162548 54.355595994,-2.942163577 54.355596594,-2.942086654 54.355641524,-2.942087682 54.355642124,-2.94201076 54.355776913,-2.942013844 54.355777513,-2.941936921 54.355912302,-2.941940005 54.355912902,-2.941863082 54.356002761,-2.941865138 54.356003362,-2.941788215 54.356048291,-2.941789242 54.356048891,-2.941712319 54.35613875,-2.941714375 54.356139951,-2.941560527 54.356184881,-2.941561555 54.356185481,-2.941484631 54.35623041,-2.941485659 54.356231611,-2.941331811 54.35632147,-2.941333866 54.35632207,-2.941256942 54.356456858,-2.941260024 54.356456258,-2.941336948 54.356591047,-2.94134003 54.356590446,-2.941416955 54.356635376,-2.941417982 54.356634776,-2.941494907 54.356724635,-2.941496962 54.356724035,-2.941573887 54.356768964,-2.941574915 54.356768364,-2.94165184 54.356858223,-2.941653895 54.356857622,-2.94173082 54.356902552,-2.941731848 54.356901952,-2.941808773 54.35703674,-2.941811857 54.35703614,-2.941888782 54.357125999,-2.941890839 54.357125398,-2.941967764 54.357170328,-2.941968792 54.357169727,-2.942045718 54.357259586,-2.942047774 54.357258985,-2.9421247 54.357303915,-2.942125728 54.357303314,-2.942202654 54.357348244,-2.942203683 54.357347643,-2.942280609 54.357392573,-2.942281637 54.357391972,-2.942358563 54.357436901,-2.942359592 54.357436301,-2.942436518 54.35748123,-2.942437546 54.357480629,-2.942514473 54.357570488,-2.94251653 54.357569887,-2.942593456 54.357749605,-2.942597572 54.357750206,-2.942520645 54.357840065,-2.942522703 54.357841267,-2.942368849 54.357886196,-2.942369878 54.357886797,-2.942292951 54.357976656,-2.942295008 54.357977857,-2.942141154 54.358157575,-2.942145267 54.358158176,-2.94206834 54.358517612,-2.942076566 54.358517011,-2.942153494 54.35856194,-2.942154523 54.35856134,-2.942231451 54.358606269,-2.942232479 54.358605669,-2.942309408 54.358650598,-2.942310436 54.358649997,-2.942387364 54.358874645,-2.942392508 54.358875245,-2.942315579 54.359144822,-2.942321751 54.359145423,-2.942244821 54.359280211,-2.942247907 54.359280812,-2.942170977 54.3594156,-2.942174063 54.35941488,-2.942266378 54.35946029,-2.942205863 54.359461131,-2.942098161 54.359595919,-2.942101246 54.35959652,-2.942024316 54.359866096,-2.942030486 54.359865496,-2.942107417 54.360000284,-2.942110502 54.359999684,-2.942187433 54.360089542,-2.94218949 54.360088942,-2.942266421 54.36022373,-2.942269507 54.360223129,-2.942346438 54.360402847,-2.942350553 54.360402246,-2.942427484 54.360581964,-2.942431599 54.360581363,-2.942508531 54.360761081,-2.942512647 54.36076048,-2.942589579 54.360850339,-2.942591637 54.360849738,-2.94266857 54.360984527,-2.942671657 54.360985128,-2.942594724 54.361119916,-2.942597811 54.361120517,-2.942520878 54.361210376,-2.942522936 54.361210977,-2.942446003 54.361570412,-2.942454233 54.361569811,-2.942531167 54.361974176,-2.942540428 54.361973576,-2.942617362 54.362198223,-2.942622507 54.362197622,-2.942699442 54.36233241,-2.94270253 54.362331809,-2.942779465 54.362376738,-2.942780494 54.362376137,-2.94285743 54.362421067,-2.942858459 54.362420466,-2.942935394 54.362465395,-2.942936424 54.362464794,-2.943013359 54.362599582,-2.943016448 54.362598981,-2.943093383 54.36268884,-2.943095442 54.362688239,-2.943172378 54.362823027,-2.943175467 54.362822426,-2.943252404 54.363002143,-2.943256523 54.363001542,-2.943333459 54.363405907,-2.943342728 54.363405305,-2.943419665 54.363629952,-2.943424815 54.363629351,-2.943501753 54.363764139,-2.943504843 54.363764741,-2.943427905 54.363854599,-2.943429965 54.363855201,-2.943353027 54.364034919,-2.943357147 54.36403552,-2.943280208 54.364350026,-2.943287417 54.364350627,-2.943210478 54.364395557,-2.943211508 54.364394955,-2.943288447 54.364950276,-2.943532038 54.365597981,-2.943454543 54.366098664,-2.943789237 54.366405267,-2.943657747 54.367076199,-2.944057923 54.367400653,-2.943942234 54.367672997,-2.94359447 54.367673599,-2.943517525 54.367763458,-2.943519585 54.367764059,-2.94344264 54.367853918,-2.9434447 54.367853317,-2.943521646 54.367943175,-2.943523706 54.367942574,-2.943600652 54.367987503,-2.943601682 54.367986901,-2.943678628 54.368031831,-2.943679658 54.368032432,-2.943602713 54.368167221,-2.943605804 54.368167822,-2.943528857 54.368212752,-2.943529888 54.36821215,-2.943606834 54.368302009,-2.943608895 54.368301407,-2.943685841 54.368391266,-2.943687902 54.368390664,-2.943764848 54.368435594,-2.943765879 54.368436195,-2.943688932 54.368705772,-2.943695115 54.368706975,-2.943541221 54.369075275,-2.943565059 54.369158073,-2.94332068 54.369158675,-2.943243732 54.369248533,-2.943245792 54.369249135,-2.943168844 54.369294064,-2.943169874 54.369293463,-2.943246822 54.369383322,-2.943248882 54.36938272,-2.943325831 54.36942765,-2.943326861 54.369427048,-2.943403809 54.369471977,-2.943404839 54.369472579,-2.943327891 54.369517508,-2.943328921 54.36951811,-2.943251972 54.369563039,-2.943253002 54.369562438,-2.943329951 54.369787085,-2.943335101 54.369786483,-2.943412051 54.369831412,-2.943413081 54.369832014,-2.943336132 54.370056661,-2.943341282 54.370056059,-2.943418232 54.370370565,-2.943425443 54.370371166,-2.943348493 54.370595813,-2.943353644 54.372329124,-2.943516537 54.372622205,-2.942815151 54.373027771,-2.942670507 54.373700149,-2.942886037 54.373698946,-2.94303995 54.373923593,-2.9430451 54.373924194,-2.942968143 54.375849659,-2.943843603 54.377470003,-2.943511293 54.377469401,-2.943588256 54.377514331,-2.943589287 54.377513729,-2.943666251 54.377873163,-2.943674497 54.377873765,-2.943597533 54.378008553,-2.943600625 54.37800735,-2.943754554 54.378097208,-2.943756615 54.378096606,-2.94383358 54.378006748,-2.943831518 54.378006146,-2.943908483 54.378140934,-2.943911576 54.378140332,-2.943988541 54.37827512,-2.943991634 54.378274518,-2.944068599 54.379173104,-2.944089224 54.379172502,-2.944166191 54.379217431,-2.944167222 54.379218033,-2.944090256 54.379307892,-2.944092318 54.379578792,-2.943929178 54.380080194,-2.944171632 54.38084339,-2.944266136 54.381376445,-2.943908849 54.381466304,-2.943910911 54.381465702,-2.943987882 54.381690348,-2.943993039 54.38169095,-2.943916067 54.381780809,-2.94391813 54.38178141,-2.943841158 54.38182634,-2.943842189 54.381826942,-2.943765218 54.381871871,-2.943766249 54.381872473,-2.943689277 54.381917402,-2.943690308 54.3819168,-2.94376728 54.381961729,-2.943768311 54.381961128,-2.943845283 54.382140845,-2.943849407 54.382140243,-2.943926379 54.382364889,-2.943931536 54.382365491,-2.943854563 54.38245535,-2.943856625 54.382456553,-2.94370268 54.382501483,-2.943703711 54.382498473,-2.944088575 54.382543402,-2.944089607 54.382540994,-2.944397499 54.382585923,-2.94439853 54.382585321,-2.944475503 54.38263025,-2.944476535 54.382628443,-2.944707455 54.382718301,-2.944709519 54.382717699,-2.944786492 54.382762628,-2.944787524 54.382762026,-2.944864498 54.383211318,-2.944874821 54.383211921,-2.944797846 54.383346709,-2.944800943 54.383346106,-2.944877918 54.383480894,-2.944881014 54.383480291,-2.944957989 54.38357015,-2.944960054 54.383569547,-2.945037029 54.383659406,-2.945039094 54.383658803,-2.945116069 54.383793591,-2.945119167 54.383792988,-2.945196142 54.383882846,-2.945198207 54.383882244,-2.945275183 54.383972102,-2.945277248 54.383971499,-2.945354224 54.384016429,-2.945355257 54.384015826,-2.945432233 54.384060755,-2.945433266 54.384060152,-2.945510241 54.38415001,-2.945512307 54.384149407,-2.945589283 54.384284195,-2.945592383 54.384283592,-2.945669359 54.384373451,-2.945671425 54.384372848,-2.945748402 54.384462706,-2.945750468 54.384460293,-2.946058375 54.384505222,-2.946059408 54.384504619,-2.946136385 54.384549548,-2.946137419 54.384548945,-2.946214395 54.384593874,-2.946215429 54.384592667,-2.946369383 54.384682526,-2.946371451 54.384681922,-2.946448428 54.384726851,-2.946449462 54.384726248,-2.946526439 54.384771177,-2.946527473 54.384770573,-2.94660445 54.384860432,-2.946606518 54.384859828,-2.946683496 54.384949686,-2.946685564 54.384949083,-2.946762542 54.38508387,-2.946765645 54.385083267,-2.946842623 54.385173125,-2.946844692 54.385172521,-2.94692167 54.38526238,-2.946923739 54.385261776,-2.947000717 54.385306705,-2.947001752 54.385306101,-2.94707873 54.38535103,-2.947079765 54.385350426,-2.947156743 54.385395355,-2.947157778 54.385394751,-2.947234756 54.38548461,-2.947236826 54.385484006,-2.947313804 54.385528935,-2.947314839 54.385528331,-2.947391818 54.385618189,-2.947393888 54.385617585,-2.947470867 54.385662514,-2.947471902 54.38566191,-2.947548881 54.385706839,-2.947549916 54.385706235,-2.947626895 54.385751164,-2.94762793 54.385751768,-2.947550951 54.385841627,-2.947553022 54.385842231,-2.947476043 54.386201665,-2.947484325 54.38620106,-2.947561305 54.386605423,-2.947570623 54.386606027,-2.947493643 54.386785744,-2.947497784 54.386786348,-2.947420803 54.386831278,-2.947421838 54.386832486,-2.947267876 54.386967274,-2.947270982 54.386966669,-2.947347963 54.387101457,-2.947351068 54.387100853,-2.94742805 54.387145782,-2.947429085 54.387145178,-2.947506066 54.387235036,-2.947508137 54.387234432,-2.947585119 54.387279361,-2.947586154 54.387278757,-2.947663136 54.387323686,-2.947664171 54.387323082,-2.947741153 54.387368011,-2.947742189 54.387367406,-2.947819171 54.387412335,-2.947820207 54.387408708,-2.948282099 54.387453637,-2.948283135 54.387453032,-2.948360117 54.387497961,-2.948361153 54.387494937,-2.948746064 54.387539866,-2.948747101 54.387538656,-2.948901065 54.387583585,-2.948902102 54.38758298,-2.948979084 54.387987342,-2.948988417 54.387986737,-2.9490654 54.388076595,-2.949067474 54.38807599,-2.949144457 54.388120919,-2.949145495 54.388120314,-2.949222478 54.388210172,-2.949224552 54.388208961,-2.949378519 54.38829882,-2.949380594 54.388298214,-2.949457578 54.388343143,-2.949458615 54.388342538,-2.949535599 54.388432396,-2.949537674 54.388431185,-2.949691642 54.388476114,-2.94969268 54.388475508,-2.949769664 54.388565367,-2.949771739 54.388564761,-2.949848724 54.38860969,-2.949849761 54.388609084,-2.949926746 54.388698943,-2.949928822 54.388697731,-2.95008279 54.38874266,-2.950083829 54.388742054,-2.950160813 54.388831912,-2.95016289 54.388831306,-2.950239874 54.388876236,-2.950240913 54.38887563,-2.950317897 54.388965488,-2.950319974 54.388964276,-2.950473944 54.389054134,-2.950476021 54.389053528,-2.950553006 54.389098457,-2.950554045 54.389097851,-2.95063103 54.389187709,-2.950633108 54.389187103,-2.950710093 54.389232032,-2.950711132 54.389230819,-2.950865103 54.389320677,-2.950867181 54.389320071,-2.950944166 54.389365,-2.950945206 54.389364394,-2.951022191 54.389454252,-2.95102427 54.389453645,-2.951101256 54.389498574,-2.951102295 54.389497968,-2.951179281 54.389542897,-2.95118032 54.38954229,-2.951257306 54.389632149,-2.951259385 54.389631542,-2.951336371 54.389766329,-2.95133949 54.389765722,-2.951416476 54.38990051,-2.951419596 54.389899903,-2.951496582 54.39003469,-2.951499702 54.390032263,-2.951807649 54.390077192,-2.951808689 54.390076585,-2.951885676 54.390121514,-2.951886716 54.3901203,-2.95204069 54.390165229,-2.952041731 54.390164622,-2.952118718 54.390209551,-2.952119758 54.390208336,-2.952273732 54.390253265,-2.952274773 54.39025205,-2.952428748 54.39029698,-2.952429788 54.390295765,-2.952583763 54.390340694,-2.952584804 54.390340086,-2.952661791 54.390385015,-2.952662832 54.390384408,-2.95273982 54.390429337,-2.952740861 54.390428729,-2.952817849 54.390518587,-2.952819931 54.39051798,-2.952896919 54.390562909,-2.95289796 54.390562301,-2.952974948 54.39060723,-2.952975989 54.390606622,-2.953052977 54.390561693,-2.953051936 54.390561085,-2.953128924 54.390606015,-2.953129965 54.390605407,-2.953206953 54.390650336,-2.953207995 54.39064912,-2.953361971 54.390694049,-2.953363013 54.390693441,-2.953440001 54.390873157,-2.953444168 54.390872549,-2.953521157 54.39082762,-2.953520115 54.390827012,-2.953597103 54.390782083,-2.953596061 54.390781475,-2.953673049 54.390826404,-2.953674091 54.390825795,-2.95375108 54.390870725,-2.953752122 54.390870116,-2.95382911 54.390915045,-2.953830153 54.390914437,-2.953907141 54.390959366,-2.953908183 54.390958758,-2.953985172 54.391138474,-2.953989342 54.391139691,-2.953835365 54.39118462,-2.953836407 54.391185228,-2.953759418 54.391230157,-2.95376046 54.391230765,-2.953683471 54.391320624,-2.953685556 54.391320015,-2.953762545 54.391364944,-2.953763587 54.391363728,-2.953917566 54.391453586,-2.953919651 54.391452978,-2.95399664 54.391228332,-2.953991427 54.391227724,-2.954068416 54.391092936,-2.954065288 54.391092328,-2.954142277 54.391047399,-2.954141234 54.39104679,-2.954218223 54.391091719,-2.954219266 54.391091111,-2.954296255 54.39113604,-2.954297298 54.391134823,-2.954451275 54.391179752,-2.954452318 54.391178534,-2.954606296 54.391223463,-2.954607339 54.391222245,-2.954761317 54.391267175,-2.954762361 54.391265348,-2.954993328 54.391310277,-2.954994372 54.391309668,-2.955071361 54.391354597,-2.955072405 54.391353987,-2.955149394 54.391488775,-2.955152525 54.391488165,-2.955229515 54.391533095,-2.955230559 54.391532485,-2.955307548 54.391577414,-2.955308592 54.391576805,-2.955385582 54.391666663,-2.95538767 54.391666054,-2.95546466 54.391710983,-2.955465704 54.391710373,-2.955542694 54.391800232,-2.955544783 54.391799013,-2.955698763 54.391843942,-2.955699807 54.391843332,-2.955776797 54.391888261,-2.955777842 54.391887652,-2.955854832 54.391932581,-2.955855877 54.391931361,-2.956009857 54.39197629,-2.956010902 54.391975071,-2.956164883 54.39202,-2.956165928 54.39201756,-2.95647389 54.392107418,-2.95647598 54.392106198,-2.956629962 54.392151127,-2.956631007 54.392150517,-2.956707998 54.392195446,-2.956709043 54.392194836,-2.956786034 54.392239765,-2.95678708 54.392239155,-2.956864071 54.392284084,-2.956865116 54.392283474,-2.956942107 54.392328403,-2.956943153 54.392325961,-2.957251117 54.39237089,-2.957252163 54.392370279,-2.957329154 54.392415208,-2.957330201 54.392414598,-2.957407192 54.392459527,-2.957408238 54.392458916,-2.957485229 54.392503845,-2.957486276 54.392502624,-2.957640259 54.392547553,-2.957641305 54.392546942,-2.957718297 54.392591871,-2.957719343 54.39259126,-2.957796335 54.392636189,-2.957797382 54.392635578,-2.957874373 54.392680507,-2.95787542 54.392679286,-2.958029403 54.392724215,-2.95803045 54.392723604,-2.958107442 54.392768533,-2.958108489 54.392767921,-2.958185481 54.392857779,-2.958187575 54.392857168,-2.958264567 54.392902097,-2.958265615 54.392901486,-2.958342607 54.392946415,-2.958343654 54.392945804,-2.958420646 54.392990733,-2.958421694 54.392990122,-2.958498686 54.39307998,-2.958500781 54.393079368,-2.958577773 54.393124297,-2.958578821 54.393121851,-2.958886791 54.393211709,-2.958888887 54.393211098,-2.95896588 54.393256027,-2.958966928 54.393254804,-2.959120913 54.393299733,-2.959121961 54.393299121,-2.959198954 54.39334405,-2.959200002 54.393342826,-2.959353988 54.393387755,-2.959355037 54.393386531,-2.959509023 54.39343146,-2.959510071 54.393813442,-2.960073445 54.394236877,-2.959944719 54.394400335,-2.95973291 54.394831776,-2.959727581 54.395094246,-2.96062703 54.395201007,-2.961892499 54.395437731,-2.962637345 54.395785106,-2.963030539 54.395874964,-2.963032644 54.39587435,-2.963109642 54.395919279,-2.963110695 54.395918664,-2.963187692 54.396008522,-2.963189798 54.396009137,-2.9631128 54.396054066,-2.963113853 54.396053451,-2.963190851 54.396143309,-2.963192956 54.396143923,-2.963115958 54.396233781,-2.963118064 54.39623501,-2.962964067 54.396279939,-2.96296512 54.396280553,-2.962888122 54.396325482,-2.962889174 54.396327324,-2.962658179 54.396372253,-2.962659231 54.396372867,-2.962582233 54.396417796,-2.962583285 54.396420865,-2.962198292 54.396465794,-2.962199344 54.396468248,-2.961891349 54.396513177,-2.961892401 54.39651379,-2.961815402 54.396603648,-2.961817505 54.396603035,-2.961894504 54.396647964,-2.961895555 54.396648577,-2.961818556 54.396783364,-2.96182171 54.396783977,-2.961744711 54.397143409,-2.961753121 54.397142796,-2.961830121 54.39736744,-2.961835378 54.397366827,-2.961912378 54.397411756,-2.96191343 54.397412369,-2.961836429 54.397502227,-2.961838532 54.397501613,-2.961915532 54.397591471,-2.961917635 54.397590244,-2.962071637 54.397725031,-2.962074792 54.397725645,-2.961997791 54.397860431,-2.962000946 54.397859204,-2.962154948 54.397904133,-2.962156 54.397902906,-2.962310003 54.398037692,-2.962313158 54.398037078,-2.96239016 54.398126936,-2.962392264 54.398128164,-2.96223826 54.398218022,-2.962240364 54.398217408,-2.962317366 54.398262337,-2.962318418 54.398261723,-2.96239542 54.398171865,-2.962393316 54.398171251,-2.962470318 54.39821618,-2.96247137 54.398214952,-2.962625374 54.398259881,-2.962626426 54.398261109,-2.962472422 54.398665469,-2.962481891 54.398666083,-2.962404888 54.399234029,-2.962187133 54.399589137,-2.961610103 54.400001957,-2.960556891 54.400135518,-2.960714053 54.40015496,-2.960529659 54.400525093,-2.960322652 54.400580967,-2.960077492 54.400648031,-2.959555317 54.400782818,-2.959558464 54.400814598,-2.96008295 54.401011305,-2.960210779 54.401190286,-2.960307387 54.401215038,-2.960585245 54.401329524,-2.960880605 54.401738456,-2.961444731 54.402094821,-2.961838186 54.402397023,-2.962261189 54.402772216,-2.962547264 54.402746628,-2.963501772 54.403016201,-2.963508092 54.403015586,-2.963585103 54.403060515,-2.963586157 54.4030599,-2.963663168 54.403104829,-2.963664221 54.403104214,-2.963741232 54.403149143,-2.963742286 54.403148529,-2.963819297 54.403238386,-2.963821405 54.403237772,-2.963898416 54.4032827,-2.96389947 54.403282086,-2.963976481 54.403327014,-2.963977535 54.403326399,-2.964054547 54.403371328,-2.964055601 54.403370713,-2.964132612 54.403460571,-2.964134721 54.403459956,-2.964211732 54.403594743,-2.964214895 54.403595358,-2.964137883 54.403640286,-2.964138937 54.403640901,-2.964061925 54.40368583,-2.964062979 54.403686445,-2.963985967 54.403776303,-2.963988075 54.403776918,-2.963911063 54.403821847,-2.963912117 54.403822461,-2.963835104 54.403777533,-2.96383405 54.403778762,-2.963680026 54.403733833,-2.963678972 54.403735063,-2.963524948 54.403779991,-2.963526001 54.403780606,-2.963448989 54.403735677,-2.963447935 54.403736906,-2.963293911 54.403691977,-2.963292858 54.403693206,-2.963138833 54.403783064,-2.96314094 54.403782449,-2.963217952 54.403872307,-2.963220058 54.403872921,-2.963143046 54.40391785,-2.963144099 54.403917236,-2.963221111 54.403962165,-2.963222164 54.403962779,-2.963145152 54.404142495,-2.963149364 54.404143109,-2.963072351 54.404277896,-2.96307551 54.40427851,-2.962998497 54.404368368,-2.963000603 54.404368982,-2.962923589 54.404503768,-2.962926748 54.404504383,-2.962849734 54.404549311,-2.962850787 54.404549926,-2.962773773 54.404594854,-2.962774826 54.404595469,-2.962697812 54.404640397,-2.962698864 54.404642239,-2.962467823 54.404687168,-2.962468875 54.404697382,-2.962315057 54.404823183,-2.962318003 54.404822569,-2.962395018 54.404867498,-2.96239607 54.404866884,-2.962473084 54.405046599,-2.962477294 54.405045985,-2.962554309 54.405090914,-2.962555361 54.405089686,-2.962709391 54.405134615,-2.962710443 54.405133387,-2.962864473 54.405178315,-2.962865526 54.405176473,-2.963096571 54.40526633,-2.963098677 54.405263873,-2.963406737 54.405218944,-2.963405684 54.405218329,-2.963482699 54.405263258,-2.963483752 54.405262644,-2.963560767 54.405307572,-2.963561821 54.405306958,-2.963638836 54.405351887,-2.96363989 54.405351272,-2.963716905 54.405396201,-2.963717959 54.405395586,-2.963794974 54.405440515,-2.963796028 54.4054399,-2.963873043 54.405484829,-2.963874097 54.405484214,-2.963951113 54.405529143,-2.963952167 54.405528528,-2.964029182 54.405573457,-2.964030236 54.405572842,-2.964107252 54.405617771,-2.964108306 54.405617156,-2.964185322 54.405662084,-2.964186376 54.405661469,-2.964263392 54.405751327,-2.964265501 54.405750097,-2.964419533 54.405795026,-2.964420587 54.40579441,-2.964497603 54.405839339,-2.964498658 54.405838724,-2.964575674 54.405883653,-2.964576729 54.405883038,-2.964653745 54.405972895,-2.964655855 54.40597228,-2.964732871 54.406107066,-2.964736036 54.406106451,-2.964813052 54.406331095,-2.964818327 54.40633171,-2.96474131 54.406466497,-2.964744475 54.406467112,-2.964667458 54.40655697,-2.964669567 54.406557585,-2.96459255 54.406602514,-2.964593604 54.406603129,-2.964516587 54.406648058,-2.964517642 54.406648673,-2.964440624 54.406918246,-2.964446951 54.406918862,-2.964369933 54.407098577,-2.964374151 54.407097962,-2.96445117 54.407367535,-2.964457498 54.407366919,-2.964534517 54.407456777,-2.964536626 54.407455546,-2.964690664 54.407500475,-2.964691719 54.40749986,-2.964768739 54.407544789,-2.964769794 54.407543558,-2.964923832 54.407633415,-2.964925943 54.4076328,-2.965002962 54.407722657,-2.965005072 54.407722042,-2.965082092 54.407766971,-2.965083147 54.407766355,-2.965160167 54.407856213,-2.965162278 54.407854981,-2.965316318 54.40789991,-2.965317373 54.407899294,-2.965394393 54.407944223,-2.965395449 54.407942991,-2.965549489 54.40798792,-2.965550545 54.407986688,-2.965704585 54.408031617,-2.965705641 54.408031001,-2.965782662 54.407986072,-2.965781606 54.40798484,-2.965935646 54.407939911,-2.96593459 54.407938679,-2.96608863 54.407848821,-2.966086517 54.407848205,-2.966163537 54.407803276,-2.96616248 54.40780266,-2.9662395 54.407667873,-2.96623633 54.407667257,-2.96631335 54.40753247,-2.96631018 54.407533087,-2.96623316 54.407173656,-2.966224708 54.407174273,-2.966147689 54.406994558,-2.966143463 54.406993941,-2.966220482 54.406904084,-2.966218369 54.406903467,-2.966295387 54.406858538,-2.96629433 54.406857922,-2.966371348 54.406812993,-2.966370291 54.406812377,-2.966447309 54.406767448,-2.966446253 54.406766831,-2.96652327 54.406721903,-2.966522214 54.406721286,-2.966599231 54.406676357,-2.966598174 54.406673891,-2.966906245 54.406628962,-2.966905188 54.406626494,-2.967213258 54.406671423,-2.967214316 54.406670189,-2.967368351 54.406715118,-2.967369409 54.406714501,-2.967446426 54.40675943,-2.967447484 54.406758812,-2.967524502 54.40684867,-2.967526618 54.406848053,-2.967603636 54.40693791,-2.967605752 54.406937293,-2.96768277 54.407027151,-2.967684886 54.407026533,-2.967761905 54.407071462,-2.967762963 54.407070845,-2.967839981 54.407340418,-2.967846331 54.4073398,-2.96792335 54.407384729,-2.967924409 54.407384111,-2.968001428 54.40742904,-2.968002486 54.407428423,-2.968079505 54.40715885,-2.968073154 54.407159468,-2.967996135 54.40706961,-2.967994018 54.407070227,-2.967917 54.40698037,-2.967914883 54.406979752,-2.967991901 54.406934824,-2.967990843 54.406933589,-2.968144879 54.40688866,-2.96814382 54.406888042,-2.968220838 54.406843114,-2.96821978 54.406842496,-2.968296798 54.406752638,-2.96829468 54.406752021,-2.968371698 54.406662163,-2.96836958 54.406661545,-2.968446598 54.40648183,-2.968442362 54.406482448,-2.968365344 54.40639259,-2.968363227 54.406393208,-2.96828621 54.406348279,-2.968285151 54.406348897,-2.968208134 54.406259039,-2.968206016 54.406259657,-2.968129 54.406035013,-2.968123707 54.406035631,-2.96804669 54.405990702,-2.968045632 54.405991319,-2.967968615 54.405856533,-2.96796544 54.40585715,-2.967888424 54.405767293,-2.967886307 54.40576791,-2.967809291 54.405633124,-2.967806117 54.405633741,-2.967729101 54.405588812,-2.967728043 54.40558943,-2.967651027 54.405544501,-2.967649969 54.405545118,-2.967572954 54.40545526,-2.967570838 54.405455878,-2.967493822 54.40536602,-2.967491707 54.405366637,-2.967414691 54.405052136,-2.967407287 54.405051518,-2.967484302 54.404737017,-2.967476897 54.4047364,-2.967553911 54.404421898,-2.967546505 54.404420664,-2.967700532 54.404375735,-2.967699474 54.404376352,-2.967622461 54.404241566,-2.967619287 54.404240948,-2.9676963 54.404016304,-2.96769101 54.404015687,-2.967768023 54.403880901,-2.967764848 54.403881518,-2.967687836 54.403746732,-2.967684662 54.403746114,-2.967761674 54.403611328,-2.9677585 54.403611945,-2.967681488 54.403477159,-2.967678314 54.403476542,-2.967755325 54.403386684,-2.967753209 54.403387301,-2.967676198 54.403342372,-2.96767514 54.403341755,-2.967752151 54.40316204,-2.967747919 54.403161423,-2.96782493 54.402981707,-2.967820697 54.40298109,-2.967897708 54.402846304,-2.967894533 54.402845686,-2.967971544 54.402800757,-2.967970486 54.40280014,-2.968047496 54.402710282,-2.968045379 54.402709665,-2.968122389 54.402664736,-2.968121331 54.402664118,-2.968198341 54.402574261,-2.968196224 54.402573643,-2.968273234 54.402528714,-2.968272175 54.402528097,-2.968349185 54.402483168,-2.968348126 54.402438857,-2.968270058 54.402339519,-2.968329337 54.402302217,-2.968497911 54.40221236,-2.968495793 54.402211742,-2.968572802 54.402166813,-2.968571743 54.402050616,-2.968491981 54.402005564,-2.968506324 54.401941551,-2.968643457 54.401940316,-2.968797475 54.401850458,-2.968795357 54.40184984,-2.968872365 54.401804911,-2.968871306 54.401804293,-2.968948314 54.401714436,-2.968946196 54.401713818,-2.969023204 54.40162396,-2.969021085 54.401623342,-2.969098093 54.401533484,-2.969095974 54.401532866,-2.969172982 54.401308222,-2.969167685 54.401307604,-2.969244692 54.401172817,-2.969241513 54.401172199,-2.969318521 54.40112727,-2.969317461 54.401126652,-2.969394468 54.401036795,-2.969392349 54.401036424,-2.969438553 54.40114215,-2.969702921 54.401169726,-2.96962655 54.401259583,-2.96962867 54.401260202,-2.969551662 54.401394988,-2.969554842 54.401395607,-2.969477835 54.401530393,-2.969481014 54.401531011,-2.969404006 54.401620869,-2.969406126 54.401621487,-2.969329118 54.402070775,-2.969339715 54.402071393,-2.969262706 54.402475753,-2.969272243 54.402475134,-2.969349253 54.402520063,-2.969350313 54.402519445,-2.969427323 54.402564374,-2.969428383 54.402563755,-2.969505393 54.402608684,-2.969506453 54.402607447,-2.969660473 54.402652376,-2.969661533 54.40265052,-2.969892563 54.402695449,-2.969893623 54.402693593,-2.970124654 54.402738522,-2.970125714 54.402729854,-2.971203857 54.402684925,-2.971202795 54.402683066,-2.971433825 54.402638137,-2.971432763 54.402636898,-2.971586783 54.402591969,-2.971585721 54.40259135,-2.971662731 54.402546421,-2.971661669 54.402545801,-2.971738678 54.402500872,-2.971737616 54.402500252,-2.971814626 54.402455324,-2.971813563 54.402454704,-2.971890573 54.402364846,-2.971888448 54.402364226,-2.971965457 54.402184511,-2.971961207 54.402183891,-2.972038216 54.402094034,-2.97203609 54.402093414,-2.972113099 54.402048485,-2.972112037 54.402047865,-2.972189045 54.402002936,-2.972187983 54.402002316,-2.972264991 54.401957387,-2.972263928 54.401956767,-2.972340937 54.401911838,-2.972339874 54.401911218,-2.972416883 54.401866289,-2.97241582 54.401865049,-2.972569837 54.40182012,-2.972568773 54.4018195,-2.972645782 54.401864428,-2.972646845 54.401863808,-2.972723854 54.401908737,-2.972724917 54.401907496,-2.972878934 54.401952424,-2.972879998 54.401951183,-2.973034015 54.401996112,-2.973035079 54.401995491,-2.973112088 54.402175206,-2.973116343 54.402174586,-2.973193352 54.402264443,-2.97319548 54.402263822,-2.97327249 54.402308751,-2.973273554 54.40230813,-2.973350563 54.402353059,-2.973351627 54.402352438,-2.973428637 54.402397367,-2.973429701 54.402396746,-2.97350671 54.402441675,-2.973507775 54.402440433,-2.973661794 54.402485361,-2.973662858 54.402483498,-2.973893887 54.402528427,-2.973894952 54.402527184,-2.974048971 54.402572113,-2.974050036 54.40257087,-2.974204056 54.402615799,-2.974205121 54.402614556,-2.974359141 54.402659485,-2.974360206 54.402658863,-2.974437216 54.402703792,-2.974438281 54.402702549,-2.974592302 54.402747478,-2.974593367 54.402746234,-2.974747387 54.402791163,-2.974748453 54.402790541,-2.974825463 54.40283547,-2.974826529 54.402834848,-2.974903539 54.402879777,-2.974904605 54.402878533,-2.975058626 54.402923462,-2.975059692 54.402922218,-2.975213713 54.402967146,-2.975214779 54.402964036,-2.975599832 54.403008964,-2.975600898 54.403005852,-2.975985951 54.402960924,-2.975984884 54.402958433,-2.976292926 54.403003362,-2.976293993 54.403002739,-2.976371004 54.40295781,-2.976369937 54.402957187,-2.976446947 54.402912259,-2.976445879 54.402911636,-2.97652289 54.402866707,-2.976521822 54.402864838,-2.976752853 54.40281991,-2.976751785 54.402817417,-2.977059826 54.402772489,-2.977058758 54.402766878,-2.977751849 54.402721949,-2.97775078 54.402720701,-2.9779048 54.40276563,-2.977905869 54.402764383,-2.978059889 54.402809311,-2.978060958 54.402808687,-2.978137969 54.402943473,-2.978141177 54.402944721,-2.977987156 54.40298965,-2.977988225 54.402990274,-2.977911215 54.403035202,-2.977912284 54.403035826,-2.977835273 54.403305398,-2.977841688 54.403304774,-2.977918699 54.403754061,-2.977929391 54.403753437,-2.978006403 54.403798366,-2.978007472 54.40379899,-2.97793046 54.404023633,-2.977935806 54.404024257,-2.977858794 54.404069185,-2.977859863 54.404071056,-2.977628825 54.404115985,-2.977629894 54.404117856,-2.977398856 54.404162784,-2.977399925 54.404164031,-2.9772459 54.40420896,-2.977246968 54.404209583,-2.977169955 54.404254512,-2.977171024 54.404255758,-2.977016998 54.404300687,-2.977018066 54.40430131,-2.976941053 54.404346239,-2.976942121 54.404346862,-2.976865108 54.404436719,-2.976867244 54.404437342,-2.976790231 54.4045272,-2.976792367 54.404528446,-2.97663834 54.404573374,-2.976639408 54.404573997,-2.976562395 54.404618926,-2.976563462 54.404619549,-2.976486449 54.404709406,-2.976488584 54.404710029,-2.97641157 54.404754958,-2.976412638 54.404755581,-2.976335624 54.404800509,-2.976336692 54.404801132,-2.976259678 54.404846061,-2.976260745 54.404847306,-2.976106717 54.404892235,-2.976107784 54.40489348,-2.975953756 54.404938409,-2.975954823 54.404940276,-2.975723781 54.404985205,-2.975724848 54.404985827,-2.975647833 54.405030756,-2.9756489 54.405032,-2.975494871 54.405121858,-2.975497004 54.405123724,-2.975265961 54.405168653,-2.975267027 54.405169275,-2.975190013 54.405214204,-2.975191079 54.405214826,-2.975114064 54.405259754,-2.97511513 54.405260376,-2.975038115 54.405305305,-2.975039182 54.405305927,-2.974962167 54.405350856,-2.974963233 54.405351477,-2.974886218 54.405306549,-2.974885152 54.405307171,-2.974808137 54.405352099,-2.974809203 54.405353343,-2.974655173 54.405398271,-2.974656238 54.405400136,-2.974425193 54.405445065,-2.974426259 54.405445687,-2.974349243 54.405490615,-2.974350309 54.405491858,-2.974196278 54.405536787,-2.974197343 54.405537408,-2.974120328 54.405582337,-2.974121393 54.405582958,-2.974044378 54.405627887,-2.974045443 54.405628508,-2.973968427 54.405673437,-2.973969492 54.405674058,-2.973892477 54.405718987,-2.973893541 54.405719608,-2.973816526 54.405764537,-2.973817591 54.405765158,-2.973740575 54.405855015,-2.973742704 54.405855637,-2.973665688 54.405900565,-2.973666753 54.405901186,-2.973589737 54.405946115,-2.973590801 54.405947357,-2.973436769 54.405992286,-2.973437833 54.405993527,-2.973283801 54.406038456,-2.973284865 54.40604156,-2.972899784 54.405996631,-2.97289872 54.405997252,-2.972821704 54.40604218,-2.972822767 54.406045903,-2.972360669 54.406000974,-2.972359606 54.406001594,-2.97228259 54.406046523,-2.972283653 54.406047763,-2.97212962 54.406002834,-2.972128557 54.406004695,-2.971897508 54.405914837,-2.971895383 54.405916077,-2.971741351 54.405871148,-2.971740288 54.405871768,-2.971663272 54.405826839,-2.97166221 54.405827459,-2.971585194 54.40578253,-2.971584131 54.40578315,-2.971507115 54.405738221,-2.971506053 54.405739461,-2.971352021 54.405694532,-2.971350959 54.405697629,-2.970965881 54.405742558,-2.970966942 54.405743177,-2.970889927 54.405833035,-2.97089205 54.405840463,-2.969967858 54.40593032,-2.969969979 54.405930939,-2.969892963 54.406110654,-2.969897205 54.406110035,-2.969974221 54.406334679,-2.969979524 54.40633406,-2.970056541 54.406378989,-2.970057602 54.40637837,-2.970134619 54.406468228,-2.970136741 54.406467609,-2.970213758 54.406512538,-2.970214819 54.406511919,-2.970291836 54.406556848,-2.970292897 54.406556229,-2.970369914 54.406601157,-2.970370976 54.406600538,-2.970447993 54.406645467,-2.970449054 54.406644848,-2.970526072 54.406689777,-2.970527133 54.406689158,-2.97060415 54.406734086,-2.970605212 54.406733467,-2.970682229 54.406778396,-2.970683291 54.406777777,-2.970760309 54.406822706,-2.97076137 54.406822086,-2.970838388 54.406867015,-2.97083945 54.406866396,-2.970916468 54.406911325,-2.970917529 54.406910705,-2.970994547 54.406955634,-2.970995609 54.406955015,-2.971072627 54.406999943,-2.971073689 54.406999324,-2.971150707 54.407044253,-2.971151769 54.407043633,-2.971228787 54.40713349,-2.971230911 54.407132871,-2.97130793 54.407267657,-2.971311116 54.407267037,-2.971388135 54.407311966,-2.971389197 54.407311347,-2.971466216 54.407401204,-2.97146834 54.407400584,-2.971545359 54.407490442,-2.971547484 54.407489822,-2.971624503 54.407579679,-2.971626628 54.40757906,-2.971703647 54.407713846,-2.971706835 54.407713226,-2.971783854 54.407758155,-2.971784917 54.407757535,-2.971861937 54.40793725,-2.971866188 54.40793663,-2.971943208 54.408071416,-2.971946396 54.408070796,-2.972023416 54.408205582,-2.972026605 54.408204962,-2.972103625 54.408294819,-2.972105751 54.408291718,-2.972490854 54.408336647,-2.972491918 54.408336026,-2.972568938 54.408380955,-2.972570002 54.408380335,-2.972647023 54.408425263,-2.972648086 54.408424643,-2.972725107 54.408469572,-2.972726171 54.408468951,-2.972803192 54.40851388,-2.972804255 54.408513259,-2.972881276 54.408603117,-2.972883404 54.408602496,-2.972960425 54.40882714,-2.972965745 54.408826519,-2.973042767 54.408871448,-2.973043831 54.408870827,-2.973120853 54.408915756,-2.973121917 54.408915135,-2.973198938 54.408960063,-2.973200003 54.408958201,-2.973431068 54.40900313,-2.973432133 54.409000645,-2.97374022 54.408955717,-2.973739155 54.408954474,-2.973893199 54.408909546,-2.973892134 54.40890706,-2.97420022 54.408862131,-2.974199155 54.40885778,-2.974738306 54.408902709,-2.974739372 54.408900843,-2.974970437 54.408945772,-2.974971503 54.408944528,-2.975125547 54.408989457,-2.975126613 54.408988835,-2.975203635 54.409033763,-2.975204701 54.409035007,-2.975050658 54.409079936,-2.975051724 54.409128748,-2.975684584 54.409181791,-2.975793696 54.409244692,-2.97579519 54.409438121,-2.975214299 54.40948305,-2.975215366 54.409482428,-2.975292389 54.409527356,-2.975293455 54.409526734,-2.975370478 54.409616592,-2.975372611 54.409615969,-2.975449634 54.409750755,-2.975452834 54.409750133,-2.975529858 54.409795062,-2.975530925 54.409794439,-2.975607948 54.409839368,-2.975609015 54.409838746,-2.975686038 54.409883674,-2.975687105 54.409883052,-2.975764129 54.409972909,-2.975766263 54.409972287,-2.975843286 54.410017215,-2.975844354 54.410016593,-2.975921377 54.410151379,-2.975924579 54.410150756,-2.976001603 54.410195685,-2.97600267 54.410195062,-2.976079695 54.410239991,-2.976080762 54.410239368,-2.976157786 54.410284297,-2.976158854 54.410283674,-2.976235878 54.410373531,-2.976238013 54.410375399,-2.97600694 54.410465257,-2.976009075 54.410464634,-2.976086099 54.410509563,-2.976087167 54.410510185,-2.976010142 54.4106899,-2.976014412 54.410688655,-2.976168462 54.410823441,-2.976171664 54.41081908,-2.976710841 54.410774152,-2.976709773 54.410773529,-2.976786798 54.4107286,-2.976785729 54.410727354,-2.97693978 54.410682425,-2.976938711 54.410680555,-2.977169786 54.410635627,-2.977168717 54.410635003,-2.977245742 54.410590075,-2.977244673 54.410589451,-2.977321698 54.410499594,-2.977319561 54.410498971,-2.977396585 54.410454042,-2.977395516 54.410453418,-2.977472541 54.41040849,-2.977471472 54.410406619,-2.977702545 54.410316762,-2.977700406 54.410314267,-2.978008503 54.410359195,-2.978009573 54.410357947,-2.978163622 54.410313019,-2.978162552 54.410312395,-2.978239576 54.410267466,-2.978238506 54.410266842,-2.97831553 54.410176985,-2.978313391 54.410176361,-2.978390415 54.409906789,-2.978383995 54.409906165,-2.978461018 54.409861236,-2.978459948 54.409859364,-2.978691018 54.409904292,-2.978692089 54.409902419,-2.978923159 54.409947348,-2.978924229 54.409944849,-2.979232323 54.409899921,-2.979231253 54.409899296,-2.979308276 54.409944225,-2.979309347 54.409942975,-2.979463394 54.410482118,-2.979476248 54.410482743,-2.979399223 54.4105726,-2.979401365 54.410571351,-2.979555415 54.410526422,-2.979554344 54.410523922,-2.979862442 54.410478994,-2.97986137 54.410476493,-2.980169468 54.410431564,-2.980168396 54.410429063,-2.980476494 54.410384134,-2.980475422 54.410381632,-2.980783519 54.410336704,-2.980782446 54.410335452,-2.980936495 54.410245595,-2.980934349 54.410244969,-2.981011373 54.410200041,-2.9810103 54.410197537,-2.981318396 54.410377251,-2.981322689 54.410376625,-2.981399713 54.410331697,-2.98139864 54.410330445,-2.981552688 54.410375373,-2.981553762 54.410374747,-2.981630786 54.410419676,-2.981631859 54.410419049,-2.981708884 54.410463978,-2.981709957 54.410463352,-2.981786982 54.410553209,-2.981789129 54.410552583,-2.981866154 54.410597511,-2.981867227 54.410596885,-2.981944252 54.410686742,-2.9819464 54.410687995,-2.98179235 54.410732923,-2.981793424 54.410735428,-2.981485324 54.410780357,-2.981486398 54.410922876,-2.981643883 54.411030203,-2.98170808 54.411113579,-2.98140191 54.41114279,-2.981125261 54.410922026,-2.98064234 54.410966955,-2.980643412 54.410971958,-2.980027209 54.411016886,-2.980028281 54.411019387,-2.979720179 54.411064315,-2.97972125 54.411066815,-2.979413148 54.411111743,-2.979414219 54.411112368,-2.979337193 54.411471797,-2.979345762 54.411472422,-2.979268735 54.41151735,-2.979269806 54.411517975,-2.97919278 54.411652761,-2.979195992 54.411652136,-2.979273019 54.411697065,-2.97927409 54.41169644,-2.979351117 54.411831226,-2.97935433 54.41183185,-2.979277303 54.412460851,-2.979292298 54.4124621,-2.979138241 54.412507029,-2.979139312 54.412508278,-2.978985255 54.412463349,-2.978984185 54.412463974,-2.978907156 54.412688617,-2.97891251 54.41269049,-2.978681424 54.412645561,-2.978680353 54.412646185,-2.978603325 54.412736043,-2.978605465 54.412736667,-2.978528437 54.412781595,-2.978529507 54.41278222,-2.978452478 54.412827148,-2.978453548 54.412828397,-2.97829949 54.412873325,-2.97830056 54.412874573,-2.978146502 54.412919502,-2.978147572 54.41292075,-2.977993514 54.412965678,-2.977994583 54.412971915,-2.977224291 54.413016843,-2.97722536 54.41301996,-2.976840214 54.413064888,-2.976841282 54.413065511,-2.976764252 54.413020583,-2.976763184 54.413022452,-2.976532096 54.412977523,-2.976531028 54.412980015,-2.976222911 54.412935086,-2.976221843 54.412936332,-2.976067785 54.412891403,-2.976066718 54.412892026,-2.975989688 54.412847097,-2.975988621 54.412848342,-2.975834563 54.412803413,-2.975833496 54.412804036,-2.975756467 54.412714179,-2.975754332 54.412714801,-2.975677304 54.412624944,-2.975675169 54.412625566,-2.975598141 54.412580638,-2.975597074 54.41258126,-2.975520045 54.412536331,-2.975518978 54.412539442,-2.975133836 54.412494514,-2.975132769 54.412499489,-2.974516542 54.412544417,-2.974517608 54.41254566,-2.974363551 54.412590589,-2.974364617 54.412592454,-2.974133531 54.412682311,-2.974135662 54.412681689,-2.974212691 54.412726618,-2.974213756 54.412725997,-2.974290785 54.412815854,-2.974292916 54.412815232,-2.974369945 54.413039876,-2.974375274 54.413040497,-2.974298244 54.413130354,-2.974300375 54.413130976,-2.974223346 54.413220833,-2.974225477 54.413221455,-2.974148447 54.413266383,-2.974149513 54.413267005,-2.974072483 54.413311933,-2.974073548 54.413313176,-2.973919488 54.413358105,-2.973920554 54.413359347,-2.973766493 54.413404276,-2.973767559 54.413404897,-2.973690528 54.41362954,-2.973695853 54.413628919,-2.973772884 54.413853562,-2.973778209 54.413852941,-2.97385524 54.41389787,-2.973856305 54.413897249,-2.973933336 54.413987106,-2.973935467 54.413986484,-2.974012498 54.414076342,-2.974014629 54.414076963,-2.973937597 54.414256678,-2.973941858 54.414257299,-2.973864827 54.414302228,-2.973865892 54.414302849,-2.97378886 54.414437635,-2.973792055 54.414438256,-2.973715023 54.414617971,-2.973719283 54.414618592,-2.973642251 54.414708449,-2.973644381 54.41470907,-2.973567348 54.414753999,-2.973568413 54.414753378,-2.973645446 54.415022949,-2.973651836 54.415023571,-2.973574803 54.415158357,-2.973577997 54.415158978,-2.973500964 54.415248835,-2.973503094 54.415249456,-2.97342606 54.41542917,-2.973430319 54.415429791,-2.973353285 54.415564577,-2.973356479 54.415565198,-2.973279445 54.415655056,-2.973281574 54.415654435,-2.973358608 54.415789221,-2.973361802 54.415790462,-2.973207733 54.416104963,-2.973215185 54.416105584,-2.97313815 54.41624037,-2.973141343 54.41624099,-2.973064308 54.416375776,-2.973067501 54.416375156,-2.973144537 54.416420084,-2.973145601 54.416419463,-2.973222637 54.416464392,-2.973223702 54.416463771,-2.973300737 54.4165087,-2.973301802 54.416508079,-2.973378838 54.416553007,-2.973379903 54.416552386,-2.973456939 54.416597315,-2.973458004 54.416596694,-2.97353504 54.416641623,-2.973536105 54.41664038,-2.973690177 54.416685309,-2.973691242 54.416684688,-2.973768278 54.416729616,-2.973769344 54.416728995,-2.97384638 54.416773924,-2.973847445 54.416773302,-2.973924482 54.416818231,-2.973925547 54.41681761,-2.974002583 54.416862538,-2.974003649 54.416861917,-2.974080685 54.416906846,-2.974081751 54.416906224,-2.974158788 54.417085939,-2.97416305 54.41708656,-2.974086013 54.417176417,-2.974088144 54.417177039,-2.974011107 54.417626325,-2.974021762 54.417627568,-2.973867686 54.417672496,-2.973868751 54.417673739,-2.973714675 54.417853453,-2.973718935 54.417854074,-2.973641897 54.417943932,-2.973644027 54.417944553,-2.973566988 54.417989481,-2.973568053 54.417990102,-2.973491015 54.41807996,-2.973493145 54.418080581,-2.973416106 54.418125509,-2.973417171 54.41812613,-2.973340132 54.418215987,-2.973342261 54.418216608,-2.973265222 54.418261537,-2.973266287 54.418262158,-2.973189247 54.418352015,-2.973191377 54.418352636,-2.973114337 54.418397565,-2.973115402 54.418398185,-2.973038362 54.418443114,-2.973039427 54.418443735,-2.972962387 54.418488663,-2.972963452 54.418489284,-2.972886412 54.418534213,-2.972887476 54.418534833,-2.972810436 54.418579762,-2.972811501 54.418580383,-2.972734461 54.418625311,-2.972735525 54.418626552,-2.972581445 54.418671481,-2.972582509 54.418672722,-2.972428429 54.41871765,-2.972429493 54.418718271,-2.972352453 54.418763199,-2.972353516 54.41876444,-2.972199436 54.418854297,-2.972201563 54.418854918,-2.972124523 54.418989703,-2.972127713 54.418990324,-2.972050672 54.419170038,-2.972054926 54.419170658,-2.971977885 54.419215587,-2.971978948 54.419216207,-2.971901907 54.419261136,-2.971902971 54.419261756,-2.971825929 54.419342627,-2.971827843 54.419412901,-2.972029852 54.419483546,-2.972185636 54.419446611,-2.972308052 54.41944537,-2.972462135 54.419390711,-2.972553308 54.419353527,-2.972706539 54.419370258,-2.972861048 54.419458873,-2.973017259 54.419458377,-2.973078892 54.419384504,-2.973323721 54.41934558,-2.973692668 54.419480366,-2.973695863 54.419541775,-2.973882254 54.419575978,-2.974098823 54.419655855,-2.974224008 54.419905715,-2.974445695 54.419995323,-2.974478644 54.420084434,-2.974573228 54.420192014,-2.974606604 54.420236197,-2.974700121 54.420253173,-2.974823817 54.420439883,-2.975074834 54.420546965,-2.975169847 54.420600879,-2.975171127 54.420689989,-2.975265713 54.420743904,-2.975266994 54.420762373,-2.975205785 54.42076511,-2.974866792 54.420730908,-2.974650216 54.42073439,-2.974218771 54.420771576,-2.974065536 54.420863173,-2.973851943 54.420907356,-2.973945462 54.420943299,-2.973946314 54.421083465,-2.974396582 54.421019322,-2.974549178 54.421015964,-2.974965218 54.421033437,-2.97502728 54.421167601,-2.975107525 54.421481355,-2.975207447 54.421480235,-2.975346128 54.421525163,-2.975347196 54.421524541,-2.975424241 54.421569469,-2.975425308 54.421568225,-2.975579399 54.421478367,-2.975577264 54.421475877,-2.975885445 54.421430949,-2.975884377 54.421429704,-2.976038467 54.421474632,-2.976039535 54.421474009,-2.976116581 54.421384152,-2.976114445 54.421382907,-2.976268535 54.421248121,-2.97626533 54.421247498,-2.976342375 54.421112712,-2.976339171 54.42111209,-2.976416215 54.420977304,-2.97641301 54.420976058,-2.976567099 54.420931129,-2.976566031 54.420930506,-2.976643075 54.420885578,-2.976642006 54.420884955,-2.97671905 54.420840026,-2.976717982 54.42083878,-2.97687207 54.420883708,-2.976873138 54.420883085,-2.976950183 54.421017871,-2.976953389 54.421017247,-2.977030433 54.421124951,-2.977048408 54.421421978,-2.976993827 54.421466533,-2.977041123 54.421465909,-2.977118168 54.421510838,-2.977119237 54.42150772,-2.977504463 54.421462792,-2.977503393 54.421460921,-2.977734529 54.421415992,-2.977733459 54.421415368,-2.977810504 54.42137044,-2.977809434 54.421369192,-2.977963524 54.421279335,-2.977961384 54.421278711,-2.978038429 54.420874354,-2.978028799 54.42087373,-2.978105843 54.420828802,-2.978104773 54.420828178,-2.978181817 54.420648463,-2.978177536 54.420647839,-2.97825458 54.420692768,-2.97825565 54.420690271,-2.978563825 54.420735199,-2.978564895 54.420733951,-2.978718983 54.420778879,-2.978720054 54.42077763,-2.978874141 54.420822559,-2.978875212 54.420820685,-2.979106344 54.420865614,-2.979107415 54.420861865,-2.979569679 54.420906793,-2.979570751 54.420904918,-2.979801883 54.420949847,-2.979802955 54.420944219,-2.980496352 54.42089929,-2.980495279 54.420897413,-2.980726411 54.420852485,-2.980725338 54.420851233,-2.980879426 54.420806305,-2.980878353 54.420805053,-2.98103244 54.420760125,-2.981031367 54.420759499,-2.981108411 54.42071457,-2.981107338 54.420713944,-2.981184381 54.420669016,-2.981183308 54.420667764,-2.981337395 54.420622835,-2.981336321 54.420622209,-2.981413365 54.420577281,-2.981412291 54.420576655,-2.981489334 54.420486798,-2.981487187 54.420486171,-2.98156423 54.420441243,-2.981563156 54.420440617,-2.981640199 54.42035076,-2.981638052 54.420350133,-2.981715095 54.420305205,-2.981714021 54.420304579,-2.981791063 54.420169793,-2.981787841 54.420169167,-2.981864884 54.420124238,-2.98186381 54.420123612,-2.981940852 54.420033755,-2.981938704 54.420033128,-2.982015746 54.419943271,-2.982013597 54.419942645,-2.98209064 54.419897716,-2.982089565 54.41989709,-2.982166607 54.419807233,-2.982164458 54.419804726,-2.982472626 54.419759797,-2.982471551 54.419758544,-2.982625635 54.419713615,-2.98262456 54.419712988,-2.982701602 54.41966806,-2.982700526 54.419667433,-2.982777568 54.419577576,-2.982775418 54.419576949,-2.982852459 54.41953202,-2.982851384 54.419531393,-2.982928425 54.419441536,-2.982926275 54.419440909,-2.983003316 54.419306124,-2.98300009 54.419305497,-2.983077131 54.419125783,-2.983072829 54.419125156,-2.98314987 54.419080227,-2.983148794 54.4190796,-2.983225835 54.419034671,-2.983224759 54.419034044,-2.9833018 54.418944187,-2.983299648 54.41894356,-2.983376689 54.418898631,-2.983375613 54.418898004,-2.983452653 54.418808147,-2.983450501 54.418805009,-2.983835701 54.418760081,-2.983834625 54.418758825,-2.983988705 54.418803754,-2.983989781 54.418801242,-2.984297941 54.418846171,-2.984299018 54.41884303,-2.984684218 54.418887959,-2.984685296 54.418886702,-2.984839376 54.418841774,-2.984838299 54.418832344,-2.985993899 54.418787415,-2.98599282 54.418785528,-2.98622394 54.4187406,-2.986222861 54.418739341,-2.986376941 54.418694413,-2.986375862 54.418693783,-2.986452902 54.418738712,-2.986453981 54.418738082,-2.986531021 54.418783011,-2.9865321 54.418781123,-2.986763219 54.418736194,-2.98676214 54.418734305,-2.986993259 54.418689377,-2.98699218 54.418687488,-2.987223299 54.418642559,-2.987222219 54.41864004,-2.987530377 54.41890961,-2.987536859 54.41891024,-2.987459819 54.419134882,-2.98746522 54.419135512,-2.98738818 54.419270297,-2.98739142 54.419269667,-2.987468461 54.419404453,-2.987471702 54.419405083,-2.987394661 54.419450011,-2.987395741 54.419449381,-2.987472782 54.419539238,-2.987474943 54.419538608,-2.987551984 54.419628465,-2.987554145 54.419627835,-2.987631186 54.419717692,-2.987633347 54.419717062,-2.987710389 54.41976199,-2.987711469 54.41976136,-2.987788511 54.419851217,-2.987790672 54.419850586,-2.987867714 54.419895515,-2.987868795 54.419894885,-2.987945837 54.419984741,-2.987947998 54.419984111,-2.988025041 54.420029039,-2.988026121 54.420028409,-2.988103164 54.420118266,-2.988105326 54.420117635,-2.988182368 54.420162564,-2.988183449 54.420161933,-2.988260491 54.420386575,-2.988265897 54.420385945,-2.98834294 54.420430873,-2.988344021 54.420430243,-2.988421064 54.420654885,-2.988426471 54.420655515,-2.988349428 54.421104799,-2.988360241 54.421104169,-2.988437285 54.421194025,-2.988439448 54.421193395,-2.988516492 54.421238323,-2.988517574 54.421237062,-2.988671662 54.421326919,-2.988673826 54.421326288,-2.98875087 54.421461073,-2.988754116 54.421460442,-2.988831161 54.421550299,-2.988833324 54.421549668,-2.988910369 54.421639525,-2.988912533 54.421638894,-2.988989578 54.421683822,-2.98899066 54.421683191,-2.989067706 54.421773048,-2.98906987 54.421773679,-2.988992824 54.421818607,-2.988993906 54.421817976,-2.989070952 54.421862905,-2.989072034 54.421864167,-2.988917943 54.421954023,-2.988920107 54.421954654,-2.988843061 54.422044511,-2.988845225 54.422042618,-2.989076363 54.422087547,-2.989077445 54.422086916,-2.989154491 54.422131844,-2.989155573 54.422131213,-2.989232619 54.422176141,-2.989233701 54.422147669,-2.989417963 54.422173995,-2.989495658 54.422618711,-2.988966939 54.422891688,-2.988557374 54.422810817,-2.988555427 54.422811448,-2.98847838 54.422766519,-2.988477298 54.42276715,-2.988400251 54.422722221,-2.98839917 54.422722852,-2.988322122 54.422677924,-2.988321041 54.422679815,-2.9880899 54.422634887,-2.988088818 54.422635517,-2.988011771 54.422590589,-2.98801069 54.422591849,-2.987856596 54.422546921,-2.987855516 54.422548181,-2.987701422 54.422503253,-2.987700341 54.422504513,-2.987546247 54.422459585,-2.987545167 54.422463994,-2.987005839 54.422508922,-2.987006919 54.42251207,-2.986621685 54.422556999,-2.986622764 54.422558257,-2.98646867 54.422378544,-2.986464353 54.422377285,-2.986618446 54.422287428,-2.986616288 54.422286799,-2.986693334 54.42224187,-2.986692254 54.422241241,-2.986769301 54.422061527,-2.986764982 54.422062786,-2.98661089 54.422017858,-2.986609811 54.422018487,-2.986532765 54.421973559,-2.986531686 54.421978592,-2.985915318 54.421933664,-2.98591424 54.421935928,-2.985636875 54.421963765,-2.985529658 54.421990722,-2.985530304 54.422072474,-2.985424381 54.422091451,-2.985301538 54.422118408,-2.985302185 54.422119665,-2.985148093 54.422164594,-2.98514917 54.422165222,-2.985072124 54.422255079,-2.98507428 54.422263873,-2.983995629 54.422218945,-2.983994553 54.422192615,-2.98391686 54.422149695,-2.983669236 54.422043248,-2.983497151 54.422035517,-2.983342844 54.422048015,-2.982911601 54.421963549,-2.982246855 54.421967684,-2.981738352 54.421924286,-2.980442691 54.421935773,-2.980134722 54.421980702,-2.980135794 54.421983202,-2.97982761 54.422028131,-2.979828682 54.422028756,-2.979751636 54.422073685,-2.979752708 54.422076809,-2.979367477 54.422121738,-2.979368548 54.422122987,-2.979214456 54.422167916,-2.979215527 54.422168541,-2.979138481 54.422213469,-2.979139552 54.422214094,-2.979062505 54.422259022,-2.979063576 54.422259647,-2.97898653 54.422304576,-2.978987601 54.422307074,-2.978679414 54.422352002,-2.978680485 54.422353376,-2.978510982 54.422428632,-2.978096642 54.422567036,-2.977652978 54.422540453,-2.977606108 54.422675488,-2.977578498 54.422766093,-2.97748818 54.422820881,-2.977381597 54.423048641,-2.977001703 54.423129512,-2.977003627 54.423130135,-2.976926579 54.423175064,-2.976927648 54.423175687,-2.976850599 54.423220616,-2.976851668 54.423221239,-2.97677462 54.423266168,-2.976775688 54.423266791,-2.97669864 54.423311719,-2.976699709 54.423312342,-2.97662266 54.423402199,-2.976624797 54.423402822,-2.976547748 54.423537733,-2.976535544 54.423665028,-2.976353617 54.423692732,-2.976261799 54.423787571,-2.975647539 54.423903239,-2.974679274 54.424056829,-2.973465296 54.424078154,-2.973049651 54.424025978,-2.972832633 54.424010365,-2.972539417 54.423948085,-2.972460878 54.423831395,-2.972442701 - - - 54.40579318,-2.964651635 54.405792565,-2.964728652 54.405837493,-2.964729706 54.405838109,-2.96465269 54.40579318,-2.964651635 - - - TargetAreaCode - 011FWFNC34 - - - - -""" - -LONG_GSM7 = WITH_PLACEHOLDER_FOR_CONTENT.format('a' * 1396) -LONG_UCS2 = WITH_PLACEHOLDER_FOR_CONTENT.format('Ε΅yl' * 205 + 'a') -MISSING_AREA_NAMES = re.sub(".*", " ", WAINFLEET) diff --git a/tests/app/v2/broadcast/test_post_broadcast.py b/tests/app/v2/broadcast/test_post_broadcast.py deleted file mode 100644 index 6fc902a37..000000000 --- a/tests/app/v2/broadcast/test_post_broadcast.py +++ /dev/null @@ -1,487 +0,0 @@ -from unittest.mock import ANY - -import pytest -from flask import json - -from app.dao.broadcast_message_dao import ( - dao_get_broadcast_message_by_id_and_service_id, -) -from tests import create_service_authorization_header -from tests.app.db import create_api_key - -from . import sample_cap_xml_documents - - -def test_broadcast_for_service_without_permission_returns_400( - client, - sample_service, -): - auth_header = create_service_authorization_header(service_id=sample_service.id) - response = client.post( - path='/v2/broadcast', - data='', - headers=[('Content-Type', 'application/json'), auth_header], - ) - - assert response.status_code == 400 - assert response.get_json()['errors'][0]['message'] == ( - 'Service is not allowed to send broadcast messages' - ) - - -def test_post_broadcast_non_cap_xml_returns_415( - client, - sample_broadcast_service, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - response = client.post( - path='/v2/broadcast', - data=json.dumps({ - 'content': 'This is a test', - 'reference': 'abc123', - 'category': 'Other', - 'areas': [ - { - 'name': 'Hackney Marshes', - 'polygons': [[ - [-0.038280487060546875, 51.55738264619775], - [-0.03184318542480469, 51.553913882566754], - [-0.023174285888671875, 51.55812972989382], - [-0.023174285888671999, 51.55812972989999], - [-0.029869079589843747, 51.56165153059717], - [-0.038280487060546875, 51.55738264619775], - ]], - }, - ], - }), - headers=[('Content-Type', 'application/json'), auth_header], - ) - - assert response.status_code == 415 - assert json.loads(response.get_data(as_text=True)) == { - 'errors': [{ - 'error': 'BadRequestError', - 'message': 'Content type application/json not supported' - }], - 'status_code': 415, - } - - -def test_valid_post_cap_xml_broadcast_returns_201( - client, - sample_broadcast_service, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - response = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response.status_code == 201 - - response_json = json.loads(response.get_data(as_text=True)) - - assert response_json['approved_at'] is None - assert response_json['approved_by_id'] is None - assert response_json['areas']['names'] == [ - 'River Steeping in Wainfleet All Saints' - ] - assert response_json['cancelled_at'] is None - assert response_json['cancelled_by_id'] is None - assert response_json['content'].startswith( - 'A severe flood warning has been issued. Storm Dennis' - ) - assert response_json['content'].endswith( - 'closely monitoring the situation throughout the night. ' - ) - assert response_json['reference'] == '50385fcb0ab7aa447bbd46d848ce8466E' - assert response_json['cap_event'] == '053/055 Issue Severe Flood Warning EA' - assert response_json['created_at'] # datetime generated by the DB so can’t freeze it - assert response_json['created_by_id'] is None - assert response_json['finishes_at'] is None - assert response_json['id'] == ANY - assert response_json['personalisation'] is None - assert response_json['service_id'] == str(sample_broadcast_service.id) - - assert len(response_json['areas']['simple_polygons']) == 1 - assert len(response_json['areas']['simple_polygons'][0]) == 29 - assert response_json['areas']['simple_polygons'][0][0] == [53.10569, 0.24453] - assert response_json['areas']['simple_polygons'][0][-1] == [53.10569, 0.24453] - assert response_json['areas']['names'] == ['River Steeping in Wainfleet All Saints'] - assert 'ids' not in response_json['areas'] # only for broadcasts created in Admin - - assert response_json['starts_at'] is None - assert response_json['status'] == 'pending-approval' - assert response_json['template_id'] is None - assert response_json['template_name'] is None - assert response_json['template_version'] is None - assert response_json['updated_at'] is None - - -@pytest.mark.parametrize("is_approved,expected_status", [ - [True, "cancelled"], - [False, "rejected"] -]) -def test_valid_cancel_broadcast_request_calls_update_broadcast_message_status_and_returns_201( - client, - sample_broadcast_service, - mocker, - is_approved, - expected_status -): - api_key = create_api_key(service=sample_broadcast_service) - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - mock_redis_delete = mocker.patch('app.redis_store.delete') - - # create a broadcast - response_for_create = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_create.status_code == 201 - - response_json_for_create = json.loads(response_for_create.get_data(as_text=True)) - - broadcast_message = dao_get_broadcast_message_by_id_and_service_id( - response_json_for_create["id"], response_json_for_create["service_id"] - ) - # approve broadcast - if is_approved: - broadcast_message.status = 'broadcasting' - - mock_update = mocker.patch( - 'app.v2.broadcast.post_broadcast.broadcast_utils.update_broadcast_message_status' - ) - - # cancel broadcast - response_for_cancel = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_REFERENCES, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_cancel.status_code == 201 - mock_update.assert_called_once_with( - broadcast_message, - expected_status, - api_key_id=api_key.id - ) - mock_redis_delete.assert_called_once_with( - f'service-{sample_broadcast_service.id}-broadcast-message-{broadcast_message.id}' - ) - - -@pytest.mark.parametrize('cap_xml_document, expected_status, expected_error', ( - ( - sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_REFERENCES, - 404, - [{'error': 'NoResultFound', 'message': 'No result found'}], - ), - ( - sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_EMPTY_REFERENCES, - 404, - [{'error': 'NoResultFound', 'message': 'No result found'}], - ), - ( - sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_MISSING_REFERENCES, - 400, - [{'error': 'BadRequestError', 'message': 'Missing '}], - ), -)) -def test_cancel_request_does_not_cancel_broadcast_if_reference_does_not_match( - client, - sample_broadcast_service, - cap_xml_document, - expected_status, - expected_error, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - # create a broadcast - response_for_create = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WINDEMERE, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_create.status_code == 201 - - response_json_for_create = json.loads(response_for_create.get_data(as_text=True)) - - assert response_json_for_create['cancelled_at'] is None - assert response_json_for_create['cancelled_by_id'] is None - assert response_json_for_create['reference'] == '4f6d28b10ab7aa447bbd46d85f1e9effE' - assert response_json_for_create['status'] == 'pending-approval' - - # try to cancel broadcast, but reference doesn't match - response_for_cancel = client.post( - path='/v2/broadcast', - data=cap_xml_document, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - response_for_cancel_json = json.loads(response_for_cancel.get_data(as_text=True)) - - assert response_for_cancel.status_code == expected_status - assert response_for_cancel_json["errors"] == expected_error - - -def test_cancel_raises_error_if_multiple_broadcasts_referenced( - client, - sample_broadcast_service, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - for cap_document in ( - sample_cap_xml_documents.WAINFLEET, - sample_cap_xml_documents.WINDEMERE, - ): - response_for_create = client.post( - path='/v2/broadcast', - data=cap_document, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_create.status_code == 201 - - # try to cancel two broadcasts with one request - response_for_cancel = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_WINDMERE_REFERENCES, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - response_for_cancel_json = json.loads(response_for_cancel.get_data(as_text=True)) - - assert response_for_cancel.status_code == 400 - assert response_for_cancel_json["errors"] == [{ - 'error': 'BadRequestError', - 'message': 'Multiple alerts found - unclear which one to cancel', - }] - - -def test_cancel_request_does_not_cancel_broadcast_if_service_id_does_not_match( - client, - sample_broadcast_service, - sample_broadcast_service_2 -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - # create a broadcast - response_for_create = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_create.status_code == 201 - - response_json_for_create = json.loads(response_for_create.get_data(as_text=True)) - - assert response_json_for_create['cancelled_at'] is None - assert response_json_for_create['cancelled_by_id'] is None - assert response_json_for_create['reference'] == '50385fcb0ab7aa447bbd46d848ce8466E' - assert response_json_for_create['status'] == 'pending-approval' - - # try to cancel broadcast, but service id doesn't match - auth_header_2 = create_service_authorization_header(service_id=sample_broadcast_service_2.id) - response_for_cancel = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_REFERENCES, - headers=[('Content-Type', 'application/cap+xml'), auth_header_2], - ) - - assert response_for_cancel.status_code == 404 - - -@pytest.mark.parametrize("is_approved, expected_cancel_tasks", ( - (True, 1), - (False, 0), -)) -def test_same_broadcast_cant_be_cancelled_twice( - mocker, - client, - sample_broadcast_service, - is_approved, - expected_cancel_tasks, -): - mock_send_broadcast_event_task = mocker.patch( - 'app.celery.broadcast_message_tasks.send_broadcast_event.apply_async' - ) - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - # create a broadcast - response_for_create = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response_for_create.status_code == 201 - - response_json_for_create = json.loads(response_for_create.get_data(as_text=True)) - - broadcast_message = dao_get_broadcast_message_by_id_and_service_id( - response_json_for_create["id"], response_json_for_create["service_id"] - ) - # approve broadcast - if is_approved: - broadcast_message.status = 'broadcasting' - - first_response_for_cancel = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_REFERENCES, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert first_response_for_cancel.status_code == 201 - - second_response_for_cancel = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET_CANCEL_WITH_REFERENCES, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert second_response_for_cancel.status_code == 404 - - assert len(mock_send_broadcast_event_task.call_args_list) == expected_cancel_tasks - - -def test_large_polygon_is_simplified( - client, - sample_broadcast_service, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - response = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WINDEMERE, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - assert response.status_code == 201 - - response_json = json.loads(response.get_data(as_text=True)) - - assert len(response_json['areas']['simple_polygons']) == 1 - assert len(response_json['areas']['simple_polygons'][0]) == 110 - - assert response_json['areas']['simple_polygons'][0][0] == [54.419546, -2.988521] - assert response_json['areas']['simple_polygons'][0][-1] == [54.419546, -2.988521] - - -@pytest.mark.parametrize("training_mode_service", [True, False]) -def test_valid_post_cap_xml_broadcast_sets_stubbed_to_true_for_training_mode_services( - client, - sample_broadcast_service, - training_mode_service -): - sample_broadcast_service.restricted = training_mode_service - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - response = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.WAINFLEET, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - - assert response.status_code == 201 - response_json = json.loads(response.get_data(as_text=True)) - - broadcast_message = dao_get_broadcast_message_by_id_and_service_id( - response_json['id'], sample_broadcast_service.id - ) - assert broadcast_message.stubbed == training_mode_service - - -@pytest.mark.parametrize('xml_document', ( - 'Oh no', - '', -)) -def test_invalid_post_cap_xml_broadcast_returns_400( - client, - sample_broadcast_service, - xml_document, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - response = client.post( - path='/v2/broadcast', - data=xml_document, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - - assert response.status_code == 400 - assert json.loads(response.get_data(as_text=True)) == { - 'errors': [{ - 'error': 'BadRequestError', - 'message': 'Request data is not valid CAP XML' - }], - 'status_code': 400, - } - - -def test_unsupported_message_types_400( - client, - sample_broadcast_service, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - - response = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.UPDATE, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - - assert response.status_code == 400 - assert { - 'error': 'ValidationError', - 'message': 'msgType Update is not one of [Alert, Cancel]', - } in ( - json.loads(response.get_data(as_text=True))['errors'] - ) - - -@pytest.mark.parametrize('xml_document, expected_error', ( - (sample_cap_xml_documents.LONG_UCS2, ( - 'description must be 615 characters or fewer (because it ' - 'could not be GSM7 encoded)' - )), - (sample_cap_xml_documents.LONG_GSM7, ( - 'description must be 1,395 characters or fewer' - )), -)) -def test_content_too_long_returns_400( - client, - sample_broadcast_service, - xml_document, - expected_error, -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - response = client.post( - path='/v2/broadcast', - data=xml_document, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - - assert json.loads(response.get_data(as_text=True)) == { - 'errors': [{ - 'error': 'ValidationError', - 'message': expected_error, - }], - 'status_code': 400, - } - - -def test_invalid_areas_returns_400( - client, - sample_broadcast_service -): - auth_header = create_service_authorization_header(service_id=sample_broadcast_service.id) - response = client.post( - path='/v2/broadcast', - data=sample_cap_xml_documents.MISSING_AREA_NAMES, - headers=[('Content-Type', 'application/cap+xml'), auth_header], - ) - - assert json.loads(response.get_data(as_text=True)) == { - 'errors': [{ - 'error': 'ValidationError', - # the blank spaces represent the blank areaDesc in the XML - 'message': 'areas does not match ([a-zA-Z1-9]+ )*[a-zA-Z1-9]+', - }], - 'status_code': 400, - } diff --git a/tests/app/v2/templates/test_get_templates.py b/tests/app/v2/templates/test_get_templates.py index 77687f084..29eacc6dc 100644 --- a/tests/app/v2/templates/test_get_templates.py +++ b/tests/app/v2/templates/test_get_templates.py @@ -112,7 +112,7 @@ def test_get_all_templates_for_invalid_type_returns_400(client, sample_service): 'status_code': 400, 'errors': [ { - 'message': 'type coconut is not one of [sms, email, letter, broadcast]', + 'message': 'type coconut is not one of [sms, email, letter]', 'error': 'ValidationError' } ] diff --git a/tests/app/v2/templates/test_templates_schemas.py b/tests/app/v2/templates/test_templates_schemas.py index ae1376695..edcf9d629 100644 --- a/tests/app/v2/templates/test_templates_schemas.py +++ b/tests/app/v2/templates/test_templates_schemas.py @@ -241,7 +241,7 @@ def test_get_all_template_request_schema_against_invalid_args_is_invalid(templat assert errors['status_code'] == 400 assert len(errors['errors']) == 1 - assert errors['errors'][0]['message'] == 'type unknown is not one of [sms, email, letter, broadcast]' + assert errors['errors'][0]['message'] == 'type unknown is not one of [sms, email, letter]' @pytest.mark.parametrize("response", valid_json_get_all_response) diff --git a/tests/conftest.py b/tests/conftest.py index 4eef62245..dabd3eb33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -128,11 +128,8 @@ def notify_db_session(_notify_db, sms_providers): "organisation_types", "service_permission_types", "auth_type", - "broadcast_status_type", "invite_status_type", - "service_callback_type", - "broadcast_channel_types", - "broadcast_provider_types"]: + "service_callback_type"]: _notify_db.engine.execute(tbl.delete()) _notify_db.session.commit() From 9d5bcdf9100dbc7f2ad17cbbf06993c4175930a0 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 4 Oct 2022 16:13:44 +0000 Subject: [PATCH 16/65] remove broadcasts from migrations --- .../versions/0322_broadcast_service_perm.py | 5 +- migrations/versions/0323_broadcast_message.py | 74 +------------------ migrations/versions/0326_broadcast_event.py | 26 +------ .../versions/0329_purge_broadcast_data.py | 2 +- .../versions/0330_broadcast_invite_email.py | 52 +------------ migrations/versions/0331_add_broadcast_org.py | 61 +-------------- .../versions/0332_broadcast_provider_msg.py | 24 +----- .../0333_service_broadcast_provider.py | 11 +-- .../versions/0334_broadcast_message_number.py | 17 +---- .../versions/0335_broadcast_msg_content.py | 10 +-- .../versions/0336_broadcast_msg_content_2.py | 20 +---- migrations/versions/0337_broadcast_msg_api.py | 9 +-- .../versions/0340_stub_training_broadcasts.py | 4 +- .../0342_service_broadcast_settings.py | 29 ++------ .../versions/0344_stubbed_not_nullable.py | 15 +--- .../versions/0345_move_broadcast_provider.py | 22 +----- ...ast_settings_migrate_broadcast_settings.py | 30 +------- .../versions/0352_broadcast_provider_types.py | 16 +--- .../0353_broadcast_provider_not_null.py | 6 +- .../versions/0354_government_channel.py | 7 +- migrations/versions/0358_operator_channel.py | 7 +- migrations/versions/0359_more_permissions.py | 34 +-------- .../versions/0362_broadcast_msg_event.py | 4 +- .../versions/0363_cancelled_by_api_key.py | 26 +------ migrations/versions/0364_drop_old_column.py | 16 +--- 25 files changed, 52 insertions(+), 475 deletions(-) diff --git a/migrations/versions/0322_broadcast_service_perm.py b/migrations/versions/0322_broadcast_service_perm.py index 2819dd8bb..4a0385bad 100644 --- a/migrations/versions/0322_broadcast_service_perm.py +++ b/migrations/versions/0322_broadcast_service_perm.py @@ -13,9 +13,8 @@ down_revision = '0321_drop_postage_constraints' def upgrade(): - op.execute("INSERT INTO service_permission_types VALUES ('broadcast')") + pass def downgrade(): - op.execute("DELETE FROM service_permissions WHERE permission = 'broadcast'") - op.execute("DELETE FROM service_permission_types WHERE name = 'broadcast'") + pass \ No newline at end of file diff --git a/migrations/versions/0323_broadcast_message.py b/migrations/versions/0323_broadcast_message.py index 39f93948d..fdfc50750 100644 --- a/migrations/versions/0323_broadcast_message.py +++ b/migrations/versions/0323_broadcast_message.py @@ -10,22 +10,10 @@ import sqlalchemy as sa from sqlalchemy.sql import column, func from sqlalchemy.dialects import postgresql -from app.models import BroadcastMessage - revision = '0323_broadcast_message' down_revision = '0322_broadcast_service_perm' -name = 'template_type' -tmp_name = 'tmp_' + name - -old_options = ('sms', 'email', 'letter') -new_options = old_options + ('broadcast',) - -new_type = sa.Enum(*new_options, name=name) -old_type = sa.Enum(*old_options, name=name) - - STATUSES = [ 'draft', 'pending-approval', @@ -38,66 +26,8 @@ STATUSES = [ def upgrade(): - op.execute(f'ALTER TYPE {name} RENAME TO {tmp_name}') - new_type.create(op.get_bind()) - - for table in ['templates', 'templates_history', 'service_contact_list']: - op.execute(f'ALTER TABLE {table} ALTER COLUMN template_type TYPE {name} USING template_type::text::{name}') - - op.execute(f'DROP TYPE {tmp_name}') - - broadcast_status_type = op.create_table( - 'broadcast_status_type', - sa.Column('name', sa.String(), nullable=False), - sa.PrimaryKeyConstraint('name') - ) - op.bulk_insert(broadcast_status_type, [{'name': state} for state in STATUSES]) - - op.create_table( - 'broadcast_message', - sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('service_id', postgresql.UUID(as_uuid=True)), - sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('template_version', sa.Integer(), nullable=False), - sa.Column('_personalisation', sa.String()), - sa.Column('areas', postgresql.JSONB(none_as_null=True, astext_type=sa.Text())), - sa.Column('status', sa.String()), - sa.Column('starts_at', sa.DateTime()), - sa.Column('finishes_at', sa.DateTime()), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.Column('approved_at', sa.DateTime()), - sa.Column('cancelled_at', sa.DateTime()), - sa.Column('updated_at', sa.DateTime()), - sa.Column('created_by_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('approved_by_id', postgresql.UUID(as_uuid=True)), - sa.Column('cancelled_by_id', postgresql.UUID(as_uuid=True)), - - sa.ForeignKeyConstraint(['approved_by_id'], ['users.id'], ), - sa.ForeignKeyConstraint(['cancelled_by_id'], ['users.id'], ), - sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), - sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), - sa.ForeignKeyConstraint(['template_id', 'template_version'], ['templates_history.id', 'templates_history.version'], ), - sa.PrimaryKeyConstraint('id') - ) - - op.add_column('templates', sa.Column('broadcast_data', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()))) - op.add_column('templates_history', sa.Column('broadcast_data', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()))) + pass def downgrade(): - op.execute("DELETE FROM template_folder_map WHERE template_id IN (SELECT id FROM templates WHERE template_type = 'broadcast')") - op.execute("DELETE FROM template_redacted WHERE template_id IN (SELECT id FROM templates WHERE template_type = 'broadcast')") - op.execute("DELETE FROM templates WHERE template_type = 'broadcast'") - op.execute("DELETE FROM templates_history WHERE template_type = 'broadcast'") - - op.execute(f'ALTER TYPE {name} RENAME TO {tmp_name}') - old_type.create(op.get_bind()) - - for table in ['templates', 'templates_history', 'service_contact_list']: - op.execute(f'ALTER TABLE {table} ALTER COLUMN template_type TYPE {name} USING template_type::text::{name}') - op.execute(f'DROP TYPE {tmp_name}') - - op.drop_column('templates_history', 'broadcast_data') - op.drop_column('templates', 'broadcast_data') - op.drop_table('broadcast_message') - op.drop_table('broadcast_status_type') + pass diff --git a/migrations/versions/0326_broadcast_event.py b/migrations/versions/0326_broadcast_event.py index 46cdb258f..5ebb1a008 100644 --- a/migrations/versions/0326_broadcast_event.py +++ b/migrations/versions/0326_broadcast_event.py @@ -14,30 +14,8 @@ down_revision = '0325_int_letter_rates_fix' def upgrade(): - op.create_table('broadcast_event', - sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('broadcast_message_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('sent_at', sa.DateTime(), nullable=False), - sa.Column('message_type', sa.String(), nullable=False), - sa.Column('transmitted_content', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), nullable=True), - sa.Column('transmitted_areas', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), nullable=False), - sa.Column('transmitted_sender', sa.String(), nullable=False), - sa.Column('transmitted_starts_at', sa.DateTime(), nullable=True), - sa.Column('transmitted_finishes_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['broadcast_message_id'], ['broadcast_message.id'], ), - sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), - sa.PrimaryKeyConstraint('id') - ) - # this shouldn't be nullable. it defaults to `[]` in python. - op.alter_column('broadcast_message', 'areas', existing_type=postgresql.JSONB(astext_type=sa.Text()), nullable=False) - # this can't be nullable. it defaults to 'draft' in python. - op.alter_column('broadcast_message', 'status', existing_type=sa.VARCHAR(), nullable=False) - op.create_foreign_key(None, 'broadcast_message', 'broadcast_status_type', ['status'], ['name']) + pass def downgrade(): - op.drop_constraint('broadcast_message_status_fkey', 'broadcast_message', type_='foreignkey') - op.alter_column('broadcast_message', 'status', existing_type=sa.VARCHAR(), nullable=True) - op.alter_column('broadcast_message', 'areas', existing_type=postgresql.JSONB(astext_type=sa.Text()), nullable=True) - op.drop_table('broadcast_event') + pass diff --git a/migrations/versions/0329_purge_broadcast_data.py b/migrations/versions/0329_purge_broadcast_data.py index b8c698c53..ed3507e33 100644 --- a/migrations/versions/0329_purge_broadcast_data.py +++ b/migrations/versions/0329_purge_broadcast_data.py @@ -14,7 +14,7 @@ down_revision = '0328_international_letters_perm' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute("TRUNCATE broadcast_event, broadcast_message;") + pass # ### end Alembic commands ### diff --git a/migrations/versions/0330_broadcast_invite_email.py b/migrations/versions/0330_broadcast_invite_email.py index 2127f8832..eb50bbdd4 100644 --- a/migrations/versions/0330_broadcast_invite_email.py +++ b/migrations/versions/0330_broadcast_invite_email.py @@ -18,57 +18,9 @@ user_id = '6af522d0-2915-4e52-83a3-3690455a5fe6' service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' template_id = '46152f7c-6901-41d5-8590-a5624d0d4359' -broadcast_invitation_template_name = 'Notify broadcast invitation email' -broadcast_invitation_subject = "((user_name)) has invited you to join ((service_name)) on GOV.UK Notify" -broadcast_invitation_content = """((user_name)) has invited you to join ((service_name)) on GOV.UK Notify. - -In an emergency, use Notify to broadcast an alert, warning the public about an imminent risk to life. - -Use this link to join the team: -((url)) - -This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure. - -Thanks - -GOV.​UK Notify team -https://www.gov.uk/notify -""" - - def upgrade(): - insert_query = """ - INSERT INTO {} - (id, name, template_type, created_at, content, archived, service_id, - subject, created_by_id, version, process_type, hidden) - VALUES - ('{}', '{}', 'email', '{}', '{}', False, '{}', '{}', '{}', 1, 'normal', False) - """ - - op.execute(insert_query.format( - 'templates_history', - template_id, - broadcast_invitation_template_name, - datetime.utcnow(), - broadcast_invitation_content, - service_id, - broadcast_invitation_subject, - user_id - )) - - op.execute(insert_query.format( - 'templates', - template_id, - broadcast_invitation_template_name, - datetime.utcnow(), - broadcast_invitation_content, - service_id, - broadcast_invitation_subject, - user_id - )) + pass def downgrade(): - op.get_bind() - op.execute("delete from templates where id = '{}'".format(template_id)) - op.execute("delete from templates_history where id = '{}'".format(template_id)) + pass diff --git a/migrations/versions/0331_add_broadcast_org.py b/migrations/versions/0331_add_broadcast_org.py index 2952c10ad..9ffc3e2f2 100644 --- a/migrations/versions/0331_add_broadcast_org.py +++ b/migrations/versions/0331_add_broadcast_org.py @@ -18,64 +18,7 @@ organisation_id = '38e4bf69-93b0-445d-acee-53ea53fe02df' def upgrade(): - # we've already done this manually on production - if environment != "production": - insert_sql = """ - INSERT INTO organisation - ( - id, - name, - active, - created_at, - agreement_signed, - crown, - organisation_type - ) - VALUES ( - :id, - :name, - :active, - current_timestamp, - :agreement_signed, - :crown, - :organisation_type - ) - """ - update_service_set_broadcast_org_sql = """ - UPDATE services - SET organisation_id = :organisation_id - WHERE id in ( - SELECT service_id - FROM service_permissions - WHERE permission = 'broadcast' - ) - """ - conn = op.get_bind() - conn.execute( - sa.text(insert_sql), - id=organisation_id, - name=f'Broadcast Services ({environment})', - active=True, - agreement_signed=None, - crown=None, - organisation_type='central', - ) - conn.execute( - sa.text(update_service_set_broadcast_org_sql), - organisation_id=organisation_id - ) - + pass def downgrade(): - update_service_remove_org_sql = """ - UPDATE services - SET organisation_id = NULL, updated_at = current_timestamp - WHERE organisation_id = :organisation_id - """ - delete_sql = """ - DELETE FROM organisation - WHERE id = :organisation_id - """ - conn = op.get_bind() - conn.execute(sa.text(update_service_remove_org_sql), organisation_id=organisation_id) - conn.execute(sa.text(delete_sql), organisation_id=organisation_id) + pass diff --git a/migrations/versions/0332_broadcast_provider_msg.py b/migrations/versions/0332_broadcast_provider_msg.py index 088f1c9df..0e3539e5d 100644 --- a/migrations/versions/0332_broadcast_provider_msg.py +++ b/migrations/versions/0332_broadcast_provider_msg.py @@ -22,28 +22,8 @@ STATUSES = [ def upgrade(): - broadcast_provider_message_status_type = op.create_table( - 'broadcast_provider_message_status_type', - sa.Column('name', sa.String(), nullable=False), - sa.PrimaryKeyConstraint('name') - ) - op.bulk_insert(broadcast_provider_message_status_type, [{'name': status} for status in STATUSES]) - - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - 'broadcast_provider_message', - sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('broadcast_event_id', postgresql.UUID(as_uuid=True), nullable=True), - sa.Column('provider', sa.String(), nullable=True), - sa.Column('status', sa.String(), nullable=True), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.Column('updated_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['broadcast_event_id'], ['broadcast_event.id'], ), - sa.PrimaryKeyConstraint('id'), - sa.UniqueConstraint('broadcast_event_id', 'provider') - ) + pass def downgrade(): - op.drop_table('broadcast_provider_message') - op.drop_table('broadcast_provider_message_status_type') + pass \ No newline at end of file diff --git a/migrations/versions/0333_service_broadcast_provider.py b/migrations/versions/0333_service_broadcast_provider.py index 3c8d3fa94..2de345328 100644 --- a/migrations/versions/0333_service_broadcast_provider.py +++ b/migrations/versions/0333_service_broadcast_provider.py @@ -14,15 +14,8 @@ down_revision = '0332_broadcast_provider_msg' def upgrade(): - op.create_table( - 'service_broadcast_provider_restriction', - sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('provider', sa.String(), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), - sa.PrimaryKeyConstraint('service_id') - ) + pass def downgrade(): - op.drop_table('service_broadcast_provider_restriction') + pass diff --git a/migrations/versions/0334_broadcast_message_number.py b/migrations/versions/0334_broadcast_message_number.py index db8360f98..0440640de 100644 --- a/migrations/versions/0334_broadcast_message_number.py +++ b/migrations/versions/0334_broadcast_message_number.py @@ -15,24 +15,11 @@ down_revision = '0333_service_broadcast_provider' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute("create sequence broadcast_provider_message_number_seq") - op.create_table( - 'broadcast_provider_message_number', - sa.Column( - 'broadcast_provider_message_number', - sa.Integer(), - server_default=sa.text("nextval('broadcast_provider_message_number_seq')"), - nullable=False - ), - sa.Column('broadcast_provider_message_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.ForeignKeyConstraint(['broadcast_provider_message_id'], ['broadcast_provider_message.id'], ), - sa.PrimaryKeyConstraint('broadcast_provider_message_number') - ) + pass # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('broadcast_provider_message_number') - op.execute("drop sequence broadcast_provider_message_number_seq") + pass # ### end Alembic commands ### diff --git a/migrations/versions/0335_broadcast_msg_content.py b/migrations/versions/0335_broadcast_msg_content.py index a96155702..8b021fc35 100644 --- a/migrations/versions/0335_broadcast_msg_content.py +++ b/migrations/versions/0335_broadcast_msg_content.py @@ -14,14 +14,8 @@ down_revision = '0334_broadcast_message_number' def upgrade(): - op.add_column('broadcast_message', sa.Column('content', sa.Text(), nullable=True)) - op.alter_column('broadcast_message', 'template_id', nullable=True) - op.alter_column('broadcast_message', 'template_version', nullable=True) + pass def downgrade(): - # downgrade fails if there are broadcasts without a template. This is deliberate cos I don't feel comfortable - # deleting broadcasts. - op.alter_column('broadcast_message', 'template_id', nullable=False) - op.alter_column('broadcast_message', 'template_version', nullable=False) - op.drop_column('broadcast_message', 'content') + pass diff --git a/migrations/versions/0336_broadcast_msg_content_2.py b/migrations/versions/0336_broadcast_msg_content_2.py index 51f05b351..1652afdec 100644 --- a/migrations/versions/0336_broadcast_msg_content_2.py +++ b/migrations/versions/0336_broadcast_msg_content_2.py @@ -11,30 +11,14 @@ from notifications_utils.template import BroadcastMessageTemplate from sqlalchemy.dialects import postgresql from sqlalchemy.orm.session import Session -from app.models import BroadcastMessage revision = '0336_broadcast_msg_content_2' down_revision = '0335_broadcast_msg_content' def upgrade(): - - conn = op.get_bind() - - results = conn.execute(sa.text(""" - UPDATE - broadcast_message - SET - content = templates_history.content - FROM - templates_history - WHERE - broadcast_message.content is NULL and - broadcast_message.template_id = templates_history.id and - broadcast_message.template_version = templates_history.version - ; - """)) + pass def downgrade(): - op.alter_column('broadcast_message', 'content', nullable=True) + pass diff --git a/migrations/versions/0337_broadcast_msg_api.py b/migrations/versions/0337_broadcast_msg_api.py index 96287e2bc..249e0e983 100644 --- a/migrations/versions/0337_broadcast_msg_api.py +++ b/migrations/versions/0337_broadcast_msg_api.py @@ -14,13 +14,8 @@ down_revision = '0336_broadcast_msg_content_2' def upgrade(): - op.alter_column('broadcast_message', 'created_by_id', nullable=True) - op.add_column('broadcast_message', sa.Column('api_key_id', postgresql.UUID(), nullable=True)) - op.create_foreign_key(None, 'broadcast_message', 'api_keys', ['api_key_id'], ['id']) - op.add_column('broadcast_message', sa.Column('reference', sa.String(length=255), nullable=True)) + pass def downgrade(): - op.alter_column('broadcast_message', 'created_by_id', nullable=False) - op.drop_column('broadcast_message', 'api_key_id') - op.add_column('broadcast_message', 'reference') + pass diff --git a/migrations/versions/0340_stub_training_broadcasts.py b/migrations/versions/0340_stub_training_broadcasts.py index beb2ac8cc..1d2091159 100644 --- a/migrations/versions/0340_stub_training_broadcasts.py +++ b/migrations/versions/0340_stub_training_broadcasts.py @@ -15,11 +15,11 @@ down_revision = '0339_service_billing_details' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('broadcast_message', sa.Column('stubbed', sa.Boolean(), nullable=True)) + pass # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_column('broadcast_message', 'stubbed') + pass # ### end Alembic commands ### diff --git a/migrations/versions/0342_service_broadcast_settings.py b/migrations/versions/0342_service_broadcast_settings.py index ba706f562..557ba55b2 100644 --- a/migrations/versions/0342_service_broadcast_settings.py +++ b/migrations/versions/0342_service_broadcast_settings.py @@ -16,28 +16,11 @@ CHANNEL_TYPES = ["test", "severe"] def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('broadcast_channel_types', - sa.Column('name', sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint('name') - ) - op.create_table('service_broadcast_settings', - sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), - sa.Column('channel', sa.String(length=255), nullable=False), - sa.Column('created_at', sa.DateTime(), nullable=False), - sa.Column('updated_at', sa.DateTime(), nullable=True), - sa.ForeignKeyConstraint(['channel'], ['broadcast_channel_types.name'], ), - sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), - sa.PrimaryKeyConstraint('service_id') - ) - # ### end Alembic commands ### - - for channel in CHANNEL_TYPES: - op.execute(f"INSERT INTO broadcast_channel_types VALUES ('{channel}')") - + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('service_broadcast_settings') - op.drop_table('broadcast_channel_types') - # ### end Alembic commands ### + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/migrations/versions/0344_stubbed_not_nullable.py b/migrations/versions/0344_stubbed_not_nullable.py index eb5e87028..408eb4b5a 100644 --- a/migrations/versions/0344_stubbed_not_nullable.py +++ b/migrations/versions/0344_stubbed_not_nullable.py @@ -15,22 +15,11 @@ down_revision = '0343_org_billing_details' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.execute("UPDATE broadcast_message SET stubbed = False WHERE stubbed is null") - op.alter_column( - 'broadcast_message', - 'stubbed', - existing_type=sa.BOOLEAN(), - nullable=False - ) + pass # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - 'broadcast_message', - 'stubbed', - existing_type=sa.BOOLEAN(), - nullable=True - ) + pass # ### end Alembic commands ### diff --git a/migrations/versions/0345_move_broadcast_provider.py b/migrations/versions/0345_move_broadcast_provider.py index fbcea3078..4d20419fb 100644 --- a/migrations/versions/0345_move_broadcast_provider.py +++ b/migrations/versions/0345_move_broadcast_provider.py @@ -14,26 +14,8 @@ down_revision = '0344_stubbed_not_nullable' def upgrade(): - op.add_column('service_broadcast_settings', sa.Column('provider', sa.String(), nullable=True)) - - sql = """ - select service_id, provider - from service_broadcast_provider_restriction - where service_id NOT IN (select service_id from service_broadcast_settings) - """ - insert_sql = """ - insert into service_broadcast_settings(service_id, channel, provider, created_at, updated_at) - values('{}', 'test', '{}', now(), null) - """ - conn = op.get_bind() - results = conn.execute(sql) - restrictions = results.fetchall() - for x in restrictions: - f = insert_sql.format(x.service_id, x.provider) - conn.execute(f) + pass def downgrade(): - # Downgrade does not try and fully undo the upgrade, in particular it does not - # delete the rows added to the service_broadcast_settings table - op.drop_column('service_broadcast_settings', 'provider') + pass diff --git a/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py b/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py index 90577f58b..4b2aaabc4 100644 --- a/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py +++ b/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py @@ -14,35 +14,7 @@ down_revision = '0347_add_dvla_volumes_template' def upgrade(): - # For every service that has the broadcast permission we want it to have - # a row in the broadcast_service_settings table - # - # If it doesnt have a row already, then: - # - if the service is in trial mode, add a row and set the channel as 'severe' - # - if the service is in live mode, add a row and set the channel as 'test' - # - # If it does have a row already no action needed - conn = op.get_bind() - - find_services_sql = """ - SELECT services.id, services.restricted - FROM services - LEFT JOIN service_permissions - ON services.id = service_permissions.service_id - WHERE service_permissions.permission = 'broadcast' - """ - - services = conn.execute(find_services_sql) - for service in services: - setting = conn.execute(f"SELECT service_id, channel, provider FROM service_broadcast_settings WHERE service_id = '{service.id}';").first() - if setting: - print(f"Service {service.id} already has service_broadcast_settings. No action required") - else: - channel = "severe" if service.restricted else "test" - print(f"Service {service.id} does not have service_broadcast_settings. Will insert one with channel {channel}") - conn.execute(f"INSERT INTO service_broadcast_settings (service_id, channel, created_at) VALUES ('{service.id}', '{channel}', now());") - + pass def downgrade(): - # No downgrade as we do not know what the state of the table was before that it should return to pass diff --git a/migrations/versions/0352_broadcast_provider_types.py b/migrations/versions/0352_broadcast_provider_types.py index 6d0d1fad2..48cc52814 100644 --- a/migrations/versions/0352_broadcast_provider_types.py +++ b/migrations/versions/0352_broadcast_provider_types.py @@ -11,22 +11,10 @@ import sqlalchemy as sa revision = '0352_broadcast_provider_types' down_revision = '0351_unique_key_annual_billing' -PROVIDER_TYPES = ('ee', 'three', 'vodafone', 'o2', 'all') - def upgrade(): - op.create_table('broadcast_provider_types', - sa.Column('name', sa.String(length=255), nullable=False), - sa.PrimaryKeyConstraint('name')) - for provider in PROVIDER_TYPES: - op.execute(f"INSERT INTO broadcast_provider_types VALUES ('{provider}')") - op.create_foreign_key('service_broadcast_settings_provider_fkey', - 'service_broadcast_settings', - 'broadcast_provider_types', - ['provider'], - ['name']) + pass def downgrade(): - op.drop_constraint('service_broadcast_settings_provider_fkey', 'service_broadcast_settings', type_='foreignkey') - op.drop_table('broadcast_provider_types') + pass diff --git a/migrations/versions/0353_broadcast_provider_not_null.py b/migrations/versions/0353_broadcast_provider_not_null.py index c470de38b..6c9f7f2b5 100644 --- a/migrations/versions/0353_broadcast_provider_not_null.py +++ b/migrations/versions/0353_broadcast_provider_not_null.py @@ -13,10 +13,8 @@ down_revision = '0352_broadcast_provider_types' def upgrade(): - op.execute("UPDATE service_broadcast_settings SET provider = 'all' WHERE provider is null") - op.alter_column('service_broadcast_settings', 'provider', existing_type=sa.VARCHAR(), nullable=False) + pass def downgrade(): - op.alter_column('service_broadcast_settings', 'provider', existing_type=sa.VARCHAR(), nullable=True) - op.execute("UPDATE service_broadcast_settings SET provider = null WHERE provider = 'all'") + pass diff --git a/migrations/versions/0354_government_channel.py b/migrations/versions/0354_government_channel.py index 25965c848..fc18b388f 100644 --- a/migrations/versions/0354_government_channel.py +++ b/migrations/versions/0354_government_channel.py @@ -12,11 +12,8 @@ down_revision = '0353_broadcast_provider_not_null' def upgrade(): - op.execute("INSERT INTO broadcast_channel_types VALUES ('government')") + pass def downgrade(): - # This can't be downgraded if there are rows in service_broadcast_settings which - # have the channel set to government or if broadcasts have already been sent on the - # government channel - it would break foreign key constraints. - op.execute("DELETE FROM broadcast_channel_types WHERE name = 'government'") + pass diff --git a/migrations/versions/0358_operator_channel.py b/migrations/versions/0358_operator_channel.py index eecfbac53..c36481510 100644 --- a/migrations/versions/0358_operator_channel.py +++ b/migrations/versions/0358_operator_channel.py @@ -12,11 +12,8 @@ down_revision = '0357_validate_constraint' def upgrade(): - op.execute("INSERT INTO broadcast_channel_types VALUES ('operator')") + pass def downgrade(): - # This can't be downgraded if there are rows in service_broadcast_settings which - # have the channel set to operator or if broadcasts have already been sent on the - # operator channel - it would break foreign key constraints. - op.execute("DELETE FROM broadcast_channel_types WHERE name = 'operator'") + pass diff --git a/migrations/versions/0359_more_permissions.py b/migrations/versions/0359_more_permissions.py index 329090e98..7188d5df3 100644 --- a/migrations/versions/0359_more_permissions.py +++ b/migrations/versions/0359_more_permissions.py @@ -11,40 +11,10 @@ import sqlalchemy as sa revision = '0359_more_permissions' down_revision = '0358_operator_channel' -enum_name = 'permission_types' -tmp_name = 'tmp_' + enum_name - -old_options = ( - 'manage_users', - 'manage_templates', - 'manage_settings', - 'send_texts', - 'send_emails', - 'send_letters', - 'manage_api_keys', - 'platform_admin', - 'view_activity', -) -old_type = sa.Enum(*old_options, name=enum_name) - def upgrade(): - # ALTER TYPE must be run outside of a transaction block (see link below for details) - # https://alembic.sqlalchemy.org/en/latest/api/runtime.html#alembic.runtime.migration.MigrationContext.autocommit_block - with op.get_context().autocommit_block(): - op.execute("ALTER TYPE permission_types ADD VALUE 'create_broadcasts'") - op.execute("ALTER TYPE permission_types ADD VALUE 'approve_broadcasts'") - op.execute("ALTER TYPE permission_types ADD VALUE 'cancel_broadcasts'") - op.execute("ALTER TYPE permission_types ADD VALUE 'reject_broadcasts'") + pass def downgrade(): - op.execute( - "DELETE FROM permissions WHERE permission in " - "('create_broadcasts', 'approve_broadcasts', 'cancel_broadcasts', 'reject_broadcasts')" - ) - - op.execute(f'ALTER TYPE {enum_name} RENAME TO {tmp_name}') - old_type.create(op.get_bind()) - op.execute(f'ALTER TABLE permissions ALTER COLUMN permission TYPE {enum_name} USING permission::text::{enum_name}') - op.execute(f'DROP TYPE {tmp_name}') + pass diff --git a/migrations/versions/0362_broadcast_msg_event.py b/migrations/versions/0362_broadcast_msg_event.py index 04146958c..da94ada7e 100644 --- a/migrations/versions/0362_broadcast_msg_event.py +++ b/migrations/versions/0362_broadcast_msg_event.py @@ -14,8 +14,8 @@ down_revision = '0361_new_user_bcast_permissions' def upgrade(): - op.add_column('broadcast_message', sa.Column('cap_event', sa.String(length=255), nullable=True)) + pass def downgrade(): - op.drop_column('broadcast_message', 'cap_event') + pass diff --git a/migrations/versions/0363_cancelled_by_api_key.py b/migrations/versions/0363_cancelled_by_api_key.py index 20856a287..9896efb1c 100644 --- a/migrations/versions/0363_cancelled_by_api_key.py +++ b/migrations/versions/0363_cancelled_by_api_key.py @@ -13,33 +13,11 @@ down_revision = '0362_broadcast_msg_event' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.add_column('broadcast_message', sa.Column('created_by_api_key_id', postgresql.UUID(as_uuid=True), nullable=True)) - op.add_column( - 'broadcast_message', sa.Column('cancelled_by_api_key_id', postgresql.UUID(as_uuid=True), nullable=True) - ) - op.drop_constraint('broadcast_message_api_key_id_fkey', 'broadcast_message', type_='foreignkey') - op.create_foreign_key( - 'broadcast_message_created_by_api_key_id_fkey', - 'broadcast_message', - 'api_keys', - ['created_by_api_key_id'], - ['id'] - ) - op.create_foreign_key( - 'broadcast_message_cancelled_by_api_key_id_fkey', - 'broadcast_message', - 'api_keys', - ['cancelled_by_api_key_id'], - ['id'] - ) + pass # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint('broadcast_message_created_by_api_key_id_fkey', 'broadcast_message', type_='foreignkey') - op.drop_constraint('broadcast_message_cancelled_by_api_key_id_fkey', 'broadcast_message', type_='foreignkey') - op.create_foreign_key('broadcast_message_api_key_id_fkey', 'broadcast_message', 'api_keys', ['api_key_id'], ['id']) - op.drop_column('broadcast_message', 'cancelled_by_api_key_id') - op.drop_column('broadcast_message', 'created_by_api_key_id') + pass # ### end Alembic commands ### diff --git a/migrations/versions/0364_drop_old_column.py b/migrations/versions/0364_drop_old_column.py index 40b730b9c..039b229a5 100644 --- a/migrations/versions/0364_drop_old_column.py +++ b/migrations/versions/0364_drop_old_column.py @@ -14,20 +14,8 @@ down_revision = '0363_cancelled_by_api_key' def upgrade(): - # move data over - op.execute("UPDATE broadcast_message SET created_by_api_key_id=api_key_id WHERE created_by_api_key_id IS NULL") - op.create_check_constraint( - "ck_broadcast_message_created_by_not_null", - "broadcast_message", - "created_by_id is not null or created_by_api_key_id is not null" - ) - op.drop_column('broadcast_message', 'api_key_id') + pass def downgrade(): - op.add_column('broadcast_message', sa.Column('api_key_id', postgresql.UUID(), autoincrement=False, nullable=True)) - op.execute("UPDATE broadcast_message SET api_key_id=created_by_api_key_id") # move data over - op.drop_constraint( - "ck_broadcast_message_created_by_not_null", - "broadcast_message" - ) + pass From 434b7b2d081ce2736090444b5c9948fe7455a047 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 4 Oct 2022 16:01:30 -0700 Subject: [PATCH 17/65] clean up and remove redundancy --- app/celery/process_ses_receipts_tasks.py | 5 -- app/celery/service_callback_tasks.py | 1 + app/notifications/callbacks.py | 52 ------------------- .../notifications_ses_callback.py | 17 +----- tests/app/db.py | 2 +- tests/app/test_schemas.py | 4 +- tests/test_all_queues_used.py | 1 + 7 files changed, 6 insertions(+), 76 deletions(-) delete mode 100644 app/notifications/callbacks.py diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 22136e7a0..227a28e0b 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -1,10 +1,6 @@ -import enum -import traceback from datetime import datetime, timedelta -from json import decoder import iso8601 -import requests from celery.exceptions import Retry from flask import current_app, json from sqlalchemy.orm.exc import NoResultFound @@ -25,7 +21,6 @@ from app.dao.service_callback_api_dao import ( get_service_delivery_status_callback_api_for_service, ) from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint -from app.notifications.callbacks import create_complaint_callback_data @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index 9ba6ad0ad..8867fca6e 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -114,6 +114,7 @@ def create_delivery_status_callback_data(notification, service_callback_api): "notification_client_reference": notification.client_reference, "notification_to": notification.to, "notification_status": notification.status, + "notification_provider_response": notification.provider_response, # TODO do we have a test for provider_response "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT), "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None, diff --git a/app/notifications/callbacks.py b/app/notifications/callbacks.py deleted file mode 100644 index 253e83af1..000000000 --- a/app/notifications/callbacks.py +++ /dev/null @@ -1,52 +0,0 @@ -from app.celery.service_callback_tasks import send_delivery_status_to_service -from app.config import QueueNames -from app.dao.service_callback_api_dao import ( - get_service_delivery_status_callback_api_for_service, -) - - -def check_and_queue_callback_task(notification): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - notification_data = create_delivery_status_callback_data(notification, service_callback_api) - - send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) - - -def create_delivery_status_callback_data(notification, service_callback_api): - from app import encryption - from app.utils import DATETIME_FORMAT - - data = { - "notification_id": str(notification.id), - "notification_client_reference": notification.client_reference, - "notification_to": notification.to, - "notification_status": notification.status, - "notification_provider_response": notification.provider_response, - "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT), - "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None, - "notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None, - "notification_type": notification.notification_type, - "service_callback_api_url": service_callback_api.url, - "service_callback_api_bearer_token": service_callback_api.bearer_token, - } - - return encryption.encrypt(data) - - -def create_complaint_callback_data(complaint, notification, service_callback_api, recipient): - from app import encryption - from app.utils import DATETIME_FORMAT - - data = { - "complaint_id": str(complaint.id), - "notification_id": str(notification.id), - "reference": notification.client_reference, - "to": recipient, - "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT), - "service_callback_api_url": service_callback_api.url, - "service_callback_api_bearer_token": service_callback_api.bearer_token, - } - - return encryption.encrypt(data) diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index 16e57384e..93f14b37a 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -1,25 +1,10 @@ -import enum from datetime import timedelta -from flask import Blueprint, current_app, json, jsonify, request +from flask import Blueprint, jsonify, request from app.celery.process_ses_receipts_tasks import process_ses_results -from app.celery.service_callback_tasks import ( - create_complaint_callback_data, - create_delivery_status_callback_data, - send_complaint_to_service, - send_delivery_status_to_service, -) from app.config import QueueNames -from app.dao.complaint_dao import save_complaint -from app.dao.notifications_dao import dao_get_notification_history_by_reference -from app.dao.service_callback_api_dao import ( - get_service_complaint_callback_api_for_service, - get_service_delivery_status_callback_api_for_service, -) from app.errors import InvalidRequest -from app.models import Complaint -from app.notifications.callbacks import create_complaint_callback_data from app.notifications.sns_handlers import sns_notification_handler ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) diff --git a/tests/app/db.py b/tests/app/db.py index 8dac3f22c..65b8a06b6 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -566,7 +566,7 @@ def create_api_key(service, key_type=KEY_TYPE_NORMAL, key_name=None): return api_key -def create_inbound_number(number, provider='mmg', active=True, service_id=None): +def create_inbound_number(number, provider='sns', active=True, service_id=None): inbound_number = InboundNumber( id=uuid.uuid4(), number=number, diff --git a/tests/app/test_schemas.py b/tests/app/test_schemas.py index 109678d43..eb7d875dd 100644 --- a/tests/app/test_schemas.py +++ b/tests/app/test_schemas.py @@ -106,7 +106,7 @@ def test_provider_details_schema_returns_user_details( restore_provider_details ): from app.schemas import provider_details_schema - current_sms_provider = get_provider_details_by_identifier('mmg') + current_sms_provider = get_provider_details_by_identifier('sns') current_sms_provider.created_by = sample_user data = provider_details_schema.dump(current_sms_provider) @@ -119,7 +119,7 @@ def test_provider_details_history_schema_returns_user_details( restore_provider_details, ): from app.schemas import provider_details_schema - current_sms_provider = get_provider_details_by_identifier('mmg') + current_sms_provider = get_provider_details_by_identifier('sns') current_sms_provider.created_by_id = sample_user.id data = provider_details_schema.dump(current_sms_provider) diff --git a/tests/test_all_queues_used.py b/tests/test_all_queues_used.py index 1a7166b96..c4c164e7d 100644 --- a/tests/test_all_queues_used.py +++ b/tests/test_all_queues_used.py @@ -1,6 +1,7 @@ from app.config import QueueNames +# NOTE 100422 pass_app_wrapper can probably be removed def test_queue_names_set_in_paas_app_wrapper(): with open("scripts/paas_app_wrapper.sh", 'r') as stream: search = ' -Q ' From 4feaa06f5d85ef232929e4c819a44b97a1f3e63a Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 4 Oct 2022 17:42:04 -0700 Subject: [PATCH 18/65] fix migrations --- app/config.py | 1 + app/notifications/receive_notifications.py | 1 + app/notifications/sns_cert_validator.py | 18 +++++++----------- .../versions/0377_add_inbound_sms_number.py | 16 +++++++++++----- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/app/config.py b/app/config.py index e2af334d5..d64873dad 100644 --- a/app/config.py +++ b/app/config.py @@ -118,6 +118,7 @@ class Config(object): NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' # AWS SNS topics for delivery receipts + VALIDATE_SNS_TOPICS = True VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] # URL of redis instance diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index f15db5306..e8ff5c892 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -27,6 +27,7 @@ INBOUND_SMS_COUNTER = Counter( @receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST']) def receive_sns_sms(): """ + Expected value of the 'Message' key in the incoming payload from SNS { "originationNumber":"+14255550182", "destinationNumber":"+12125550101", diff --git a/app/notifications/sns_cert_validator.py b/app/notifications/sns_cert_validator.py index 08841e1ae..c1da6b030 100644 --- a/app/notifications/sns_cert_validator.py +++ b/app/notifications/sns_cert_validator.py @@ -10,8 +10,7 @@ import six from app import redis_store from app.config import Config -USE_CACHE = True -VALIDATE_ARN = True +VALIDATE_SNS_TOPICS = Config.VALIDATE_SNS_TOPICS VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS @@ -27,19 +26,16 @@ class ValidationError(Exception): def get_certificate(url): - if USE_CACHE: - res = redis_store.get(url) - if res is not None: - return res - res = requests.get(url).text - redis_store.set(url, res, ex=60 * 60) # 60 minutes + res = redis_store.get(url) + if res is not None: return res - else: - return requests.get(url).text + res = requests.get(url).text + redis_store.set(url, res, ex=60 * 60) # 60 minutes + return res def validate_arn(sns_payload): - if VALIDATE_ARN: + if VALIDATE_SNS_TOPICS: arn = sns_payload.get('TopicArn') topic_name = arn.split(':')[5] if topic_name not in VALID_SNS_TOPICS: diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py index 6a8ccab0f..0e808bae7 100644 --- a/migrations/versions/0377_add_inbound_sms_number.py +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -20,6 +20,12 @@ DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): op.get_bind() + + # delete the previous inbound_number with mmg as provider + table_name = 'inbound_numbers' + select_by_col = 'number' + select_by_val = INBOUND_NUMBER + op.execute(f"delete from {table_name} where {select_by_col} = '{select_by_val}'") # add the inbound number for the default service to inbound_numbers table_name = 'inbound_numbers' @@ -29,12 +35,10 @@ def upgrade(): # add the inbound number for the default service to service_sms_senders table_name = 'service_sms_senders' - id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' - is_default = 'true' sms_sender = INBOUND_NUMBER - inbound_number_id = INBOUND_NUMBER_ID - archived = 'false' - op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") + select_by_col = 'id' + select_by_val = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' + op.execute(f"update {table_name} set {'sms_sender'}='{sms_sender}' where {select_by_col} = '{select_by_val}'") # add the inbound number for the default service to inbound_numbers table_name = 'service_permissions' @@ -48,7 +52,9 @@ def downgrade(): delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" + recreate_mmg_inbound_number = f"insert into inbound_numbers (id, number, provider, service_id, active, created_at) VALUES('d7aea27f-340b-4428-9b20-4470dd978bda', '{INBOUND_NUMBER}', 'mmg', 'null', 'false', 'now()')" op.execute(delete_sms_sender) op.execute(delete_inbound_number) op.execute(delete_service_inbound_permission) + op.execute(recreate_mmg_inbound_number) # pass From 1230579f83820fd09cfb2bd93b14af83343eeca2 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 4 Oct 2022 18:03:38 -0700 Subject: [PATCH 19/65] sns_cert_validator caching --- app/notifications/sns_cert_validator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/notifications/sns_cert_validator.py b/app/notifications/sns_cert_validator.py index c1da6b030..57c0396ca 100644 --- a/app/notifications/sns_cert_validator.py +++ b/app/notifications/sns_cert_validator.py @@ -31,6 +31,7 @@ def get_certificate(url): return res res = requests.get(url).text redis_store.set(url, res, ex=60 * 60) # 60 minutes + _signing_cert_cache[url] = res return res From 53204c307b82dbf8c6f47d88fe99d3d4b1c56329 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 5 Oct 2022 01:12:35 +0000 Subject: [PATCH 20/65] tests are, uh, mostly passing --- app/cloudfoundry_config.py | 11 +++++++++-- app/commands.py | 16 +++++++--------- app/config.py | 4 ++-- app/template/rest.py | 8 ++------ app/user/rest.py | 6 +++--- app/utils.py | 6 +----- tests/app/celery/test_scheduled_tasks.py | 6 +----- tests/app/conftest.py | 5 +---- .../test_notifications_sms_callbacks.py | 8 ++++++++ tests/app/test_commands.py | 3 +-- tests/app/test_config.py | 2 +- tests/conftest.py | 2 +- 12 files changed, 37 insertions(+), 40 deletions(-) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index c87727bb7..8ba6de836 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -8,6 +8,7 @@ def find_by_service_name(services, service_name): return services[i] return None + def extract_cloudfoundry_config(): vcap_services = json.loads(os.environ['VCAP_SERVICES']) @@ -18,7 +19,10 @@ def extract_cloudfoundry_config(): os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss') # CSV Upload Bucket Name - bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") + bucket_service = find_by_service_name( + vcap_services['s3'], + f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}" + ) if bucket_service: os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] @@ -26,7 +30,10 @@ def extract_cloudfoundry_config(): os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] # Contact List Bucket Name - bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") + bucket_service = find_by_service_name( + vcap_services['s3'], + f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}" + ) if bucket_service: os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] diff --git a/app/commands.py b/app/commands.py index 803908c4f..fbbbbaf42 100644 --- a/app/commands.py +++ b/app/commands.py @@ -40,7 +40,6 @@ from app.dao.organisation_dao import ( dao_get_organisation_by_email_address, dao_get_organisation_by_id, ) -from app.dao.permissions_dao import permission_dao from app.dao.services_dao import ( dao_fetch_all_services_by_user, dao_fetch_all_services_created_by_user, @@ -64,7 +63,6 @@ from app.models import ( LetterBranding, Notification, Organisation, - Permission, Service, User, ) @@ -151,8 +149,8 @@ def backfill_notification_statuses(): `Notification._status_enum` """ LIMIT = 250000 - subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input + subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -169,7 +167,7 @@ def update_notification_international_flag(): """ # 250,000 rows takes 30 seconds to update. subq = "select id from notifications where international is null limit 250000" - update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input + update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -180,7 +178,7 @@ def update_notification_international_flag(): # Now update notification_history subq_history = "select id from notification_history where international is null limit 250000" - update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input + update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input result_history = db.session.execute(subq_history).fetchall() while len(result_history) > 0: db.session.execute(update_history) @@ -201,8 +199,8 @@ def fix_notification_statuses_not_in_sync(): """ MAX = 10000 - subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input - update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input + subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input + update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input result = db.session.execute(subq).fetchall() while len(result) > 0: @@ -212,7 +210,7 @@ def fix_notification_statuses_not_in_sync(): result = db.session.execute(subq).fetchall() subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input + update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input result = db.session.execute(subq_hist).fetchall() while len(result) > 0: diff --git a/app/config.py b/app/config.py index 6897dcf65..941ae897c 100644 --- a/app/config.py +++ b/app/config.py @@ -144,7 +144,7 @@ class Config(object): MAX_VERIFY_CODE_COUNT = 5 MAX_FAILED_LOGIN_COUNT = 10 - SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service + SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # be careful increasing this size without being sure that we won't see slowness in pysftp MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb @@ -164,7 +164,7 @@ class Config(object): SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246' EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164' NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27' - PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password + PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb' CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884' SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde' diff --git a/app/template/rest.py b/app/template/rest.py index 45675eba9..cf313d297 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -28,12 +28,7 @@ from app.dao.templates_dao import ( ) from app.errors import InvalidRequest, register_errors from app.letters.utils import get_letter_pdf_and_metadata -from app.models import ( - LETTER_TYPE, - SECOND_CLASS, - SMS_TYPE, - Template, -) +from app.models import LETTER_TYPE, SECOND_CLASS, SMS_TYPE, Template from app.notifications.validators import check_reply_to, service_has_permission from app.schema_validation import validate from app.schemas import ( @@ -171,6 +166,7 @@ def get_all_templates_for_service(service_id): data = template_schema.dump(templates, many=True) else: data = template_schema_no_detail.dump(templates, many=True) + print(data) return jsonify(data=data) diff --git a/app/user/rest.py b/app/user/rest.py index b0832e6a8..9014d8dad 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -380,7 +380,7 @@ def send_new_user_email_verification(user_id): template = dao_get_template_by_id(current_app.config['NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID']) service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) - + current_app.logger.info('template.id is {}'.format(template.id)) current_app.logger.info('service.id is {}'.format(service.id)) @@ -438,11 +438,11 @@ def send_already_registered_email(user_id): key_type=KEY_TYPE_NORMAL, reply_to_text=service.get_default_reply_to_email_address() ) - + current_app.logger.info('Sending notification to queue') send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) - + current_app.logger.info('Sent notification to queue') return jsonify({}), 204 diff --git a/app/utils.py b/app/utils.py index 4ed476d58..c04e37793 100644 --- a/app/utils.py +++ b/app/utils.py @@ -87,11 +87,7 @@ def get_london_month_from_utc_column(column): def get_public_notify_type_text(notify_type, plural=False): - from app.models import ( - PRECOMPILED_LETTER, - SMS_TYPE, - UPLOAD_DOCUMENT, - ) + from app.models import PRECOMPILED_LETTER, SMS_TYPE, UPLOAD_DOCUMENT notify_type_text = notify_type if notify_type == SMS_TYPE: notify_type_text = 'text message' diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 32b1adcbb..652f79148 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -34,11 +34,7 @@ from app.models import ( NOTIFICATION_PENDING_VIRUS_CHECK, ) from tests.app import load_example_csv -from tests.app.db import ( - create_job, - create_notification, - create_template, -) +from tests.app.db import create_job, create_notification, create_template from tests.conftest import set_config diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 36229c59b..fad01bc04 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -13,10 +13,7 @@ from app.dao.api_key_dao import save_model_api_key from app.dao.invited_user_dao import save_invited_user from app.dao.jobs_dao import dao_create_job from app.dao.notifications_dao import dao_create_notification -from app.dao.organisation_dao import ( - dao_add_service_to_organisation, - dao_create_organisation, -) +from app.dao.organisation_dao import dao_create_organisation from app.dao.services_dao import dao_add_user_to_service, dao_create_service from app.dao.templates_dao import dao_create_template from app.dao.users_dao import create_secret_code, create_user_code diff --git a/tests/app/notifications/test_notifications_sms_callbacks.py b/tests/app/notifications/test_notifications_sms_callbacks.py index 3b35fe361..523e7e584 100644 --- a/tests/app/notifications/test_notifications_sms_callbacks.py +++ b/tests/app/notifications/test_notifications_sms_callbacks.py @@ -17,6 +17,7 @@ def mmg_post(client, data): data=data, headers=[('Content-Type', 'application/json')]) + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_not_need_auth(client, mocker): mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response') @@ -36,6 +37,7 @@ def test_firetext_callback_should_return_400_if_empty_reference(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: reference missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_400_if_no_reference(client, mocker): data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00' @@ -45,6 +47,7 @@ def test_firetext_callback_should_return_400_if_no_reference(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: reference missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_400_if_no_status(client, mocker): data = 'mobile=441234123123&time=2016-03-10 14:17:00&reference=notification_id' @@ -54,6 +57,7 @@ def test_firetext_callback_should_return_400_if_no_status(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: status missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( @@ -70,6 +74,7 @@ def test_firetext_callback_should_return_200_and_call_task_with_valid_data(clien queue='sms-callbacks', ) + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_including_a_code_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( @@ -86,6 +91,7 @@ def test_firetext_callback_including_a_code_should_return_200_and_call_task_with queue='sms-callbacks', ) + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification): mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response') @@ -98,6 +104,7 @@ def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification): response = mmg_post(client, data) assert response.status_code == 200 + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_process_mmg_response_returns_400_for_malformed_data(client): data = json.dumps({"reference": "mmg_reference", @@ -114,6 +121,7 @@ def test_process_mmg_response_returns_400_for_malformed_data(client): assert "{} callback failed: {} missing".format('MMG', 'status') in json_data['message'] assert "{} callback failed: {} missing".format('MMG', 'CID') in json_data['message'] + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_mmg_callback_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index d883371bb..e163cdeff 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -5,9 +5,8 @@ from app.commands import ( populate_annual_billing_with_defaults, ) from app.dao.inbound_numbers_dao import dao_get_available_inbound_numbers -from app.dao.services_dao import dao_add_user_to_service from app.models import AnnualBilling -from tests.app.db import create_annual_billing, create_service, create_user +from tests.app.db import create_annual_billing, create_service def test_insert_inbound_numbers_from_file(notify_db_session, notify_api, tmpdir): diff --git a/tests/app/test_config.py b/tests/app/test_config.py index 8e8ba42e1..17bb96bb6 100644 --- a/tests/app/test_config.py +++ b/tests/app/test_config.py @@ -60,7 +60,7 @@ def test_load_config_if_cloudfoundry_not_available(reload_config): def test_queue_names_all_queues_correct(): # Need to ensure that all_queues() only returns queue names used in API queues = QueueNames.all_queues() - assert len(queues) == 18 + assert len(queues) == 17 assert set([ QueueNames.PRIORITY, QueueNames.PERIODIC, diff --git a/tests/conftest.py b/tests/conftest.py index dabd3eb33..5678bfa3a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest import sqlalchemy from alembic.command import upgrade from alembic.config import Config -from flask import Flask +from flask import Flask, current_app from app import create_app, db from app.dao.provider_details_dao import get_provider_details_by_identifier From 97aa118fee446410d855530ced80aa2018e51f1d Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Tue, 4 Oct 2022 18:16:19 -0700 Subject: [PATCH 21/65] additional type checking in process_ses_receipt_tasks --- app/celery/process_ses_receipts_tasks.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 227a28e0b..959f1ab36 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -108,7 +108,7 @@ def determine_notification_bounce_type(ses_message): return notification_type if notification_type != "Bounce": - raise KeyError(f"Unhandled notification type {notification_type}") + raise KeyError(f"Unhandled sns notification type {notification_type}") remove_emails_from_bounce(ses_message) current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) @@ -116,6 +116,13 @@ def determine_notification_bounce_type(ses_message): return "Permanent" return "Temporary" +def determine_notification_type(ses_message): + notification_type = ses_message["notificationType"] + if notification_type not in ["Bounce","Complaint","Delivery"]: + raise KeyError(f"Unhandled sns notification type {notification_type}") + if notification_type == 'Bounce': + return determine_notification_bounce_type(ses_message) + return notification_type def _determine_provider_response(ses_message): if ses_message["notificationType"] != "Bounce": @@ -136,7 +143,7 @@ def _determine_provider_response(ses_message): def get_aws_responses(ses_message): - status = determine_notification_bounce_type(ses_message) + status = determine_notification_type(ses_message) base = { "Permanent": { From 863dfe39bacf49e50fdade18b2b967d922aad3b7 Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Fri, 7 Oct 2022 17:25:31 -0700 Subject: [PATCH 22/65] config flag for turning off inbound SMS replies --- app/config.py | 3 +++ app/notifications/receive_notifications.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/app/config.py b/app/config.py index d64873dad..1564bc467 100644 --- a/app/config.py +++ b/app/config.py @@ -113,6 +113,9 @@ class Config(object): # Firetext API Key FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") + + # Whether to ignore POSTs from SNS for replies to SMS we sent + RECEIVE_INBOUND_SMS = False # Use notify.sandbox.10x sending domain unless overwritten by environment NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index e8ff5c892..b1099d00b 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -38,6 +38,12 @@ def receive_sns_sms(): } """ + # Whether or not to ignore inbound SMS replies + if not current_app.config['RECEIVE_INBOUND_SMS']: + return jsonify( + result="success", message="SMS-SNS callback succeeded" + ), 200 + try: post_data = sns_notification_handler(request.data, request.headers) except Exception as e: From 0186095920f7a779b4186a48635d5b41f2c0801e Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 26 Sep 2022 16:47:57 +0000 Subject: [PATCH 23/65] swap out uk org types for us-specific org types --- app/broadcast_message/utils.py | 2 +- app/commands.py | 2 +- app/dao/annual_billing_dao.py | 37 ++++---------------- app/dao/services_dao.py | 11 ------ app/models.py | 8 ++--- app/organisation/rest.py | 8 +---- migrations/versions/0376_add_org_names.py | 29 ++++++++++++++++ tests/app/billing/test_rest.py | 2 +- tests/app/dao/test_annual_billing_dao.py | 42 ++++++++--------------- tests/app/dao/test_services_dao.py | 1 + tests/app/db.py | 2 +- tests/app/organisation/test_rest.py | 35 ++++++++++--------- tests/app/test_commands.py | 5 ++- 13 files changed, 80 insertions(+), 104 deletions(-) create mode 100644 migrations/versions/0376_add_org_names.py diff --git a/app/broadcast_message/utils.py b/app/broadcast_message/utils.py index 0aa500eca..429e34607 100644 --- a/app/broadcast_message/utils.py +++ b/app/broadcast_message/utils.py @@ -89,7 +89,7 @@ def _create_p1_zendesk_alert(broadcast_message): ticket_type=NotifySupportTicket.TYPE_INCIDENT, technical_ticket=True, org_id=current_app.config['BROADCAST_ORGANISATION_ID'], - org_type='central', + org_type='federal', service_id=str(broadcast_message.service_id), p1=True ) diff --git a/app/commands.py b/app/commands.py index ad944178d..0515a755f 100644 --- a/app/commands.py +++ b/app/commands.py @@ -462,7 +462,7 @@ def replay_daily_sorted_count_files(file_extension): help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains") def populate_organisations_from_file(file_name): # [0] organisation name:: name of the organisation insert if organisation is missing. - # [1] sector:: Central | Local | NHS only + # [1] sector:: Federal | State only # [2] crown:: TRUE | FALSE only # [3] argeement_signed:: TRUE | FALSE # [4] domains:: comma separated list of domains related to the organisation diff --git a/app/dao/annual_billing_dao.py b/app/dao/annual_billing_dao.py index 2a593bfb1..56cd7901d 100644 --- a/app/dao/annual_billing_dao.py +++ b/app/dao/annual_billing_dao.py @@ -55,46 +55,21 @@ def dao_get_all_free_sms_fragment_limit(service_id): def set_default_free_allowance_for_service(service, year_start=None): default_free_sms_fragment_limits = { - 'central': { + 'federal': { 2020: 250_000, 2021: 150_000, 2022: 40_000, }, - 'local': { - 2020: 25_000, - 2021: 25_000, - 2022: 20_000, - }, - 'nhs_central': { + 'state': { 2020: 250_000, 2021: 150_000, 2022: 40_000, }, - 'nhs_local': { - 2020: 25_000, - 2021: 25_000, - 2022: 20_000, - }, - 'nhs_gp': { - 2020: 25_000, - 2021: 10_000, - 2022: 10_000, - }, - 'emergency_service': { - 2020: 25_000, - 2021: 25_000, - 2022: 20_000, - }, - 'school_or_college': { - 2020: 25_000, - 2021: 10_000, - 2022: 10_000, - }, 'other': { - 2020: 25_000, - 2021: 10_000, - 2022: 10_000, - }, + 2020: 250_000, + 2021: 150_000, + 2022: 40_000, + } } if not year_start: year_start = get_current_financial_year_start_year() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 1f951dcb7..f8c60272f 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -16,14 +16,11 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id from app.models import ( - CROWN_ORGANISATION_TYPES, EMAIL_TYPE, INTERNATIONAL_LETTERS, INTERNATIONAL_SMS_TYPE, KEY_TYPE_TEST, LETTER_TYPE, - NHS_ORGANISATION_TYPES, - NON_CROWN_ORGANISATION_TYPES, NOTIFICATION_PERMANENT_FAILURE, SMS_TYPE, UPLOAD_LETTERS, @@ -324,16 +321,8 @@ def dao_create_service( if organisation.letter_branding: service.letter_branding = organisation.letter_branding - elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address): - service.email_branding = dao_get_email_branding_by_name('NHS') - service.letter_branding = dao_get_letter_branding_by_name('NHS') - if organisation: service.crown = organisation.crown - elif service.organisation_type in CROWN_ORGANISATION_TYPES: - service.crown = True - elif service.organisation_type in NON_CROWN_ORGANISATION_TYPES: - service.crown = False service.count_as_live = not user.platform_admin db.session.add(service) diff --git a/app/models.py b/app/models.py index a69188531..2447ea58e 100644 --- a/app/models.py +++ b/app/models.py @@ -352,12 +352,12 @@ class Domain(db.Model): ORGANISATION_TYPES = [ - "central", "local", "nhs_central", "nhs_local", "nhs_gp", "emergency_service", "school_or_college", "other", + "federal", "state", "other" ] -CROWN_ORGANISATION_TYPES = ["nhs_central"] -NON_CROWN_ORGANISATION_TYPES = ["local", "nhs_local", "nhs_gp", "emergency_service", "school_or_college"] -NHS_ORGANISATION_TYPES = ["nhs_central", "nhs_local", "nhs_gp"] +# CROWN_ORGANISATION_TYPES = ["nhs_central"] +# NON_CROWN_ORGANISATION_TYPES = ["local", "nhs_local", "nhs_gp", "emergency_service", "school_or_college"] +# NHS_ORGANISATION_TYPES = ["nhs_central", "nhs_local", "nhs_gp"] class OrganisationTypes(db.Model): diff --git a/app/organisation/rest.py b/app/organisation/rest.py index f7184fbe3..a44cb8b9c 100644 --- a/app/organisation/rest.py +++ b/app/organisation/rest.py @@ -22,7 +22,7 @@ from app.dao.services_dao import dao_fetch_service_by_id from app.dao.templates_dao import dao_get_template_by_id from app.dao.users_dao import get_user_by_id from app.errors import InvalidRequest, register_errors -from app.models import KEY_TYPE_NORMAL, NHS_ORGANISATION_TYPES, Organisation +from app.models import KEY_TYPE_NORMAL, Organisation from app.notifications.process_notifications import ( persist_notification, send_notification_to_queue, @@ -93,9 +93,6 @@ def create_organisation(): validate(data, post_create_organisation_schema) - if data["organisation_type"] in NHS_ORGANISATION_TYPES: - data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID'] - organisation = Organisation(**data) dao_create_organisation(organisation) return jsonify(organisation.serialize()), 201 @@ -108,9 +105,6 @@ def update_organisation(organisation_id): organisation = dao_get_organisation_by_id(organisation_id) - if data.get('organisation_type') in NHS_ORGANISATION_TYPES and not organisation.email_branding_id: - data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID'] - result = dao_update_organisation(organisation_id, **data) if data.get('agreement_signed') is True: diff --git a/migrations/versions/0376_add_org_names.py b/migrations/versions/0376_add_org_names.py new file mode 100644 index 000000000..76d539e88 --- /dev/null +++ b/migrations/versions/0376_add_org_names.py @@ -0,0 +1,29 @@ +""" + +Revision ID: 0376_add_org_names +Revises: 0375_fix_service_name +Create Date: 2022-09-23 20:04:00.766980 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0376_add_org_names' +down_revision = '0375_fix_service_name' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.get_bind() + + op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000');") + + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + pass + # ### end Alembic commands ### diff --git a/tests/app/billing/test_rest.py b/tests/app/billing/test_rest.py index f02b41b88..b4213a8f7 100644 --- a/tests/app/billing/test_rest.py +++ b/tests/app/billing/test_rest.py @@ -108,7 +108,7 @@ def test_get_free_sms_fragment_limit_current_year_creates_new_row_if_annual_bill ) assert json_response['financial_year_start'] == 2021 - assert json_response['free_sms_fragment_limit'] == 10000 # based on other organisation type + assert json_response['free_sms_fragment_limit'] == 150000 # based on other organisation type def test_update_free_sms_fragment_limit_data(client, sample_service): diff --git a/tests/app/dao/test_annual_billing_dao.py b/tests/app/dao/test_annual_billing_dao.py index de91048de..56e191462 100644 --- a/tests/app/dao/test_annual_billing_dao.py +++ b/tests/app/dao/test_annual_billing_dao.py @@ -47,31 +47,17 @@ def test_dao_update_annual_billing_for_future_years(notify_db_session, sample_se @pytest.mark.parametrize('org_type, year, expected_default', - [('central', 2021, 150000), - ('local', 2021, 25000), - ('nhs_central', 2021, 150000), - ('nhs_local', 2021, 25000), - ('nhs_gp', 2021, 10000), - ('emergency_service', 2021, 25000), - ('school_or_college', 2021, 10000), - ('other', 2021, 10000), - (None, 2021, 10000), - ('central', 2020, 250000), - ('local', 2020, 25000), - ('nhs_central', 2020, 250000), - ('nhs_local', 2020, 25000), - ('nhs_gp', 2020, 25000), - ('emergency_service', 2020, 25000), - ('school_or_college', 2020, 25000), - ('other', 2020, 25000), - (None, 2020, 25000), - ('central', 2019, 250000), - ('school_or_college', 2022, 10000), - ('central', 2022, 40000), - ('local', 2022, 20000), - ('nhs_local', 2022, 20000), - ('emergency_service', 2022, 20000), - ('central', 2023, 40000), + [('federal', 2021, 150000), + ('state', 2021, 150000), + (None, 2021, 150000), + ('federal', 2020, 250000), + ('state', 2020, 250000), + ('other', 2020, 250000), + (None, 2020, 250000), + ('federal', 2019, 250000), + ('federal', 2022, 40000), + ('state', 2022, 40000), + ('federal', 2023, 40000), ]) def test_set_default_free_allowance_for_service(notify_db_session, org_type, year, expected_default): @@ -93,7 +79,7 @@ def test_set_default_free_allowance_for_service_using_correct_year(sample_servic mock_dao.assert_called_once_with( sample_service.id, - 25000, + 250000, 2020 ) @@ -105,9 +91,9 @@ def test_set_default_free_allowance_for_service_updates_existing_year(sample_ser assert not sample_service.organisation_type assert len(annual_billing) == 1 assert annual_billing[0].service_id == sample_service.id - assert annual_billing[0].free_sms_fragment_limit == 10000 + assert annual_billing[0].free_sms_fragment_limit == 150000 - sample_service.organisation_type = 'central' + sample_service.organisation_type = 'federal' set_default_free_allowance_for_service(service=sample_service, year_start=None) annual_billing = AnnualBilling.query.all() diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index ef6589277..626bdc3b4 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -162,6 +162,7 @@ def test_create_service_with_organisation(notify_db_session): # the NHS branding set up ('SHN', False), )) +@pytest.mark.skip(reason='Update for TTS') def test_create_nhs_service_get_default_branding_based_on_email_address( notify_db_session, branding_name_to_create, diff --git a/tests/app/db.py b/tests/app/db.py index 65b8a06b6..1a95df800 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -122,7 +122,7 @@ def create_service( email_from=None, prefix_sms=True, message_limit=1000, - organisation_type='central', + organisation_type='federal', check_if_service_exists=False, go_live_user=None, go_live_at=None, diff --git a/tests/app/organisation/test_rest.py b/tests/app/organisation/test_rest.py index 13f2fa305..cc5d68a23 100644 --- a/tests/app/organisation/test_rest.py +++ b/tests/app/organisation/test_rest.py @@ -26,7 +26,7 @@ from tests.app.db import ( def test_get_all_organisations(admin_request, notify_db_session): - create_organisation(name='inactive org', active=False, organisation_type='nhs_central') + create_organisation(name='inactive org', active=False, organisation_type='federal') create_organisation(name='active org', domains=['example.com']) response = admin_request.get( @@ -52,7 +52,7 @@ def test_get_all_organisations(admin_request, notify_db_session): assert response[1]['active'] is False assert response[1]['count_of_live_services'] == 0 assert response[1]['domains'] == [] - assert response[1]['organisation_type'] == 'nhs_central' + assert response[1]['organisation_type'] == 'federal' def test_get_organisation_by_id(admin_request, notify_db_session): @@ -169,7 +169,7 @@ def test_post_create_organisation(admin_request, notify_db_session, crown): 'name': 'test organisation', 'active': True, 'crown': crown, - 'organisation_type': 'local', + 'organisation_type': 'state', } response = admin_request.post( @@ -191,6 +191,7 @@ def test_post_create_organisation(admin_request, notify_db_session, crown): @pytest.mark.parametrize('org_type', ["nhs_central", "nhs_local", "nhs_gp"]) +@pytest.mark.skip(reason='Update for TTS') def test_post_create_organisation_sets_default_nhs_branding_for_nhs_orgs( admin_request, notify_db_session, nhs_email_branding, org_type ): @@ -218,7 +219,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample 'name': sample_organisation.name, 'active': True, 'crown': True, - 'organisation_type': 'central', + 'organisation_type': 'federal', } response = admin_request.post( @@ -237,12 +238,12 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample ({ 'active': False, 'crown': True, - 'organisation_type': 'central', + 'organisation_type': 'federal', }, 'name is a required property'), ({ 'active': False, 'name': 'Service name', - 'organisation_type': 'central', + 'organisation_type': 'federal', }, 'crown is a required property'), ({ 'active': False, @@ -253,7 +254,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample 'active': False, 'name': 'Service name', 'crown': None, - 'organisation_type': 'central', + 'organisation_type': 'federal', }, 'crown None is not of type boolean'), ({ 'active': False, @@ -262,7 +263,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample 'organisation_type': 'foo', }, ( 'organisation_type foo is not one of ' - '[central, local, nhs_central, nhs_local, nhs_gp, emergency_service, school_or_college, other]' + '[federal, state, other]' )), )) def test_post_create_organisation_with_missing_data_gives_validation_error( @@ -295,7 +296,7 @@ def test_post_update_organisation_updates_fields( 'name': 'new organisation name', 'active': False, 'crown': crown, - 'organisation_type': 'central', + 'organisation_type': 'federal', } assert org.crown is None @@ -314,7 +315,7 @@ def test_post_update_organisation_updates_fields( assert organisation[0].active == data['active'] assert organisation[0].crown == crown assert organisation[0].domains == [] - assert organisation[0].organisation_type == 'central' + assert organisation[0].organisation_type == 'federal' @pytest.mark.parametrize('domain_list', ( @@ -371,6 +372,7 @@ def test_update_other_organisation_attributes_doesnt_clear_domains( @pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"]) +@pytest.mark.skip(reason='Update for TTS') def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present( admin_request, nhs_email_branding, @@ -398,6 +400,7 @@ def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present( @pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"]) +@pytest.mark.skip(reason='Update for TTS') def test_post_update_organisation_to_nhs_type_does_not_update_branding_if_default_branding_set( admin_request, nhs_email_branding, @@ -581,7 +584,7 @@ def test_post_link_service_to_organisation(admin_request, sample_service): data = { 'service_id': str(sample_service.id) } - organisation = create_organisation(organisation_type='central') + organisation = create_organisation(organisation_type='federal') admin_request.post( 'organisation.link_service_to_organisation', @@ -590,7 +593,7 @@ def test_post_link_service_to_organisation(admin_request, sample_service): _expected_status=204 ) assert len(organisation.services) == 1 - assert sample_service.organisation_type == 'central' + assert sample_service.organisation_type == 'federal' @freeze_time('2021-09-24 13:30') @@ -598,7 +601,7 @@ def test_post_link_service_to_organisation_inserts_annual_billing(admin_request, data = { 'service_id': str(sample_service.id) } - organisation = create_organisation(organisation_type='central') + organisation = create_organisation(organisation_type='federal') assert len(organisation.services) == 0 assert len(AnnualBilling.query.all()) == 0 admin_request.post( @@ -623,7 +626,7 @@ def test_post_link_service_to_organisation_rollback_service_if_annual_billing_up } assert not sample_service.organisation_type - organisation = create_organisation(organisation_type='central') + organisation = create_organisation(organisation_type='federal') assert len(organisation.services) == 0 assert len(AnnualBilling.query.all()) == 0 with pytest.raises(expected_exception=SQLAlchemyError): @@ -655,7 +658,7 @@ def test_post_link_service_to_another_org( assert len(sample_organisation.services) == 1 assert not sample_service.organisation_type - new_org = create_organisation(organisation_type='central') + new_org = create_organisation(organisation_type='federal') admin_request.post( 'organisation.link_service_to_organisation', _data=data, @@ -664,7 +667,7 @@ def test_post_link_service_to_another_org( ) assert not sample_organisation.services assert len(new_org.services) == 1 - assert sample_service.organisation_type == 'central' + assert sample_service.organisation_type == 'federal' annual_billing = AnnualBilling.query.all() assert len(annual_billing) == 1 assert annual_billing[0].free_sms_fragment_limit == 150000 diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 16b21116a..2e0c4c580 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -43,9 +43,8 @@ def test_local_dev_broadcast_permissions( @pytest.mark.parametrize("organisation_type, expected_allowance", - [('central', 40000), - ('local', 20000), - ('nhs_gp', 10000)]) + [('federal', 40000), + ('state', 40000)]) def test_populate_annual_billing_with_defaults( notify_db_session, notify_api, organisation_type, expected_allowance ): From 91c57bd2b4cca1d1e0f9eefc30ebfdb5e2c92d22 Mon Sep 17 00:00:00 2001 From: Steven Reilly Date: Wed, 28 Sep 2022 16:05:57 -0400 Subject: [PATCH 24/65] remove unused org types before adding ours --- migrations/versions/0376_add_org_names.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/migrations/versions/0376_add_org_names.py b/migrations/versions/0376_add_org_names.py index 76d539e88..19bb292b8 100644 --- a/migrations/versions/0376_add_org_names.py +++ b/migrations/versions/0376_add_org_names.py @@ -17,7 +17,8 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.get_bind() - op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000');") + op.execute("TRUNCATE TABLE organisation_types CASCADE;") + op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000'),('other','f','250000');") # ### end Alembic commands ### From b0ed88e7a3518aac045a317e1a52c00d1fccfa45 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 11 Oct 2022 20:07:18 +0000 Subject: [PATCH 25/65] update tests --- tests/app/dao/test_organisation_dao.py | 26 +++++++++++++------------- tests/app/dao/test_services_dao.py | 14 +++++++------- tests/app/service/test_rest.py | 4 ++-- tests/app/test_commands.py | 2 +- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/app/dao/test_organisation_dao.py b/tests/app/dao/test_organisation_dao.py index 91136de8a..3b7dc8ca4 100644 --- a/tests/app/dao/test_organisation_dao.py +++ b/tests/app/dao/test_organisation_dao.py @@ -65,7 +65,7 @@ def test_update_organisation(notify_db_session): data = { 'name': 'new name', "crown": True, - "organisation_type": 'local', + "organisation_type": 'state', "agreement_signed": True, "agreement_signed_at": datetime.datetime.utcnow(), "agreement_signed_by_id": user.id, @@ -124,8 +124,8 @@ def test_update_organisation_does_not_update_the_service_if_certain_attributes_n email_branding = create_email_branding() letter_branding = create_letter_branding() - sample_service.organisation_type = 'local' - sample_organisation.organisation_type = 'central' + sample_service.organisation_type = 'state' + sample_organisation.organisation_type = 'federal' sample_organisation.email_branding = email_branding sample_organisation.letter_branding = letter_branding @@ -138,8 +138,8 @@ def test_update_organisation_does_not_update_the_service_if_certain_attributes_n assert sample_organisation.name == 'updated org name' - assert sample_organisation.organisation_type == 'central' - assert sample_service.organisation_type == 'local' + assert sample_organisation.organisation_type == 'federal' + assert sample_service.organisation_type == 'state' assert sample_organisation.email_branding == email_branding assert sample_service.email_branding is None @@ -152,20 +152,20 @@ def test_update_organisation_updates_the_service_org_type_if_org_type_is_provide sample_service, sample_organisation, ): - sample_service.organisation_type = 'local' - sample_organisation.organisation_type = 'local' + sample_service.organisation_type = 'state' + sample_organisation.organisation_type = 'state' sample_organisation.services.append(sample_service) db.session.commit() - dao_update_organisation(sample_organisation.id, organisation_type='central') + dao_update_organisation(sample_organisation.id, organisation_type='federal') - assert sample_organisation.organisation_type == 'central' - assert sample_service.organisation_type == 'central' + assert sample_organisation.organisation_type == 'federal' + assert sample_service.organisation_type == 'federal' assert Service.get_history_model().query.filter_by( id=sample_service.id, version=2 - ).one().organisation_type == 'central' + ).one().organisation_type == 'federal' def test_update_organisation_updates_the_service_branding_if_branding_is_provided( @@ -228,8 +228,8 @@ def test_update_organisation_updates_services_with_new_crown_type( def test_add_service_to_organisation(sample_service, sample_organisation): assert sample_organisation.services == [] - sample_service.organisation_type = "central" - sample_organisation.organisation_type = "local" + sample_service.organisation_type = "federal" + sample_organisation.organisation_type = "state" sample_organisation.crown = False dao_add_service_to_organisation(sample_service, sample_organisation.id) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 626bdc3b4..7137194ed 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -98,7 +98,7 @@ def test_create_service(notify_db_session): email_from="email_from", message_limit=1000, restricted=False, - organisation_type='central', + organisation_type='federal', created_by=user) dao_create_service(service, user) assert Service.query.count() == 1 @@ -110,7 +110,7 @@ def test_create_service(notify_db_session): assert service_db.prefix_sms is True assert service.active is True assert user in service_db.users - assert service_db.organisation_type == 'central' + assert service_db.organisation_type == 'federal' assert service_db.crown is None assert not service.letter_branding assert not service.organisation_id @@ -119,13 +119,13 @@ def test_create_service(notify_db_session): def test_create_service_with_organisation(notify_db_session): user = create_user(email='local.authority@local-authority.gov.uk') organisation = create_organisation( - name='Some local authority', organisation_type='local', domains=['local-authority.gov.uk']) + name='Some local authority', organisation_type='state', domains=['local-authority.gov.uk']) assert Service.query.count() == 0 service = Service(name="service_name", email_from="email_from", message_limit=1000, restricted=False, - organisation_type='central', + organisation_type='federal', created_by=user) dao_create_service(service, user) assert Service.query.count() == 1 @@ -138,7 +138,7 @@ def test_create_service_with_organisation(notify_db_session): assert service_db.prefix_sms is True assert service.active is True assert user in service_db.users - assert service_db.organisation_type == 'local' + assert service_db.organisation_type == 'state' assert service_db.crown is None assert not service.letter_branding assert service.organisation_id == organisation.id @@ -447,7 +447,7 @@ def test_get_all_user_services_should_return_empty_list_if_no_services_for_user( @freeze_time('2019-04-23T10:00:00') def test_dao_fetch_live_services_data(sample_user): - org = create_organisation(organisation_type='nhs_central') + org = create_organisation(organisation_type='federal') service = create_service(go_live_user=sample_user, go_live_at='2014-04-20T10:00:00') sms_template = create_template(service=service) service_2 = create_service(service_name='second', go_live_at='2017-04-20T10:00:00', go_live_user=sample_user) @@ -485,7 +485,7 @@ def test_dao_fetch_live_services_data(sample_user): # checks the results and that they are ordered by date: assert results == [ {'service_id': mock.ANY, 'service_name': 'Sample service', 'organisation_name': 'test_org_1', - 'organisation_type': 'nhs_central', 'consent_to_research': None, 'contact_name': 'Test User', + 'organisation_type': 'federal', 'consent_to_research': None, 'contact_name': 'Test User', 'contact_email': 'notify@digital.cabinet-office.gov.uk', 'contact_mobile': '+447700900986', 'live_date': datetime(2014, 4, 20, 10, 0), 'sms_volume_intent': None, 'email_volume_intent': None, 'letter_volume_intent': None, 'sms_totals': 2, 'email_totals': 1, 'letter_totals': 1, diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 6102da61a..12e375cbb 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -715,7 +715,7 @@ def test_update_service(client, notify_db_session, sample_service): 'email_from': 'updated.service.name', 'created_by': str(sample_service.created_by.id), 'email_branding': str(brand.id), - 'organisation_type': 'school_or_college', + 'organisation_type': 'federal', } auth_header = create_admin_authorization_header() @@ -730,7 +730,7 @@ def test_update_service(client, notify_db_session, sample_service): assert result['data']['name'] == 'updated service name' assert result['data']['email_from'] == 'updated.service.name' assert result['data']['email_branding'] == str(brand.id) - assert result['data']['organisation_type'] == 'school_or_college' + assert result['data']['organisation_type'] == 'federal' def test_cant_update_service_org_type_to_random_value(client, sample_service): diff --git a/tests/app/test_commands.py b/tests/app/test_commands.py index 2e0c4c580..394cac002 100644 --- a/tests/app/test_commands.py +++ b/tests/app/test_commands.py @@ -66,7 +66,7 @@ def test_populate_annual_billing_with_defaults( def test_populate_annual_billing_with_defaults_sets_free_allowance_to_zero_if_previous_year_is_zero( notify_db_session, notify_api ): - service = create_service(organisation_type='central') + service = create_service(organisation_type='federal') create_annual_billing(service_id=service.id, free_sms_fragment_limit=0, financial_year_start=2021) notify_api.test_cli_runner().invoke( populate_annual_billing_with_defaults, ['-y', 2022] From 0383a4963ea01ff51c93072b5722cf60449b03a2 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 11 Oct 2022 20:38:00 +0000 Subject: [PATCH 26/65] rename migration to be the last --- .../{0376_add_org_names.py => 0378_add_org_names.py} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename migrations/versions/{0376_add_org_names.py => 0378_add_org_names.py} (80%) diff --git a/migrations/versions/0376_add_org_names.py b/migrations/versions/0378_add_org_names.py similarity index 80% rename from migrations/versions/0376_add_org_names.py rename to migrations/versions/0378_add_org_names.py index 19bb292b8..6972bfa66 100644 --- a/migrations/versions/0376_add_org_names.py +++ b/migrations/versions/0378_add_org_names.py @@ -1,7 +1,7 @@ """ -Revision ID: 0376_add_org_names -Revises: 0375_fix_service_name +Revision ID: 0378_add_org_names +Revises: 0377_add_inbound_sms_number Create Date: 2022-09-23 20:04:00.766980 """ @@ -9,8 +9,8 @@ from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import postgresql -revision = '0376_add_org_names' -down_revision = '0375_fix_service_name' +revision = '0378_add_org_names' +down_revision = '0377_add_inbound_sms_number' def upgrade(): From 47904bf0bde1a1bbcc600fb4ecf589b12a8d430d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 11 Oct 2022 20:59:37 +0000 Subject: [PATCH 27/65] swap out org value on existing data before truncating --- migrations/versions/0378_add_org_names.py | 29 ++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/migrations/versions/0378_add_org_names.py b/migrations/versions/0378_add_org_names.py index 6972bfa66..ee25e0846 100644 --- a/migrations/versions/0378_add_org_names.py +++ b/migrations/versions/0378_add_org_names.py @@ -17,14 +17,37 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### op.get_bind() - op.execute("TRUNCATE TABLE organisation_types CASCADE;") - op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000'),('other','f','250000');") + # bluntly swap out data + op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000');") + op.execute("UPDATE services SET organisation_type = 'federal';") + op.execute("UPDATE organisation SET organisation_type = 'federal';") + op.execute("UPDATE services_history SET organisation_type = 'federal';") + # remove uk values + service_delete = """DELETE FROM organisation_types WHERE name IN + ('central','local','nhs','nhs_central','nhs_local','emergency_service','school_or_college','nhs_gp') + """ + op.execute(service_delete) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + service_insert = """INSERT INTO organisation_types VALUES + ('central','','250000') + ('local','f','25000') + ('nhs','','25000') + ('nhs_central','t','250000') + ('nhs_local','f','25000') + ('emergency_service','f','25000') + ('school_or_college','f','25000') + ('nhs_gp','f','25000') + """ + op.execute(service_insert) + op.execute("UPDATE services SET organisation_type = 'central';") + op.execute("UPDATE organisation SET organisation_type = 'central';") + op.execute("UPDATE services_history SET organisation_type = 'central';") + op.execute("DELETE FROM organisation_types WHERE name IN ('federal','state')") + # ### end Alembic commands ### From 1fa48ef353d2cdae69878cb1c28074832f98685e Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Mon, 17 Oct 2022 13:45:38 +0000 Subject: [PATCH 28/65] remove lingering comment --- app/models.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/models.py b/app/models.py index 2447ea58e..ed37aa9cc 100644 --- a/app/models.py +++ b/app/models.py @@ -355,10 +355,6 @@ ORGANISATION_TYPES = [ "federal", "state", "other" ] -# CROWN_ORGANISATION_TYPES = ["nhs_central"] -# NON_CROWN_ORGANISATION_TYPES = ["local", "nhs_local", "nhs_gp", "emergency_service", "school_or_college"] -# NHS_ORGANISATION_TYPES = ["nhs_central", "nhs_local", "nhs_gp"] - class OrganisationTypes(db.Model): __tablename__ = 'organisation_types' From 8dc0b4eb19e08f8a8c22e3190e7b9b317a65d8fa Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 18 Oct 2022 11:22:58 -0400 Subject: [PATCH 29/65] Move process commands from Procfile to manifest.yml --- .github/workflows/deploy.yml | 2 +- Procfile | 2 -- deploy-config/production.yml | 5 +++++ deploy-config/staging.yml | 5 +++++ manifest.yml | 10 ++++++++++ 5 files changed, 21 insertions(+), 3 deletions(-) delete mode 100644 Procfile create mode 100644 deploy-config/production.yml create mode 100644 deploy-config/staging.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index d6120a8b6..b3946fb48 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -70,7 +70,7 @@ jobs: cf_org: gsa-10x-prototyping cf_space: 10x-notifications push_arguments: >- - --var env=staging + --vars-file deploy-config/staging.yml --var DANGEROUS_SALT="$DANGEROUS_SALT" --var SECRET_KEY="$SECRET_KEY" --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET" diff --git a/Procfile b/Procfile deleted file mode 100644 index 4815f7e08..000000000 --- a/Procfile +++ /dev/null @@ -1,2 +0,0 @@ -web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application -worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null \ No newline at end of file diff --git a/deploy-config/production.yml b/deploy-config/production.yml new file mode 100644 index 000000000..c16fff8a0 --- /dev/null +++ b/deploy-config/production.yml @@ -0,0 +1,5 @@ +env: production +web_instances: 2 +web_memory: 1G +worker_instances: 1 +worker_memory: 512M diff --git a/deploy-config/staging.yml b/deploy-config/staging.yml new file mode 100644 index 000000000..43478a524 --- /dev/null +++ b/deploy-config/staging.yml @@ -0,0 +1,5 @@ +env: staging +web_instances: 1 +web_memory: 1G +worker_instances: 1 +worker_memory: 512M diff --git a/manifest.yml b/manifest.yml index 2d365c9dd..baebba899 100644 --- a/manifest.yml +++ b/manifest.yml @@ -17,6 +17,16 @@ applications: - notifications-api-csv-upload-bucket-((env)) - notifications-api-contact-list-bucket-((env)) + processes: + - type: web + instances: ((web_instances)) + memory: ((web_memory)) + command: ./scripts/run_app_paas.sh gunicorn -c ./gunicorn_config.py application + - type: worker + instances: ((worker_instances)) + memory: ((worker_memory)) + command: ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 + env: NOTIFY_APP_NAME: api NOTIFY_LOG_PATH: /home/vcap/logs/app.log From b7e2dfa7e37b432069e3ffe29ff03061de07928d Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 18 Oct 2022 11:54:54 -0400 Subject: [PATCH 30/65] Remove unused scripts files --- scripts/paas_app_wrapper.sh | 67 ----------- scripts/run_multi_worker_app_paas.sh | 167 --------------------------- tests/test_all_queues_used.py | 19 --- 3 files changed, 253 deletions(-) delete mode 100755 scripts/paas_app_wrapper.sh delete mode 100755 scripts/run_multi_worker_app_paas.sh delete mode 100644 tests/test_all_queues_used.py diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh deleted file mode 100755 index 7aeb46868..000000000 --- a/scripts/paas_app_wrapper.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -case $NOTIFY_APP_NAME in - api) - unset GUNICORN_CMD_ARGS - exec scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application - ;; - delivery-worker-retry-tasks) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q retry-tasks 2> /dev/null - ;; - delivery-worker-letters) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null - ;; - delivery-worker-jobs) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q database-tasks,job-tasks 2> /dev/null - ;; - delivery-worker-research) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q research-mode-tasks 2> /dev/null - ;; - delivery-worker-sender) - exec scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 4 -A run_celery.notify_celery --loglevel=INFO \ - --logfile=/dev/null --pidfile=/tmp/celery%N.pid -Q send-sms-tasks,send-email-tasks - ;; - delivery-worker-periodic) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ - -Q periodic-tasks 2> /dev/null - ;; - delivery-worker-reporting) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q reporting-tasks 2> /dev/null - ;; - delivery-worker-priority) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q priority-tasks 2> /dev/null - ;; - # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority - delivery-worker-internal) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q notify-internal-tasks 2> /dev/null - ;; - delivery-worker-broadcasts) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ - -Q broadcast-tasks 2> /dev/null - ;; - delivery-worker-receipts) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q ses-callbacks,sms-callbacks 2> /dev/null - ;; - delivery-worker-service-callbacks) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q service-callbacks,service-callbacks-retry 2> /dev/null - ;; - delivery-worker-save-api-notifications) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q save-api-email-tasks,save-api-sms-tasks 2> /dev/null - ;; - delivery-celery-beat) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO - ;; - *) - echo "Unknown notify_app_name $NOTIFY_APP_NAME" - exit 1 - ;; -esac diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh deleted file mode 100755 index 15495b873..000000000 --- a/scripts/run_multi_worker_app_paas.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -TERMINATE_TIMEOUT=9 -MAX_DISK_SPACE_USAGE=75 -readonly LOGS_DIR="/home/vcap/logs" - -function check_params { - if [ -z "${NOTIFY_APP_NAME}" ]; then - echo "You must set NOTIFY_APP_NAME" - exit 1 - fi - - if [ -z "${CW_APP_NAME}" ]; then - CW_APP_NAME=${NOTIFY_APP_NAME} - fi -} - -function configure_aws_logs { - # create files so that aws logs agent doesn't complain - touch ${LOGS_DIR}/gunicorn_error.log - touch ${LOGS_DIR}/app.log.json - - aws configure set plugins.cwlogs cwlogs - - cat > /home/vcap/app/awslogs.conf << EOF -[general] -state_file = ${LOGS_DIR}/awslogs-state - -[${LOGS_DIR}/app.log] -file = ${LOGS_DIR}/app.log.json -log_group_name = paas-${CW_APP_NAME}-application -log_stream_name = {hostname} - -[${LOGS_DIR}/gunicorn_error.log] -file = ${LOGS_DIR}/gunicorn_error.log -log_group_name = paas-${CW_APP_NAME}-gunicorn -log_stream_name = {hostname} -EOF -} - -# For every PID, check if it's still running. if it is, send the sigterm. then wait 9 seconds before sending sigkill -function on_exit { - echo "multi worker app exiting" - wait_time=0 - - send_signal_to_celery_processes TERM - - # check if the apps are still running every second - while [[ "$wait_time" -le "$TERMINATE_TIMEOUT" ]]; do - get_celery_pids - ensure_celery_is_running - let wait_time=wait_time+1 - sleep 1 - done - - send_signal_to_celery_processes KILL -} - -function check_disk_space { - # get something like: - # - # Filesystem Use% - # overlay 56% - # tmpfs 0% - # - # and only keep '56' - SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%") - - if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then - echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space" - echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%" - exit - fi -} - -function get_celery_pids { - # get the PIDs of the process whose parent is the root process - # print only pid and their command, get the ones with "celery" in their name - # and keep only these PIDs - - set +o pipefail # so grep returning no matches does not premature fail pipe - APP_PIDS=$(pgrep -P 1 | xargs ps -o pid=,command= -p | grep celery | cut -f1 -d/) - set -o pipefail # pipefail should be set everywhere else -} - -function send_signal_to_celery_processes { - # refresh pids to account for the case that some workers may have terminated but others not - get_celery_pids - # send signal to all remaining apps - echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: " - echo ${APP_PIDS} | xargs kill -s ${1} -} - -function start_application { - echo "Starting application..." - eval "$@" - get_celery_pids - echo "Application process pids: "${APP_PIDS} -} - -function start_aws_logs_agent { - echo "Starting aws logs agent..." - exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & - AWSLOGS_AGENT_PID=$! - echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" -} - -function start_logs_tail { - echo "Starting logs tail..." - exec tail -n0 -f ${LOGS_DIR}/app.log.json & - LOGS_TAIL_PID=$! - echo "tail pid: ${LOGS_TAIL_PID}" -} - -function ensure_celery_is_running { - if [ "${APP_PIDS}" = "" ]; then - echo "There are no celery processes running, this container is bad" - - echo "Exporting CF information for diagnosis" - - env | grep CF - - echo "Sleeping 15 seconds for logs to get shipped" - - sleep 15 - - echo "Killing awslogs_agent and tail" - kill -9 ${AWSLOGS_AGENT_PID} - kill -9 ${LOGS_TAIL_PID} - - exit 1 - fi -} - -function run { - while true; do - check_disk_space - get_celery_pids - - ensure_celery_is_running - - for APP_PID in ${APP_PIDS}; do - kill -0 ${APP_PID} 2&>/dev/null || return 1 - done - kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent - kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail - sleep 1 - done -} - -echo "Run script pid: $$" - -check_params - -trap "on_exit" EXIT TERM - -configure_aws_logs - -# The application has to start first! -start_application "$@" - -start_aws_logs_agent -start_logs_tail - -run diff --git a/tests/test_all_queues_used.py b/tests/test_all_queues_used.py deleted file mode 100644 index c4c164e7d..000000000 --- a/tests/test_all_queues_used.py +++ /dev/null @@ -1,19 +0,0 @@ -from app.config import QueueNames - - -# NOTE 100422 pass_app_wrapper can probably be removed -def test_queue_names_set_in_paas_app_wrapper(): - with open("scripts/paas_app_wrapper.sh", 'r') as stream: - search = ' -Q ' - - watched_queues = set() - for line in stream.readlines(): - start_of_queue_arg = line.find(search) - if start_of_queue_arg > 0: - start_of_queue_names = start_of_queue_arg + len(search) - end_of_queue_names = line.find('2>') if '2>' in line else len(line) - watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')}) - - # ses-callbacks isn't used in api (only used in SNS lambda) - ignored_queues = {'ses-callbacks'} - assert watched_queues == set(QueueNames.all_queues()) | ignored_queues From 5682af3747da4c2264d6055699e12c7bc7b9e11e Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 18 Oct 2022 12:27:55 -0400 Subject: [PATCH 31/65] Run migrations on the first web instance startup --- manifest.yml | 2 +- scripts/migrate_and_run_web.sh | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100755 scripts/migrate_and_run_web.sh diff --git a/manifest.yml b/manifest.yml index baebba899..c07573fc6 100644 --- a/manifest.yml +++ b/manifest.yml @@ -21,7 +21,7 @@ applications: - type: web instances: ((web_instances)) memory: ((web_memory)) - command: ./scripts/run_app_paas.sh gunicorn -c ./gunicorn_config.py application + command: ./scripts/migrate_and_run_web.sh - type: worker instances: ((worker_instances)) memory: ((worker_memory)) diff --git a/scripts/migrate_and_run_web.sh b/scripts/migrate_and_run_web.sh new file mode 100755 index 000000000..ae611ab43 --- /dev/null +++ b/scripts/migrate_and_run_web.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then + flask db upgrade +fi + +${HOME}/scripts/run_app_paas.sh gunicorn -c ${HOME}/gunicorn_config.py application From cd7da37fa91f2c8a7dbd6a49cccca4f954cb2acc Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Wed, 19 Oct 2022 10:09:09 -0400 Subject: [PATCH 32/65] Only run pip-audit on runtime dependencies in CI --- .github/workflows/checks.yml | 2 +- .github/workflows/daily_checks.yml | 2 +- Makefile | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8cbbda589..57e11688e 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -73,7 +73,7 @@ jobs: - uses: ./.github/actions/setup-project - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: - inputs: requirements.txt requirements_for_test.txt + inputs: requirements.txt ignore-vulns: PYSEC-2022-237 static-scan: diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 3846c3a79..06dd0bc19 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -40,7 +40,7 @@ jobs: - uses: ./.github/actions/setup-project - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: - inputs: requirements.txt requirements_for_test.txt + inputs: requirements.txt ignore-vulns: PYSEC-2022-237 static-scan: diff --git a/Makefile b/Makefile index 18caff76d..701ae3380 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,8 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req .PHONY: audit audit: pip install --upgrade pip-audit - pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237 + pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237 + -pip-audit -r requirements_for_test.txt -l .PHONY: static-scan static-scan: From 65f15b21b06a9fdff58316a8220b585e2bdf8e2d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 14 Oct 2022 00:13:43 +0000 Subject: [PATCH 33/65] uncomment flake8 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 701ae3380..97ef2b5d6 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ generate-version-file: ## Generates the app version file .PHONY: test test: ## Run tests - # flake8 . + flake8 . isort --check-only ./app ./tests pytest -n4 --maxfail=10 From e9fdfd59f40193f4551c689980eeff41ab6cb9b5 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 14 Oct 2022 14:45:27 +0000 Subject: [PATCH 34/65] clean flake8 except provider code --- app/__init__.py | 2 +- app/authentication/auth.py | 5 +- app/aws/s3.py | 13 ++- app/celery/process_ses_receipts_tasks.py | 30 ++++--- .../process_sms_client_response_tasks.py | 1 - app/celery/research_mode_tasks.py | 2 +- app/celery/service_callback_tasks.py | 2 +- app/clients/__init__.py | 5 +- app/clients/sms/__init__.py | 2 +- app/clients/sms/aws_sns.py | 4 +- app/cloudfoundry_config.py | 13 ++- app/commands.py | 80 ------------------- app/config.py | 34 +++++--- app/dao/notifications_dao.py | 2 + app/delivery/send_to_providers.py | 10 ++- app/inbound_sms/rest.py | 2 +- app/models.py | 8 +- .../notifications_ses_callback.py | 3 +- .../notifications_sms_callback.py | 6 +- app/notifications/process_notifications.py | 8 +- app/notifications/receive_notifications.py | 13 +-- app/notifications/sns_cert_validator.py | 16 ++-- app/notifications/sns_handlers.py | 16 +++- app/user/rest.py | 12 +-- .../celery/test_process_ses_receipts_tasks.py | 26 +++++- .../test_process_sms_client_response_tasks.py | 6 +- tests/app/celery/test_provider_tasks.py | 1 + tests/app/celery/test_reporting_tasks.py | 1 + tests/app/clients/test_aws_sns.py | 2 +- tests/app/clients/test_sms.py | 2 + tests/app/dao/test_provider_details_dao.py | 2 + tests/app/delivery/test_send_to_providers.py | 3 + tests/app/letters/test_letter_utils.py | 2 +- .../test_notifications_sms_callbacks.py | 8 ++ .../notifications/test_post_notifications.py | 2 + 35 files changed, 178 insertions(+), 166 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 4d3e47721..2314485cd 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -199,7 +199,7 @@ def register_blueprint(application): status_blueprint.before_request(requires_no_auth) application.register_blueprint(status_blueprint) - + # delivery receipts ses_callback_blueprint.before_request(requires_no_auth) application.register_blueprint(ses_callback_blueprint) diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 301ed7853..57842ee0c 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -18,7 +18,10 @@ from sqlalchemy.orm.exc import NoResultFound from app.serialised_models import SerialisedService -GENERAL_TOKEN_ERROR_MESSAGE = 'Invalid token: make sure your API token matches the example at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header' # nosec B105 +GENERAL_TOKEN_ERROR_MESSAGE = ''' + Invalid token: make sure your API token matches the example + at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header + ''' # nosec B105 AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram( 'auth_db_connection_duration_seconds', diff --git a/app/aws/s3.py b/app/aws/s3.py index 39189e0ae..7312f4cbe 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -10,18 +10,25 @@ default_access_key = os.environ.get('AWS_ACCESS_KEY_ID') default_secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') default_region = os.environ.get('AWS_REGION') -def get_s3_file(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region): + +def get_s3_file( + bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region +): s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region) return s3_file.get()['Body'].read().decode('utf-8') -def get_s3_object(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region): +def get_s3_object( + bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region +): session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region) s3 = session.resource('s3') return s3.Object(bucket_name, file_location) -def file_exists(bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region): +def file_exists( + bucket_name, file_location, access_key=default_access_key, secret_key=default_secret_key, region=default_region +): try: # try and access metadata of object get_s3_object(bucket_name, file_location, access_key, secret_key, region).metadata diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 959f1ab36..0b8f4bd06 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -29,7 +29,10 @@ def process_ses_results(self, response): ses_message = json.loads(response["Message"]) notification_type = ses_message["notificationType"] # TODO remove after smoke testing on prod is implemented - current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}") + current_app.logger.info( + f"Attempting to process SES delivery status message \ + from SNS with type: {notification_type} and body: {ses_message}" + ) bounce_message = None if notification_type == 'Bounce': @@ -49,13 +52,16 @@ def process_ses_results(self, response): message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): current_app.logger.info( - f"notification not found for reference: {reference} (while attempting update to {notification_status}). " - f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue" + f"notification not found for reference: {reference} \ + (while attempting update to {notification_status}). " + f"Callback may have arrived before notification was \ + persisted to the DB. Adding task to retry queue" ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - "notification not found for reference: {} (while attempting update to {})".format(reference, notification_status) + "notification not found for reference: {} (while \ + attempting update to {})".format(reference, notification_status) ) return @@ -64,7 +70,7 @@ def process_ses_results(self, response): if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}: notifications_dao._duplicate_update_warning( - notification, + notification, notification_status ) return @@ -102,6 +108,7 @@ def process_ses_results(self, response): current_app.logger.exception("Error processing SES results: {}".format(type(e))) self.retry(queue=QueueNames.RETRY) + def determine_notification_bounce_type(ses_message): notification_type = ses_message["notificationType"] if notification_type in ["Delivery", "Complaint"]: @@ -116,14 +123,16 @@ def determine_notification_bounce_type(ses_message): return "Permanent" return "Temporary" + def determine_notification_type(ses_message): notification_type = ses_message["notificationType"] - if notification_type not in ["Bounce","Complaint","Delivery"]: + if notification_type not in ["Bounce", "Complaint", "Delivery"]: raise KeyError(f"Unhandled sns notification type {notification_type}") if notification_type == 'Bounce': return determine_notification_bounce_type(ses_message) return notification_type + def _determine_provider_response(ses_message): if ses_message["notificationType"] != "Bounce": return None @@ -175,7 +184,9 @@ def get_aws_responses(ses_message): def handle_complaint(ses_message): recipient_email = remove_emails_from_complaint(ses_message)[0] - current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + current_app.logger.info( + "Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")) + ) try: reference = ses_message["mail"]["messageId"] except KeyError as e: @@ -219,7 +230,9 @@ def check_and_queue_callback_task(notification): service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) if service_callback_api: notification_data = create_delivery_status_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) + send_delivery_status_to_service.apply_async( + [str(notification.id), notification_data], queue=QueueNames.CALLBACKS + ) def _check_and_queue_complaint_callback_task(complaint, notification, recipient): @@ -228,4 +241,3 @@ def _check_and_queue_complaint_callback_task(complaint, notification, recipient) if service_callback_api: complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) - \ No newline at end of file diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index 4da60d477..8bdf84301 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -1,7 +1,6 @@ import uuid from datetime import datetime -import pytest from flask import current_app from notifications_utils.template import SMSMessageTemplate diff --git a/app/celery/research_mode_tasks.py b/app/celery/research_mode_tasks.py index ac99bc842..4477b3994 100644 --- a/app/celery/research_mode_tasks.py +++ b/app/celery/research_mode_tasks.py @@ -124,7 +124,7 @@ def create_fake_letter_response_file(self, reference): dvla_response_data = '{}|Sent|0|Sorted'.format(reference) # try and find a filename that hasn't been taken yet - from a random time within the last 30 seconds - for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related + for i in sorted(range(30), key=lambda _: random.random()): # nosec B311 - not security related upload_file_name = 'NOTIFY-{}-RSP.TXT'.format((now - timedelta(seconds=i)).strftime('%Y%m%d%H%M%S')) if not file_exists(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], upload_file_name): break diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py index 8867fca6e..0e81d38d0 100644 --- a/app/celery/service_callback_tasks.py +++ b/app/celery/service_callback_tasks.py @@ -114,7 +114,7 @@ def create_delivery_status_callback_data(notification, service_callback_api): "notification_client_reference": notification.client_reference, "notification_to": notification.to, "notification_status": notification.status, - "notification_provider_response": notification.provider_response, # TODO do we have a test for provider_response + "notification_provider_response": notification.provider_response, # TODO do we test for provider_response? "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT), "notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None, diff --git a/app/clients/__init__.py b/app/clients/__init__.py index 71c66bed3..4553dfc48 100644 --- a/app/clients/__init__.py +++ b/app/clients/__init__.py @@ -1,6 +1,3 @@ -from celery import current_app - - class ClientException(Exception): ''' Base Exceptions for sending notifications that fail @@ -38,7 +35,7 @@ class NotificationProviderClients(object): return self.email_clients.get(name) def get_client_by_name_and_type(self, name, notification_type): - assert notification_type in ['email', 'sms'] # nosec B101 + assert notification_type in ['email', 'sms'] # nosec B101 if notification_type == 'email': return self.get_email_client(name) diff --git a/app/clients/sms/__init__.py b/app/clients/sms/__init__.py index c1d38616e..2e7f27cdc 100644 --- a/app/clients/sms/__init__.py +++ b/app/clients/sms/__init__.py @@ -25,4 +25,4 @@ class SmsClient(Client): raise NotImplementedError("TODO Need to implement.") def get_name(self): - raise NotImplementedError("TODO Need to implement.") \ No newline at end of file + raise NotImplementedError("TODO Need to implement.") diff --git a/app/clients/sms/aws_sns.py b/app/clients/sms/aws_sns.py index 96d593ec0..4637bdfe4 100644 --- a/app/clients/sms/aws_sns.py +++ b/app/clients/sms/aws_sns.py @@ -20,7 +20,7 @@ class AwsSnsClient(SmsClient): self.current_app = current_app self.statsd_client = statsd_client self.long_code_regex = re.compile(r"^\+1\d{10}$") - + @property def name(self): return 'sns' @@ -86,4 +86,4 @@ class AwsSnsClient(SmsClient): raise ValueError("No valid numbers found for SMS delivery") def _send_with_dedicated_phone_number(self, sender): - return sender and re.match(self.long_code_regex, sender) \ No newline at end of file + return sender and re.match(self.long_code_regex, sender) diff --git a/app/cloudfoundry_config.py b/app/cloudfoundry_config.py index 3ad142bc1..509033988 100644 --- a/app/cloudfoundry_config.py +++ b/app/cloudfoundry_config.py @@ -8,16 +8,20 @@ def find_by_service_name(services, service_name): return services[i] return None + def extract_cloudfoundry_config(): vcap_services = json.loads(os.environ['VCAP_SERVICES']) # Postgres config - os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql') + os.environ['SQLALCHEMY_DATABASE_URI'] = \ + vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres', 'postgresql') # Redis config - os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://') + os.environ['REDIS_URL'] = \ + vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://', 'rediss://') # CSV Upload Bucket Name - bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") + bucket_service = \ + find_by_service_name(vcap_services['s3'], f"notifications-api-csv-upload-bucket-{os.environ['DEPLOY_ENV']}") if bucket_service: os.environ['CSV_UPLOAD_BUCKET_NAME'] = bucket_service['credentials']['bucket'] os.environ['CSV_UPLOAD_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] @@ -25,7 +29,8 @@ def extract_cloudfoundry_config(): os.environ['CSV_UPLOAD_REGION'] = bucket_service['credentials']['region'] # Contact List Bucket Name - bucket_service = find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") + bucket_service = \ + find_by_service_name(vcap_services['s3'], f"notifications-api-contact-list-bucket-{os.environ['DEPLOY_ENV']}") if bucket_service: os.environ['CONTACT_LIST_BUCKET_NAME'] = bucket_service['credentials']['bucket'] os.environ['CONTACT_LIST_ACCESS_KEY'] = bucket_service['credentials']['access_key_id'] diff --git a/app/commands.py b/app/commands.py index 0515a755f..92c49eefd 100644 --- a/app/commands.py +++ b/app/commands.py @@ -142,86 +142,6 @@ def purge_functional_test_data(user_email_prefix): delete_model_user(usr) -@notify_command() -def backfill_notification_statuses(): - """ - DEPRECATED. Populates notification_status. - - This will be used to populate the new `Notification._status_fkey` with the old - `Notification._status_enum` - """ - LIMIT = 250000 - subq = "SELECT id FROM notification_history WHERE notification_status is NULL LIMIT {}".format(LIMIT) # nosec B608 no user-controlled input - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input - result = db.session.execute(subq).fetchall() - - while len(result) > 0: - db.session.execute(update) - print('commit {} updates at {}'.format(LIMIT, datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() - - -@notify_command() -def update_notification_international_flag(): - """ - DEPRECATED. Set notifications.international=false. - """ - # 250,000 rows takes 30 seconds to update. - subq = "select id from notifications where international is null limit 250000" - update = "update notifications set international = False where id in ({})".format(subq) # nosec B608 no user-controlled input - result = db.session.execute(subq).fetchall() - - while len(result) > 0: - db.session.execute(update) - print('commit 250000 updates at {}'.format(datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() - - # Now update notification_history - subq_history = "select id from notification_history where international is null limit 250000" - update_history = "update notification_history set international = False where id in ({})".format(subq_history) # nosec B608 no user-controlled input - result_history = db.session.execute(subq_history).fetchall() - while len(result_history) > 0: - db.session.execute(update_history) - print('commit 250000 updates at {}'.format(datetime.utcnow())) - db.session.commit() - result_history = db.session.execute(subq_history).fetchall() - - -@notify_command() -def fix_notification_statuses_not_in_sync(): - """ - DEPRECATED. - This will be used to correct an issue where Notification._status_enum and NotificationHistory._status_fkey - became out of sync. See 979e90a. - - Notification._status_enum is the source of truth so NotificationHistory._status_fkey will be updated with - these values. - """ - MAX = 10000 - - subq = "SELECT id FROM notifications WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 no user-controlled input - update = "UPDATE notifications SET notification_status = status WHERE id in ({})".format(subq) # nosec B608 no user-controlled input - result = db.session.execute(subq).fetchall() - - while len(result) > 0: - db.session.execute(update) - print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq).fetchall() - - subq_hist = "SELECT id FROM notification_history WHERE cast (status as text) != notification_status LIMIT {}".format(MAX) # nosec B608 - update = "UPDATE notification_history SET notification_status = status WHERE id in ({})".format(subq_hist) # nosec B608 no user-controlled input - result = db.session.execute(subq_hist).fetchall() - - while len(result) > 0: - db.session.execute(update) - print('Committed {} updates at {}'.format(len(result), datetime.utcnow())) - db.session.commit() - result = db.session.execute(subq_hist).fetchall() - - @notify_command(name='insert-inbound-numbers') @click.option('-f', '--file_name', required=True, help="""Full path of the file to upload, file is a contains inbound numbers, diff --git a/app/config.py b/app/config.py index 1564bc467..e5bf044ce 100644 --- a/app/config.py +++ b/app/config.py @@ -89,11 +89,11 @@ class Config(object): # secrets that internal apps, such as the admin app or document download, must use to authenticate with the API ADMIN_CLIENT_ID = 'notify-admin' - GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove? + GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts' # TODO: can remove? INTERNAL_CLIENT_API_KEYS = json.loads( os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}') - ) # TODO: handled by varsfile? + ) # TODO: handled by varsfile? # encyption secret/salt ADMIN_CLIENT_SECRET = os.environ.get('ADMIN_CLIENT_SECRET') @@ -113,13 +113,13 @@ class Config(object): # Firetext API Key FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder") FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder") - + # Whether to ignore POSTs from SNS for replies to SMS we sent RECEIVE_INBOUND_SMS = False # Use notify.sandbox.10x sending domain unless overwritten by environment NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov' - + # AWS SNS topics for delivery receipts VALIDATE_SNS_TOPICS = True VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound'] @@ -165,7 +165,7 @@ class Config(object): MAX_VERIFY_CODE_COUNT = 5 MAX_FAILED_LOGIN_COUNT = 10 - SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service + SES_STUB_URL = None # TODO: set to a URL in env and remove this to use a stubbed SES service # be careful increasing this size without being sure that we won't see slowness in pysftp MAX_LETTER_PDF_ZIP_FILESIZE = 40 * 1024 * 1024 # 40mb @@ -186,7 +186,7 @@ class Config(object): SMS_CODE_TEMPLATE_ID = '36fb0730-6259-4da1-8a80-c8de22ad4246' EMAIL_2FA_TEMPLATE_ID = '299726d2-dba6-42b8-8209-30e1d66ea164' NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID = 'ece42649-22a8-4d06-b87f-d52d5d3f0a27' - PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password + PASSWORD_RESET_TEMPLATE_ID = '474e9242-823b-4f99-813d-ed392e7f1201' # nosec B105 - this is not a password ALREADY_REGISTERED_EMAIL_TEMPLATE_ID = '0880fbb1-a0c6-46f0-9a8e-36c986381ceb' CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884' SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde' @@ -437,7 +437,7 @@ class Development(Config): # Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key'] # } - SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development + SECRET_KEY = 'dev-notify-secret-key' # nosec B105 - this is only used in development DANGEROUS_SALT = 'dev-notify-salt' MMG_INBOUND_SMS_AUTH = ['testkey'] @@ -448,7 +448,10 @@ class Development(Config): NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov') - SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api') + SQLALCHEMY_DATABASE_URI = os.environ.get( + 'SQLALCHEMY_DATABASE_URI', + 'postgresql://postgres:chummy@db:5432/notification_api' + ) ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1' @@ -486,7 +489,10 @@ class Test(Development): # LETTER_SANITISE_BUCKET_NAME = 'test-letters-sanitise' # this is overriden in CI - SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_TEST_URI', 'postgresql://postgres:chummy@db:5432/test_notification_api') + SQLALCHEMY_DATABASE_URI = os.getenv( + 'SQLALCHEMY_DATABASE_TEST_URI', + 'postgresql://postgres:chummy@db:5432/test_notification_api' + ) CELERY = { **Config.CELERY, @@ -546,11 +552,17 @@ class Staging(Config): class Live(Config): NOTIFY_ENVIRONMENT = 'live' # buckets - CSV_UPLOAD_BUCKET_NAME = os.environ.get('CSV_UPLOAD_BUCKET_NAME', 'notifications-prototype-csv-upload') # created in gsa sandbox + CSV_UPLOAD_BUCKET_NAME = os.environ.get( + 'CSV_UPLOAD_BUCKET_NAME', + 'notifications-prototype-csv-upload' + ) # created in gsa sandbox CSV_UPLOAD_ACCESS_KEY = os.environ.get('CSV_UPLOAD_ACCESS_KEY') CSV_UPLOAD_SECRET_KEY = os.environ.get('CSV_UPLOAD_SECRET_KEY') CSV_UPLOAD_REGION = os.environ.get('CSV_UPLOAD_REGION') - CONTACT_LIST_BUCKET_NAME = os.environ.get('CONTACT_LIST_BUCKET_NAME', 'notifications-prototype-contact-list-upload') # created in gsa sandbox + CONTACT_LIST_BUCKET_NAME = os.environ.get( + 'CONTACT_LIST_BUCKET_NAME', + 'notifications-prototype-contact-list-upload' + ) # created in gsa sandbox CONTACT_LIST_ACCESS_KEY = os.environ.get('CONTACT_LIST_ACCESS_KEY') CONTACT_LIST_SECRET_KEY = os.environ.get('CONTACT_LIST_SECRET_KEY') CONTACT_LIST_REGION = os.environ.get('CONTACT_LIST_REGION') diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index efdbfaefb..c62b2908f 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -87,6 +87,7 @@ def country_records_delivery(phone_prefix): dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr'] return dlr and dlr.lower() == 'yes' + def _decide_permanent_temporary_failure(current_status, status): # If we go from pending to delivered we need to set failure type as temporary-failure if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE: @@ -102,6 +103,7 @@ def _update_notification_status(notification, status, provider_response=None): dao_update_notification(notification) return notification + @autocommit def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None): notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first() diff --git a/app/delivery/send_to_providers.py b/app/delivery/send_to_providers.py index e3e8aa6b4..c1532e83e 100644 --- a/app/delivery/send_to_providers.py +++ b/app/delivery/send_to_providers.py @@ -166,7 +166,7 @@ def update_notification_to_sending(notification, provider): # We currently have no callback method for SMS deliveries # TODO create celery task to request SMS delivery receipts from cloudwatch api notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING - + dao_update_notification(notification) @@ -175,10 +175,12 @@ provider_cache = TTLCache(maxsize=8, ttl=10) @cached(cache=provider_cache) def provider_to_use(notification_type, international=True): - international = False # TODO: remove or resolve the functionality of this flag + international = False # TODO: remove or resolve the functionality of this flag # TODO rip firetext and mmg out of early migrations and clean up the expression below active_providers = [ - p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg'] + p for p in get_provider_details_by_notification_type( + notification_type, international + ) if p.active and p.identifier not in ['firetext', 'mmg'] ] if not active_providers: @@ -191,7 +193,7 @@ def provider_to_use(notification_type, international=True): chosen_provider = active_providers[0] else: weights = [p.priority for p in active_providers] - chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - this is not security/cryptography related + chosen_provider = random.choices(active_providers, weights=weights)[0] # nosec B311 - not sec/crypto related return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type) diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py index 9afc90425..893bfc174 100644 --- a/app/inbound_sms/rest.py +++ b/app/inbound_sms/rest.py @@ -1,5 +1,5 @@ from flask import Blueprint, jsonify, request -from notifications_utils.recipients import try_validate_and_format_phone_number +# from notifications_utils.recipients import try_validate_and_format_phone_number from app.dao.inbound_sms_dao import ( dao_count_inbound_sms_for_service, diff --git a/app/models.py b/app/models.py index ed37aa9cc..79b65ca52 100644 --- a/app/models.py +++ b/app/models.py @@ -122,7 +122,9 @@ class User(db.Model): state = db.Column(db.String, nullable=False, default='pending') platform_admin = db.Column(db.Boolean, nullable=False, default=False) current_session_id = db.Column(UUID(as_uuid=True), nullable=True) - auth_type = db.Column(db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE) + auth_type = db.Column( + db.String, db.ForeignKey('auth_type.name'), index=True, nullable=False, default=EMAIL_AUTH_TYPE + ) email_access_validated_at = db.Column( db.DateTime, index=False, unique=False, nullable=False, default=datetime.datetime.utcnow ) @@ -608,7 +610,7 @@ class AnnualBilling(db.Model): "name": self.service.name } - return{ + return { "id": str(self.id), 'free_sms_fragment_limit': self.free_sms_fragment_limit, 'service_id': self.service_id, @@ -1645,7 +1647,7 @@ class Notification(db.Model): """ # this should only ever be called for letter notifications - it makes no sense otherwise and I'd rather not # get the two code flows mixed up at all - assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code already validates the correct type + assert self.notification_type == LETTER_TYPE # nosec B101 - current calling code validates correct type if self.status in [NOTIFICATION_CREATED, NOTIFICATION_SENDING]: return NOTIFICATION_STATUS_LETTER_ACCEPTED diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index 93f14b37a..bee2c9561 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -10,6 +10,7 @@ from app.notifications.sns_handlers import sns_notification_handler ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) DEFAULT_MAX_AGE = timedelta(days=10000) + # 400 counts as a permanent failure so SNS will not retry. # 500 counts as a failed delivery attempt so SNS will retry. # See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies @@ -21,7 +22,7 @@ def email_ses_callback_handler(): return jsonify( result="error", message=str(e.message) ), e.status_code - + message = data.get("Message") if "mail" in message: process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py index 50c345f49..b585e3a20 100644 --- a/app/notifications/notifications_sms_callback.py +++ b/app/notifications/notifications_sms_callback.py @@ -1,10 +1,10 @@ -from flask import Blueprint, json, jsonify, request +from flask import Blueprint # , json, jsonify, request # from app.celery.process_sms_client_response_tasks import ( # process_sms_client_response, # ) -from app.config import QueueNames -from app.errors import InvalidRequest, register_errors +# from app.config import QueueNames +from app.errors import register_errors sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms") register_errors(sms_callback_blueprint) diff --git a/app/notifications/process_notifications.py b/app/notifications/process_notifications.py index 673e7ff49..1c8d090e0 100644 --- a/app/notifications/process_notifications.py +++ b/app/notifications/process_notifications.py @@ -106,13 +106,13 @@ def persist_notification( updated_at=None ): current_app.logger.info('Presisting notification') - + notification_created_at = created_at or datetime.utcnow() if not notification_id: notification_id = uuid.uuid4() - + current_app.logger.info('Presisting notification with id {}'.format(notification_id)) - + notification = Notification( id=notification_id, template_id=template_id, @@ -135,7 +135,7 @@ def persist_notification( document_download_count=document_download_count, updated_at=updated_at ) - + current_app.logger.info('Presisting notification with to address: {}'.format(notification.to)) if notification_type == SMS_TYPE: diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index b1099d00b..8b93b935e 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -24,6 +24,7 @@ INBOUND_SMS_COUNTER = Counter( ['provider'] ) + @receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST']) def receive_sns_sms(): """ @@ -37,13 +38,13 @@ def receive_sns_sms(): "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" } """ - + # Whether or not to ignore inbound SMS replies if not current_app.config['RECEIVE_INBOUND_SMS']: return jsonify( result="success", message="SMS-SNS callback succeeded" ), 200 - + try: post_data = sns_notification_handler(request.data, request.headers) except Exception as e: @@ -53,13 +54,16 @@ def receive_sns_sms(): # TODO wrap this up if "inboundMessageId" in message: # TODO use standard formatting we use for all US numbers - inbound_number = message['destinationNumber'].replace('+','') + inbound_number = message['destinationNumber'].replace('+', '') service = fetch_potential_service(inbound_number, 'sns') if not service: # since this is an issue with our service <-> number mapping, or no inbound_sms service permission # we should still tell SNS that we received it successfully - current_app.logger.warning(f"Mapping between service and inbound number: {inbound_number} is broken, or service does not have permission to receive inbound sms") + current_app.logger.warning( + f"Mapping between service and inbound number: {inbound_number} is broken, \ + or service does not have permission to receive inbound sms" + ) return jsonify( result="success", message="SMS-SNS callback succeeded" ), 200 @@ -79,7 +83,6 @@ def receive_sns_sms(): date_received=date_received, provider_name=provider_name) - # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users, then uncomment the task below tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) current_app.logger.debug( diff --git a/app/notifications/sns_cert_validator.py b/app/notifications/sns_cert_validator.py index 57c0396ca..1fbddd811 100644 --- a/app/notifications/sns_cert_validator.py +++ b/app/notifications/sns_cert_validator.py @@ -19,6 +19,7 @@ _cert_url_re = re.compile( r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com', ) + class ValidationError(Exception): """ ValidationError. Raised when a message fails integrity checks. @@ -56,7 +57,7 @@ def get_string_to_sign(sns_payload): for field in fields: field_value = sns_payload.get(field) if not isinstance(field_value, str): - if field == 'Subject' and field_value == None: + if field == 'Subject' and field_value is None: continue raise ValidationError(f"In {field}, found non-string value: {field_value}") string_to_sign += field + '\n' + field_value + '\n' @@ -77,25 +78,26 @@ def validate_sns_cert(sns_payload): # Amazon SNS currently supports signature version 1. if sns_payload.get('SignatureVersion') != '1': raise ValidationError("Wrong Signature Version (expected 1)") - + validate_arn(sns_payload) - + string_to_sign = get_string_to_sign(sns_payload) # Key signing cert url via Lambda and via webhook are slightly different - signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL') + signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in \ + sns_payload else sns_payload.get('SigningCertURL') if not isinstance(signing_cert_url, str): raise ValidationError("Signing cert url must be a string") cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url) if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc): raise ValidationError("Cert does not appear to be from AWS") - + certificate = _signing_cert_cache.get(signing_cert_url) if certificate is None: certificate = get_certificate(signing_cert_url) if isinstance(certificate, six.text_type): certificate = certificate.encode() - + signature = base64.b64decode(sns_payload["Signature"]) try: @@ -107,4 +109,4 @@ def validate_sns_cert(sns_payload): ) return True except oscrypto.errors.SignatureError: - raise ValidationError("Invalid signature") \ No newline at end of file + raise ValidationError("Invalid signature") diff --git a/app/notifications/sns_handlers.py b/app/notifications/sns_handlers.py index a0a6d3b98..dcb6a7e3f 100644 --- a/app/notifications/sns_handlers.py +++ b/app/notifications/sns_handlers.py @@ -45,7 +45,9 @@ def sns_notification_handler(data, headers): try: validate_sns_cert(message) except Exception as e: - current_app.logger.error(f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}") + current_app.logger.error( + f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}" + ) raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': @@ -55,12 +57,18 @@ def sns_notification_handler(data, headers): try: response.raise_for_status() except Exception as e: - current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}") - raise InvalidRequest("SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation Type message failed", 400) + current_app.logger.warning( + f"Attempt to raise_for_status()SubscriptionConfirmation Type \ + message files for response: {response.text} with error {e}" + ) + raise InvalidRequest( + "SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation \ + Type message failed", 400 + ) current_app.logger.info("SES-SNS auto-confirm subscription callback succeeded") return message # TODO remove after smoke testing on prod is implemented current_app.logger.info(f"SNS message: {message} is a valid message. Attempting to process it now.") - + return message diff --git a/app/user/rest.py b/app/user/rest.py index b0832e6a8..3ec2adb9b 100644 --- a/app/user/rest.py +++ b/app/user/rest.py @@ -380,7 +380,7 @@ def send_new_user_email_verification(user_id): template = dao_get_template_by_id(current_app.config['NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID']) service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) - + current_app.logger.info('template.id is {}'.format(template.id)) current_app.logger.info('service.id is {}'.format(service.id)) @@ -404,7 +404,7 @@ def send_new_user_email_verification(user_id): current_app.logger.info('Sending notification to queue') send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) - + current_app.logger.info('Sent notification to queue') return jsonify({}), 204 @@ -414,12 +414,12 @@ def send_new_user_email_verification(user_id): def send_already_registered_email(user_id): current_app.logger.info('Email already registered for user {}'.format(user_id)) to = email_data_request_schema.load(request.get_json()) - + current_app.logger.info('To email is {}'.format(to['email'])) template = dao_get_template_by_id(current_app.config['ALREADY_REGISTERED_EMAIL_TEMPLATE_ID']) service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) - + current_app.logger.info('template.id is {}'.format(template.id)) current_app.logger.info('service.id is {}'.format(service.id)) @@ -438,11 +438,11 @@ def send_already_registered_email(user_id): key_type=KEY_TYPE_NORMAL, reply_to_text=service.get_default_reply_to_email_address() ) - + current_app.logger.info('Sending notification to queue') send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) - + current_app.logger.info('Sent notification to queue') return jsonify({}), 204 diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 0cc82cddd..896ddc079 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -87,7 +87,7 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker): def test_notifications_ses_200_call_process_task(client, mocker): process_mock = mocker.patch("app.notifications.notifications_ses_callback.process_ses_results.apply_async") mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True) - data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"} } + data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"}} mocker.patch("app.notifications.sns_handlers.sns_notification_handler", return_value=data) json_data = json.dumps(data) response = client.post( @@ -156,7 +156,10 @@ def test_ses_callback_should_update_notification_status( status='sending', sent_at=datetime.utcnow() ) - callback_api = create_service_callback_api(service=sample_email_template.service, url="https://original_url.com") + callback_api = create_service_callback_api( + service=sample_email_template.service, + url="https://original_url.com" + ) assert get_notification_by_id(notification.id).status == 'sending' assert process_ses_results(ses_notification_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'delivered' @@ -186,13 +189,19 @@ def test_ses_callback_should_retry_if_notification_is_new(mocker): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_logger.call_count == 0 assert mock_retry.call_count == 1 + + def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning') with freeze_time('2017-11-17T12:34:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 - mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)') + mock_logger.assert_called_once_with( + 'notification not found for reference: ref (while attempting update to delivered)' + ) + + def test_ses_callback_should_not_retry_if_notification_is_old(mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') @@ -200,6 +209,8 @@ def test_ses_callback_should_not_retry_if_notification_is_old(mocker): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_logger.call_count == 0 assert mock_retry.call_count == 0 + + def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry( client, _notify_db, @@ -222,6 +233,8 @@ def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry( assert process_ses_results(ses_notification_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'delivered' send_mock.assert_not_called() + + def test_ses_callback_should_update_multiple_notification_status_sent( client, _notify_db, @@ -257,6 +270,8 @@ def test_ses_callback_should_update_multiple_notification_status_sent( assert process_ses_results(ses_notification_callback(reference='ref2')) assert process_ses_results(ses_notification_callback(reference='ref3')) assert send_mock.called + + def test_ses_callback_should_set_status_to_temporary_failure(client, _notify_db, notify_db_session, @@ -278,6 +293,8 @@ def test_ses_callback_should_set_status_to_temporary_failure(client, assert process_ses_results(ses_soft_bounce_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'temporary-failure' assert send_mock.called + + def test_ses_callback_should_set_status_to_permanent_failure(client, _notify_db, notify_db_session, @@ -299,6 +316,8 @@ def test_ses_callback_should_set_status_to_permanent_failure(client, assert process_ses_results(ses_hard_bounce_callback(reference='ref')) assert get_notification_by_id(notification.id).status == 'permanent-failure' assert send_mock.called + + def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker): send_mock = mocker.patch( 'app.celery.service_callback_tasks.send_complaint_to_service.apply_async' @@ -321,4 +340,3 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email 'service_callback_api_url': 'https://original_url.com', 'to': 'recipient1@example.com' } - \ No newline at end of file diff --git a/tests/app/celery/test_process_sms_client_response_tasks.py b/tests/app/celery/test_process_sms_client_response_tasks.py index 3d08811b9..34bd6807c 100644 --- a/tests/app/celery/test_process_sms_client_response_tasks.py +++ b/tests/app/celery/test_process_sms_client_response_tasks.py @@ -5,9 +5,9 @@ import pytest from freezegun import freeze_time from app import statsd_client -# from app.celery.process_sms_client_response_tasks import ( -# process_sms_client_response, -# ) +from app.celery.process_sms_client_response_tasks import ( + process_sms_client_response, +) from app.clients import ClientException from app.models import NOTIFICATION_TECHNICAL_FAILURE diff --git a/tests/app/celery/test_provider_tasks.py b/tests/app/celery/test_provider_tasks.py index 9cecfbb29..2f241bc24 100644 --- a/tests/app/celery/test_provider_tasks.py +++ b/tests/app/celery/test_provider_tasks.py @@ -126,6 +126,7 @@ def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_ta app.delivery.send_to_providers.send_email_to_provider.assert_not_called() app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks") + @pytest.mark.skip(reason="Needs updating for TTS: Failing for unknown reason") @pytest.mark.parametrize( 'exception_class', [ diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index e4f0db279..056621820 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -290,6 +290,7 @@ def test_create_nightly_billing_for_day_different_sent_by( assert record.billable_units == 1 assert record.rate_multiplier == 1.0 + @pytest.mark.skip(reason="Needs updating for TTS: Remove mail") def test_create_nightly_billing_for_day_different_letter_postage( notify_db_session, diff --git a/tests/app/clients/test_aws_sns.py b/tests/app/clients/test_aws_sns.py index 0a61ffca0..7215c64b8 100644 --- a/tests/app/clients/test_aws_sns.py +++ b/tests/app/clients/test_aws_sns.py @@ -24,4 +24,4 @@ def test_send_sms_returns_raises_error_if_there_is_no_valid_number_is_found(noti content = reference = 'foo' with pytest.raises(ValueError) as excinfo: aws_sns_client.send_sms(to, content, reference) - assert 'No valid numbers found for SMS delivery' in str(excinfo.value) \ No newline at end of file + assert 'No valid numbers found for SMS delivery' in str(excinfo.value) diff --git a/tests/app/clients/test_sms.py b/tests/app/clients/test_sms.py index c457e365b..59d053845 100644 --- a/tests/app/clients/test_sms.py +++ b/tests/app/clients/test_sms.py @@ -15,6 +15,7 @@ def fake_client(notify_api): fake_client.init_app(notify_api, statsd_client) return fake_client + @pytest.mark.skip(reason="Needs updating for TTS: New SMS client") def test_send_sms(fake_client, mocker): mock_send = mocker.patch.object(fake_client, 'try_send_sms') @@ -31,6 +32,7 @@ def test_send_sms(fake_client, mocker): 'to', 'content', 'reference', False, 'testing' ) + @pytest.mark.skip(reason="Needs updating for TTS: New SMS client") def test_send_sms_error(fake_client, mocker): mocker.patch.object( diff --git a/tests/app/dao/test_provider_details_dao.py b/tests/app/dao/test_provider_details_dao.py index 727b520ec..939f8b9d8 100644 --- a/tests/app/dao/test_provider_details_dao.py +++ b/tests/app/dao/test_provider_details_dao.py @@ -144,6 +144,7 @@ def test_adjust_provider_priority_sets_priority( assert mmg_provider.created_by.id == notify_user.id assert mmg_provider.priority == 50 + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") @freeze_time('2016-01-01 00:30') def test_adjust_provider_priority_adds_history( @@ -172,6 +173,7 @@ def test_adjust_provider_priority_adds_history( assert updated_provider_history_rows[0].version - old_provider_history_rows[0].version == 1 assert updated_provider_history_rows[0].priority == 50 + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") @freeze_time('2016-01-01 01:00') def test_get_sms_providers_for_update_returns_providers(restore_provider_details): diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index a9c2a3568..316e24e84 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -43,6 +43,7 @@ def setup_function(_function): # state of the cache is not shared between tests. send_to_providers.provider_cache.clear() + @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") def test_provider_to_use_should_return_random_provider(mocker, notify_db_session): mmg = get_provider_details_by_identifier('mmg') @@ -72,6 +73,7 @@ def test_provider_to_use_should_cache_repeated_calls(mocker, notify_db_session): assert all(result == results[0] for result in results) assert len(mock_choices.call_args_list) == 1 + @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") @pytest.mark.parametrize('international_provider_priority', ( # Since there’s only one international provider it should always @@ -592,6 +594,7 @@ def test_should_not_update_notification_if_research_mode_on_exception( assert persisted_notification.billable_units == 0 assert update_mock.called + @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") @pytest.mark.parametrize("starting_status, expected_status", [ ("delivered", "delivered"), diff --git a/tests/app/letters/test_letter_utils.py b/tests/app/letters/test_letter_utils.py index f4f178950..deb3639f3 100644 --- a/tests/app/letters/test_letter_utils.py +++ b/tests/app/letters/test_letter_utils.py @@ -31,8 +31,8 @@ from tests.app.db import create_notification FROZEN_DATE_TIME = "2018-03-14 17:00:00" -pytest.skip(reason="Skipping letter-related functionality for now", allow_module_level=True) +@pytest.skip(reason="Skipping letter-related functionality for now", allow_module_level=True) @pytest.fixture(name='sample_precompiled_letter_notification') def _sample_precompiled_letter_notification(sample_letter_notification): sample_letter_notification.template.hidden = True diff --git a/tests/app/notifications/test_notifications_sms_callbacks.py b/tests/app/notifications/test_notifications_sms_callbacks.py index 3b35fe361..523e7e584 100644 --- a/tests/app/notifications/test_notifications_sms_callbacks.py +++ b/tests/app/notifications/test_notifications_sms_callbacks.py @@ -17,6 +17,7 @@ def mmg_post(client, data): data=data, headers=[('Content-Type', 'application/json')]) + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_not_need_auth(client, mocker): mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response') @@ -36,6 +37,7 @@ def test_firetext_callback_should_return_400_if_empty_reference(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: reference missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_400_if_no_reference(client, mocker): data = 'mobile=441234123123&status=0&time=2016-03-10 14:17:00' @@ -45,6 +47,7 @@ def test_firetext_callback_should_return_400_if_no_reference(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: reference missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_400_if_no_status(client, mocker): data = 'mobile=441234123123&time=2016-03-10 14:17:00&reference=notification_id' @@ -54,6 +57,7 @@ def test_firetext_callback_should_return_400_if_no_status(client, mocker): assert json_resp['result'] == 'error' assert json_resp['message'] == ['Firetext callback failed: status missing'] + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( @@ -70,6 +74,7 @@ def test_firetext_callback_should_return_200_and_call_task_with_valid_data(clien queue='sms-callbacks', ) + @pytest.mark.skip(reason="Needs updating for TTS: Firetext removal") def test_firetext_callback_including_a_code_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( @@ -86,6 +91,7 @@ def test_firetext_callback_including_a_code_should_return_200_and_call_task_with queue='sms-callbacks', ) + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification): mocker.patch('app.notifications.notifications_sms_callback.process_sms_client_response') @@ -98,6 +104,7 @@ def test_mmg_callback_should_not_need_auth(client, mocker, sample_notification): response = mmg_post(client, data) assert response.status_code == 200 + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_process_mmg_response_returns_400_for_malformed_data(client): data = json.dumps({"reference": "mmg_reference", @@ -114,6 +121,7 @@ def test_process_mmg_response_returns_400_for_malformed_data(client): assert "{} callback failed: {} missing".format('MMG', 'status') in json_data['message'] assert "{} callback failed: {} missing".format('MMG', 'CID') in json_data['message'] + @pytest.mark.skip(reason="Needs updating for TTS: MMG removal") def test_mmg_callback_should_return_200_and_call_task_with_valid_data(client, mocker): mock_celery = mocker.patch( diff --git a/tests/app/v2/notifications/test_post_notifications.py b/tests/app/v2/notifications/test_post_notifications.py index f58cb2751..e5b2ab073 100644 --- a/tests/app/v2/notifications/test_post_notifications.py +++ b/tests/app/v2/notifications/test_post_notifications.py @@ -238,6 +238,7 @@ def test_should_cache_template_lookups_in_memory(mocker, client, sample_template ] assert Notification.query.count() == 5 + @pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails") def test_should_cache_template_and_service_in_redis(mocker, client, sample_template): @@ -288,6 +289,7 @@ def test_should_cache_template_and_service_in_redis(mocker, client, sample_templ assert json.loads(templates_call[0][1]) == {'data': template_dict} assert templates_call[1]['ex'] == 604_800 + @pytest.mark.skip(reason="Needs updating for TTS: cloud.gov redis fails, local docker works, mock redis fails") def test_should_return_template_if_found_in_redis(mocker, client, sample_template): From 55adb3e03530862bd4c48cd057b27608105c4786 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 18 Oct 2022 16:05:40 +0000 Subject: [PATCH 35/65] more flake8 cleanup --- app/inbound_sms/rest.py | 4 +- tests/app/delivery/test_send_to_providers.py | 511 +++++++++---------- 2 files changed, 258 insertions(+), 257 deletions(-) diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py index 893bfc174..1bc3dec07 100644 --- a/app/inbound_sms/rest.py +++ b/app/inbound_sms/rest.py @@ -1,5 +1,4 @@ from flask import Blueprint, jsonify, request -# from notifications_utils.recipients import try_validate_and_format_phone_number from app.dao.inbound_sms_dao import ( dao_count_inbound_sms_for_service, @@ -16,6 +15,9 @@ from app.inbound_sms.inbound_sms_schemas import ( ) from app.schema_validation import validate +# from notifications_utils.recipients import try_validate_and_format_phone_number + + inbound_sms = Blueprint( 'inbound_sms', __name__, diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index 316e24e84..cf4f9676c 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -6,8 +6,6 @@ from unittest.mock import ANY import pytest from flask import current_app -from notifications_utils.recipients import validate_and_format_phone_number -from requests import HTTPError import app # from app import firetext_client, mmg_client, notification_provider_clients @@ -27,16 +25,17 @@ from app.models import ( Notification, ) from app.serialised_models import SerialisedService -from tests.app.db import ( +from tests.app.db import ( # create_service,; create_service_with_defined_sms_sender,; create_template, create_email_branding, create_notification, create_reply_to_email, - create_service, create_service_sms_sender, - create_service_with_defined_sms_sender, - create_template, ) +# from notifications_utils.recipients import validate_and_format_phone_number +# from requests import HTTPError + + def setup_function(_function): # pytest will run this function before each test. It makes sure the @@ -117,39 +116,39 @@ def test_provider_to_use_raises_if_no_active_providers(mocker, restore_provider_ send_to_providers.provider_to_use('sms', international=True) -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_send_personalised_template_to_correct_sms_provider_and_persist( - sample_sms_template_with_html, - mocker -): - db_notification = create_notification(template=sample_sms_template_with_html, - to_field="+447234123123", personalisation={"name": "Jo"}, - status='created', - reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(), - normalised_to="447234123123" - ) +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_send_personalised_template_to_correct_sms_provider_and_persist( +# sample_sms_template_with_html, +# mocker +# ): +# db_notification = create_notification(template=sample_sms_template_with_html, +# to_field="+447234123123", personalisation={"name": "Jo"}, +# status='created', +# reply_to_text=sample_sms_template_with_html.service.get_default_sms_sender(), +# normalised_to="447234123123" +# ) - mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.mmg_client.send_sms') - send_to_providers.send_sms_to_provider( - db_notification - ) +# send_to_providers.send_sms_to_provider( +# db_notification +# ) - mmg_client.send_sms.assert_called_once_with( - to="447234123123", - content="Sample service: Hello Jo\nHere is some HTML & entities", - reference=str(db_notification.id), - sender=current_app.config['FROM_NUMBER'], - international=False - ) +# mmg_client.send_sms.assert_called_once_with( +# to="447234123123", +# content="Sample service: Hello Jo\nHere is some HTML & entities", +# reference=str(db_notification.id), +# sender=current_app.config['FROM_NUMBER'], +# international=False +# ) - notification = Notification.query.filter_by(id=db_notification.id).one() +# notification = Notification.query.filter_by(id=db_notification.id).one() - assert notification.status == 'sending' - assert notification.sent_at <= datetime.utcnow() - assert notification.sent_by == 'mmg' - assert notification.billable_units == 1 - assert notification.personalisation == {"name": "Jo"} +# assert notification.status == 'sending' +# assert notification.sent_at <= datetime.utcnow() +# assert notification.sent_by == 'mmg' +# assert notification.billable_units == 1 +# assert notification.personalisation == {"name": "Jo"} @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") @@ -217,145 +216,145 @@ def test_should_not_send_sms_message_when_service_is_inactive_notification_is_in assert Notification.query.get(sample_notification.id).status == 'technical-failure' -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_send_sms_should_use_template_version_from_notification_not_latest( - sample_template, - mocker): - db_notification = create_notification(template=sample_template, to_field='+447234123123', status='created', - reply_to_text=sample_template.service.get_default_sms_sender(), - normalised_to='447234123123') +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_send_sms_should_use_template_version_from_notification_not_latest( +# sample_template, +# mocker): +# db_notification = create_notification(template=sample_template, to_field='+447234123123', status='created', +# reply_to_text=sample_template.service.get_default_sms_sender(), +# normalised_to='447234123123') - mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.mmg_client.send_sms') - version_on_notification = sample_template.version - expected_template_id = sample_template.id +# version_on_notification = sample_template.version +# expected_template_id = sample_template.id - # Change the template - from app.dao.templates_dao import ( - dao_get_template_by_id, - dao_update_template, - ) - sample_template.content = sample_template.content + " another version of the template" - dao_update_template(sample_template) - t = dao_get_template_by_id(sample_template.id) - assert t.version > version_on_notification +# # Change the template +# from app.dao.templates_dao import ( +# dao_get_template_by_id, +# dao_update_template, +# ) +# sample_template.content = sample_template.content + " another version of the template" +# dao_update_template(sample_template) +# t = dao_get_template_by_id(sample_template.id) +# assert t.version > version_on_notification - send_to_providers.send_sms_to_provider( - db_notification - ) +# send_to_providers.send_sms_to_provider( +# db_notification +# ) - mmg_client.send_sms.assert_called_once_with( - to=validate_and_format_phone_number("+447234123123"), - content="Sample service: This is a template:\nwith a newline", - reference=str(db_notification.id), - sender=current_app.config['FROM_NUMBER'], - international=False - ) +# mmg_client.send_sms.assert_called_once_with( +# to=validate_and_format_phone_number("+447234123123"), +# content="Sample service: This is a template:\nwith a newline", +# reference=str(db_notification.id), +# sender=current_app.config['FROM_NUMBER'], +# international=False +# ) - t = dao_get_template_by_id(expected_template_id) +# t = dao_get_template_by_id(expected_template_id) - persisted_notification = notifications_dao.get_notification_by_id(db_notification.id) - assert persisted_notification.to == db_notification.to - assert persisted_notification.template_id == expected_template_id - assert persisted_notification.template_version == version_on_notification - assert persisted_notification.template_version != t.version - assert persisted_notification.status == 'sending' - assert not persisted_notification.personalisation +# persisted_notification = notifications_dao.get_notification_by_id(db_notification.id) +# assert persisted_notification.to == db_notification.to +# assert persisted_notification.template_id == expected_template_id +# assert persisted_notification.template_version == version_on_notification +# assert persisted_notification.template_version != t.version +# assert persisted_notification.status == 'sending' +# assert not persisted_notification.personalisation -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -@pytest.mark.parametrize('research_mode,key_type', [ - (True, KEY_TYPE_NORMAL), - (False, KEY_TYPE_TEST) -]) -def test_should_call_send_sms_response_task_if_research_mode( - notify_db_session, sample_service, sample_notification, mocker, research_mode, key_type -): - mocker.patch('app.mmg_client.send_sms') - mocker.patch('app.delivery.send_to_providers.send_sms_response') +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# @pytest.mark.parametrize('research_mode,key_type', [ +# (True, KEY_TYPE_NORMAL), +# (False, KEY_TYPE_TEST) +# ]) +# def test_should_call_send_sms_response_task_if_research_mode( +# notify_db_session, sample_service, sample_notification, mocker, research_mode, key_type +# ): +# mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.delivery.send_to_providers.send_sms_response') - if research_mode: - sample_service.research_mode = True - notify_db_session.add(sample_service) - notify_db_session.commit() +# if research_mode: +# sample_service.research_mode = True +# notify_db_session.add(sample_service) +# notify_db_session.commit() - sample_notification.key_type = key_type +# sample_notification.key_type = key_type - send_to_providers.send_sms_to_provider( - sample_notification - ) - assert not mmg_client.send_sms.called +# send_to_providers.send_sms_to_provider( +# sample_notification +# ) +# assert not mmg_client.send_sms.called - app.delivery.send_to_providers.send_sms_response.assert_called_once_with( - 'mmg', str(sample_notification.id), sample_notification.to - ) +# app.delivery.send_to_providers.send_sms_response.assert_called_once_with( +# 'mmg', str(sample_notification.id), sample_notification.to +# ) - persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id) - assert persisted_notification.to == sample_notification.to - assert persisted_notification.template_id == sample_notification.template_id - assert persisted_notification.status == 'sending' - assert persisted_notification.sent_at <= datetime.utcnow() - assert persisted_notification.sent_by == 'mmg' - assert not persisted_notification.personalisation +# persisted_notification = notifications_dao.get_notification_by_id(sample_notification.id) +# assert persisted_notification.to == sample_notification.to +# assert persisted_notification.template_id == sample_notification.template_id +# assert persisted_notification.status == 'sending' +# assert persisted_notification.sent_at <= datetime.utcnow() +# assert persisted_notification.sent_by == 'mmg' +# assert not persisted_notification.personalisation -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_have_sending_status_if_fake_callback_function_fails(sample_notification, mocker): - mocker.patch('app.delivery.send_to_providers.send_sms_response', side_effect=HTTPError) +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_have_sending_status_if_fake_callback_function_fails(sample_notification, mocker): +# mocker.patch('app.delivery.send_to_providers.send_sms_response', side_effect=HTTPError) - sample_notification.key_type = KEY_TYPE_TEST +# sample_notification.key_type = KEY_TYPE_TEST - with pytest.raises(HTTPError): - send_to_providers.send_sms_to_provider( - sample_notification - ) - assert sample_notification.status == 'sending' - assert sample_notification.sent_by == 'mmg' +# with pytest.raises(HTTPError): +# send_to_providers.send_sms_to_provider( +# sample_notification +# ) +# assert sample_notification.status == 'sending' +# assert sample_notification.sent_by == 'mmg' -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_not_send_to_provider_when_status_is_not_created( - sample_template, - mocker -): - notification = create_notification(template=sample_template, status='sending') - mocker.patch('app.mmg_client.send_sms') - response_mock = mocker.patch('app.delivery.send_to_providers.send_sms_response') +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_not_send_to_provider_when_status_is_not_created( +# sample_template, +# mocker +# ): +# notification = create_notification(template=sample_template, status='sending') +# mocker.patch('app.mmg_client.send_sms') +# response_mock = mocker.patch('app.delivery.send_to_providers.send_sms_response') - send_to_providers.send_sms_to_provider( - notification - ) +# send_to_providers.send_sms_to_provider( +# notification +# ) - app.mmg_client.send_sms.assert_not_called() - response_mock.assert_not_called() +# app.mmg_client.send_sms.assert_not_called() +# response_mock.assert_not_called() -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_send_sms_with_downgraded_content(notify_db_session, mocker): - # Γ©, o, and u are in GSM. - # Δ«, grapes, tabs, zero width space and ellipsis are not - # Γ³ isn't in GSM, but it is in the welsh alphabet so will still be sent - msg = "a Γ© Δ« o u πŸ‡ foo\tbar\u200bbaz((misc))…" - placeholder = 'βˆ†βˆ†βˆ†abc' - gsm_message = "?Γ³dz Housing Service: a Γ© i o u ? foo barbaz???abc..." - service = create_service(service_name='ŁódΕΊ Housing Service') - template = create_template(service, content=msg) - db_notification = create_notification( - template=template, - personalisation={'misc': placeholder} - ) +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_send_sms_with_downgraded_content(notify_db_session, mocker): +# # Γ©, o, and u are in GSM. +# # Δ«, grapes, tabs, zero width space and ellipsis are not +# # Γ³ isn't in GSM, but it is in the welsh alphabet so will still be sent +# msg = "a Γ© Δ« o u πŸ‡ foo\tbar\u200bbaz((misc))…" +# placeholder = 'βˆ†βˆ†βˆ†abc' +# gsm_message = "?Γ³dz Housing Service: a Γ© i o u ? foo barbaz???abc..." +# service = create_service(service_name='ŁódΕΊ Housing Service') +# template = create_template(service, content=msg) +# db_notification = create_notification( +# template=template, +# personalisation={'misc': placeholder} +# ) - mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.mmg_client.send_sms') - send_to_providers.send_sms_to_provider(db_notification) +# send_to_providers.send_sms_to_provider(db_notification) - mmg_client.send_sms.assert_called_once_with( - to=ANY, - content=gsm_message, - reference=ANY, - sender=ANY, - international=False - ) +# mmg_client.send_sms.assert_called_once_with( +# to=ANY, +# content=gsm_message, +# reference=ANY, +# sender=ANY, +# international=False +# ) @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") @@ -595,22 +594,22 @@ def test_should_not_update_notification_if_research_mode_on_exception( assert update_mock.called -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -@pytest.mark.parametrize("starting_status, expected_status", [ - ("delivered", "delivered"), - ("created", "sending"), - ("technical-failure", "technical-failure"), -]) -def test_update_notification_to_sending_does_not_update_status_from_a_final_status( - sample_service, notify_db_session, starting_status, expected_status -): - template = create_template(sample_service) - notification = create_notification(template=template, status=starting_status) - send_to_providers.update_notification_to_sending( - notification, - notification_provider_clients.get_client_by_name_and_type("mmg", "sms") - ) - assert notification.status == expected_status +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# @pytest.mark.parametrize("starting_status, expected_status", [ +# ("delivered", "delivered"), +# ("created", "sending"), +# ("technical-failure", "technical-failure"), +# ]) +# def test_update_notification_to_sending_does_not_update_status_from_a_final_status( +# sample_service, notify_db_session, starting_status, expected_status +# ): +# template = create_template(sample_service) +# notification = create_notification(template=template, status=starting_status) +# send_to_providers.update_notification_to_sending( +# notification, +# notification_provider_clients.get_client_by_name_and_type("mmg", "sms") +# ) +# assert notification.status == expected_status def __update_notification(notification_to_update, research_mode, expected_status): @@ -668,116 +667,116 @@ def test_should_set_notification_billable_units_and_reduces_provider_priority_if mock_reduce.assert_called_once_with('mmg', time_threshold=timedelta(minutes=1)) -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_send_sms_to_international_providers( - sample_template, - sample_user, - mocker -): - mocker.patch('app.mmg_client.send_sms') - mocker.patch('app.firetext_client.send_sms') +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_send_sms_to_international_providers( +# sample_template, +# sample_user, +# mocker +# ): +# mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.firetext_client.send_sms') - # set firetext to active - get_provider_details_by_identifier('firetext').priority = 100 - get_provider_details_by_identifier('mmg').priority = 0 +# # set firetext to active +# get_provider_details_by_identifier('firetext').priority = 100 +# get_provider_details_by_identifier('mmg').priority = 0 - notification_international = create_notification( - template=sample_template, - to_field="+6011-17224412", - personalisation={"name": "Jo"}, - status='created', - international=True, - reply_to_text=sample_template.service.get_default_sms_sender(), - normalised_to='601117224412' - ) +# notification_international = create_notification( +# template=sample_template, +# to_field="+6011-17224412", +# personalisation={"name": "Jo"}, +# status='created', +# international=True, +# reply_to_text=sample_template.service.get_default_sms_sender(), +# normalised_to='601117224412' +# ) - send_to_providers.send_sms_to_provider( - notification_international - ) +# send_to_providers.send_sms_to_provider( +# notification_international +# ) - mmg_client.send_sms.assert_called_once_with( - to="601117224412", - content=ANY, - reference=str(notification_international.id), - sender=current_app.config['FROM_NUMBER'], - international=True - ) +# mmg_client.send_sms.assert_called_once_with( +# to="601117224412", +# content=ANY, +# reference=str(notification_international.id), +# sender=current_app.config['FROM_NUMBER'], +# international=True +# ) - assert notification_international.status == 'sent' - assert notification_international.sent_by == 'mmg' +# assert notification_international.status == 'sent' +# assert notification_international.sent_by == 'mmg' -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -def test_should_send_non_international_sms_to_default_provider( - sample_template, - sample_user, - mocker -): - mocker.patch('app.mmg_client.send_sms') - mocker.patch('app.firetext_client.send_sms') +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# def test_should_send_non_international_sms_to_default_provider( +# sample_template, +# sample_user, +# mocker +# ): +# mocker.patch('app.mmg_client.send_sms') +# mocker.patch('app.firetext_client.send_sms') - # set firetext to active - get_provider_details_by_identifier('firetext').priority = 100 - get_provider_details_by_identifier('mmg').priority = 0 +# # set firetext to active +# get_provider_details_by_identifier('firetext').priority = 100 +# get_provider_details_by_identifier('mmg').priority = 0 - notification_uk = create_notification( - template=sample_template, - to_field="+447234123999", - personalisation={"name": "Jo"}, - status='created', - international=False, - reply_to_text=sample_template.service.get_default_sms_sender(), - normalised_to="447234123999" - ) +# notification_uk = create_notification( +# template=sample_template, +# to_field="+447234123999", +# personalisation={"name": "Jo"}, +# status='created', +# international=False, +# reply_to_text=sample_template.service.get_default_sms_sender(), +# normalised_to="447234123999" +# ) - send_to_providers.send_sms_to_provider( - notification_uk - ) +# send_to_providers.send_sms_to_provider( +# notification_uk +# ) - firetext_client.send_sms.assert_called_once_with( - to="447234123999", - content=ANY, - reference=str(notification_uk.id), - sender=current_app.config['FROM_NUMBER'], - international=False - ) +# firetext_client.send_sms.assert_called_once_with( +# to="447234123999", +# content=ANY, +# reference=str(notification_uk.id), +# sender=current_app.config['FROM_NUMBER'], +# international=False +# ) - assert notification_uk.status == 'sending' - assert notification_uk.sent_by == 'firetext' +# assert notification_uk.status == 'sending' +# assert notification_uk.sent_by == 'firetext' -@pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") -@pytest.mark.parametrize('sms_sender, expected_sender, prefix_sms, expected_content', [ - ('foo', 'foo', False, 'bar'), - ('foo', 'foo', True, 'Sample service: bar'), - # if 40604 is actually in DB then treat that as if entered manually - ('40604', '40604', False, 'bar'), - # 'testing' is the FROM_NUMBER during unit tests - ('testing', 'testing', True, 'Sample service: bar'), - ('testing', 'testing', False, 'bar'), -]) -def test_should_handle_sms_sender_and_prefix_message( - mocker, - sms_sender, - prefix_sms, - expected_sender, - expected_content, - notify_db_session -): - mocker.patch('app.mmg_client.send_sms') - service = create_service_with_defined_sms_sender(sms_sender_value=sms_sender, prefix_sms=prefix_sms) - template = create_template(service, content='bar') - notification = create_notification(template, reply_to_text=sms_sender) +# @pytest.mark.skip(reason="Needs updating for TTS: Update with new providers") +# @pytest.mark.parametrize('sms_sender, expected_sender, prefix_sms, expected_content', [ +# ('foo', 'foo', False, 'bar'), +# ('foo', 'foo', True, 'Sample service: bar'), +# # if 40604 is actually in DB then treat that as if entered manually +# ('40604', '40604', False, 'bar'), +# # 'testing' is the FROM_NUMBER during unit tests +# ('testing', 'testing', True, 'Sample service: bar'), +# ('testing', 'testing', False, 'bar'), +# ]) +# def test_should_handle_sms_sender_and_prefix_message( +# mocker, +# sms_sender, +# prefix_sms, +# expected_sender, +# expected_content, +# notify_db_session +# ): +# mocker.patch('app.mmg_client.send_sms') +# service = create_service_with_defined_sms_sender(sms_sender_value=sms_sender, prefix_sms=prefix_sms) +# template = create_template(service, content='bar') +# notification = create_notification(template, reply_to_text=sms_sender) - send_to_providers.send_sms_to_provider(notification) +# send_to_providers.send_sms_to_provider(notification) - mmg_client.send_sms.assert_called_once_with( - content=expected_content, - sender=expected_sender, - to=ANY, - reference=ANY, - international=False - ) +# mmg_client.send_sms.assert_called_once_with( +# content=expected_content, +# sender=expected_sender, +# to=ANY, +# reference=ANY, +# international=False +# ) def test_send_email_to_provider_uses_reply_to_from_notification( From 7fb471a10c06e5278af66784f03ac9c33e76a30e Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 18 Oct 2022 20:20:09 +0000 Subject: [PATCH 36/65] test tweaks --- app/authentication/auth.py | 9 +++++---- tests/app/celery/test_process_ses_receipts_tasks.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 57842ee0c..67be729e8 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -18,10 +18,11 @@ from sqlalchemy.orm.exc import NoResultFound from app.serialised_models import SerialisedService -GENERAL_TOKEN_ERROR_MESSAGE = ''' - Invalid token: make sure your API token matches the example - at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header - ''' # nosec B105 +# stvnrlly - this is silly, but bandit has a multiline string bug (https://github.com/PyCQA/bandit/issues/658) +# and flake8 wants a multiline quote here. TODO: check on bug status and restore sanity once possible +TOKEN_MESSAGE_ONE = "Invalid token: make sure your API token matches the example " # nosec B105 +TOKEN_MESSAGE_TWO = "at https://docs.notifications.service.gov.uk/rest-api.html#authorisation-header" # nosec B105 +GENERAL_TOKEN_ERROR_MESSAGE = TOKEN_MESSAGE_ONE + TOKEN_MESSAGE_TWO AUTH_DB_CONNECTION_DURATION_SECONDS = Histogram( 'auth_db_connection_duration_seconds', diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 896ddc079..cd54a187c 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -198,7 +198,8 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 mock_logger.assert_called_once_with( - 'notification not found for reference: ref (while attempting update to delivered)' + 'notification not found for reference: ref (while \ + attempting update to delivered)' ) From 788f5e2d86e7da5d64ad4f52653116782fca77c5 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 18 Oct 2022 20:20:21 +0000 Subject: [PATCH 37/65] reactivate flake8 in checks.yml --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 57e11688e..49f256e4b 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,8 +57,8 @@ jobs: run: make bootstrap env: SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api - # - name: Run style checks - # run: flake8 . + - name: Run style checks + run: flake8 . - name: Check imports alphabetized run: isort --check-only ./app ./tests - name: Run tests From 2d947c8d334c3a0c7bb5ccd965a319cce8165e6d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 18 Oct 2022 20:26:41 +0000 Subject: [PATCH 38/65] flake8 post-isort --- tests/app/delivery/test_send_to_providers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/app/delivery/test_send_to_providers.py b/tests/app/delivery/test_send_to_providers.py index cf4f9676c..eda4edae3 100644 --- a/tests/app/delivery/test_send_to_providers.py +++ b/tests/app/delivery/test_send_to_providers.py @@ -36,7 +36,6 @@ from tests.app.db import ( # create_service,; create_service_with_defined_sms_s # from requests import HTTPError - def setup_function(_function): # pytest will run this function before each test. It makes sure the # state of the cache is not shared between tests. From 5dfc26c1f56538b0dfb1f3674f0eb3d36ed69d50 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 19 Oct 2022 13:55:59 +0000 Subject: [PATCH 39/65] pass pytest multiline preferences --- app/celery/process_ses_receipts_tasks.py | 4 ++-- tests/app/celery/test_process_ses_receipts_tasks.py | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 0b8f4bd06..232dfafaf 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -60,8 +60,8 @@ def process_ses_results(self, response): self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - "notification not found for reference: {} (while \ - attempting update to {})".format(reference, notification_status) + f"notification not found for reference: {reference} \ + (while attempting update to {notification_status})" ) return diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index cd54a187c..fcd258926 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -197,9 +197,11 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, with freeze_time('2017-11-17T12:34:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 + # the multiline indent must be the same as in the application code + # for the assertion to completely match mock_logger.assert_called_once_with( - 'notification not found for reference: ref (while \ - attempting update to delivered)' + 'notification not found for reference: ref \ + (while attempting update to delivered)' ) From 478b6215fbc761d5f2b5aa2c1b75b64157134608 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 19 Oct 2022 13:56:09 +0000 Subject: [PATCH 40/65] bump flake8 version --- requirements_for_test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_for_test.txt b/requirements_for_test.txt index 3c6756bc1..28106fd6c 100644 --- a/requirements_for_test.txt +++ b/requirements_for_test.txt @@ -1,6 +1,6 @@ --requirement requirements.txt -flake8==4.0.1 -flake8-bugbear==22.4.25 +flake8==5.0.4 +flake8-bugbear==22.9.23 isort==5.10.1 moto==3.1.9 pytest==7.1.2 From f5b5ecb661756143d82c3220cc1cdf448eba30f6 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 19 Oct 2022 16:23:34 +0000 Subject: [PATCH 41/65] flake8 fixes for rebased commits --- app/dao/services_dao.py | 7 +++---- app/organisation/rest.py | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index f8c60272f..5a1540895 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -9,8 +9,8 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class from app.dao.date_util import get_current_financial_year -from app.dao.email_branding_dao import dao_get_email_branding_by_name -from app.dao.letter_branding_dao import dao_get_letter_branding_by_name +# from app.dao.email_branding_dao import dao_get_email_branding_by_name +# from app.dao.letter_branding_dao import dao_get_letter_branding_by_name from app.dao.organisation_dao import dao_get_organisation_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -46,8 +46,7 @@ from app.models import ( User, VerifyCode, ) -from app.utils import ( - email_address_is_nhs, +from app.utils import ( # email_address_is_nhs, escape_special_characters, get_archived_db_column_value, get_london_midnight_in_utc, diff --git a/app/organisation/rest.py b/app/organisation/rest.py index a44cb8b9c..30d819131 100644 --- a/app/organisation/rest.py +++ b/app/organisation/rest.py @@ -103,8 +103,6 @@ def update_organisation(organisation_id): data = request.get_json() validate(data, post_update_organisation_schema) - organisation = dao_get_organisation_by_id(organisation_id) - result = dao_update_organisation(organisation_id, **data) if data.get('agreement_signed') is True: From 412575c81d20e4d9b741eb2925461453d0b2611b Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 19 Oct 2022 13:13:50 -0400 Subject: [PATCH 42/65] no longer using pyup --- .pyup.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .pyup.yml diff --git a/.pyup.yml b/.pyup.yml deleted file mode 100644 index 2dec6592b..000000000 --- a/.pyup.yml +++ /dev/null @@ -1,8 +0,0 @@ -# see https://pyup.io/docs/configuration/ for all available options - -schedule: "every week on wednesday" - -search: False -requirements: - - requirements.in - - requirements_for_test.txt From 9df489d03a620f3a63cef4d4647555bb511e5e83 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:03:52 -0400 Subject: [PATCH 43/65] remove unnecessary files --- dump.rdb | Bin 88 -> 0 bytes varsfile.sample | 30 ------------------------------ 2 files changed, 30 deletions(-) delete mode 100644 dump.rdb delete mode 100644 varsfile.sample diff --git a/dump.rdb b/dump.rdb deleted file mode 100644 index 6ec88245cd38ae9dd6d5f1433877fbb343911e97..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 88 zcmWG?b@2=~FfcUu#aWb^l3A=P;l-K50g)B_CvH=CNW&V0YN9{|UdBq0C* diff --git a/varsfile.sample b/varsfile.sample deleted file mode 100644 index 9cd9a537a..000000000 --- a/varsfile.sample +++ /dev/null @@ -1,30 +0,0 @@ -SECRET_KEY: "dev-notify-secret-key" # pragma: allowlist secret -DANGEROUS_SALT: "dev-notify-salt" - -ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov -ADMIN_CLIENT_ID: notify-admin -ADMIN_CLIENT_SECRET: dev-notify-secret-key -API_HOST_NAME: https://notifications-api.app.cloud.gov -AWS_PINPOINT_REGION: us-west-2 -AWS_REGION: us-west-2 -AWS_US_TOLL_FREE_NUMBER: 18446120782 -DANGEROUS_SALT: dev-notify-salt -DVLA_EMAIL_ADDRESSES: [] -FIRETEXT_API_KEY: placeholder -FIRETEXT_INBOUND_SMS_AUTH: {} -FIRETEXT_INTERNATIONAL_API_KEY: placeholder -FLASK_APP: application.py -FLASK_ENV: production -INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["dev-notify-secret-key"]}' -MMG_API_KEY: placeholder -MMG_INBOUND_SMS_AUTH: {} -MMG_INBOUND_SMS_USERNAME: {} -NOTIFICATION_QUEUE_PREFIX: prototype_10x -NOTIFY_APP_NAME: api -NOTIFY_EMAIL_DOMAIN: dispostable.com -NOTIFY_ENVIRONMENT: live -NOTIFY_LOG_PATH: /home/vcap/logs/app.log -ROUTE_SECRET_KEY_1: dev-route-secret-key-1 -ROUTE_SECRET_KEY_2: dev-route-secret-key-2 -SECRET_KEY: dev-notify-secret-key -STATSD_HOST: localhost From cb9e098c6507d4ca15f94889e4cca872516e3b77 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:04:10 -0400 Subject: [PATCH 44/65] streamline makefile --- Makefile | 92 +++++++++++++++----------------------------------------- 1 file changed, 24 insertions(+), 68 deletions(-) diff --git a/Makefile b/Makefile index 701ae3380..1de8c9589 100644 --- a/Makefile +++ b/Makefile @@ -7,8 +7,6 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -CF_API ?= api.cloud.service.gov.uk -CF_ORG ?= govuk-notify CF_SPACE ?= ${DEPLOY_ENV} CF_HOME ?= ${HOME} $(eval export CF_HOME) @@ -90,24 +88,6 @@ clean: ## DEPLOYMENT -.PHONY: preview -preview: ## Set environment to preview - $(eval export DEPLOY_ENV=preview) - $(eval export DNS_NAME="notify.works") - @true - -.PHONY: staging -staging: ## Set environment to staging - $(eval export DEPLOY_ENV=staging) - $(eval export DNS_NAME="staging-notify.works") - @true - -.PHONY: production -production: ## Set environment to production - $(eval export DEPLOY_ENV=production) - $(eval export DNS_NAME="notifications.service.gov.uk") - @true - .PHONY: cf-login cf-login: ## Log in to Cloud Foundry $(if ${CF_USERNAME},,$(error Must specify CF_USERNAME)) @@ -116,64 +96,40 @@ cf-login: ## Log in to Cloud Foundry @echo "Logging in to Cloud Foundry on ${CF_API}" @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" -.PHONY: cf-deploy -cf-deploy: ## Deploys the app to Cloud Foundry - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - $(if ${CF_APP},,$(error Must specify CF_APP)) - cf target -o ${CF_ORG} -s ${CF_SPACE} - @cf app --guid ${CF_APP} || exit 1 - - # cancel any existing deploys to ensure we can apply manifest (if a deploy is in progress you'll see ScaleDisabledDuringDeployment) - cf cancel-deployment ${CF_APP} || true - - # fails after 15 mins if deploy doesn't work - CF_STARTUP_TIMEOUT=15 cf push ${CF_APP} --strategy=rolling - -.PHONY: cf-deploy-api-db-migration -cf-deploy-api-db-migration: - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - cf target -o ${CF_ORG} -s ${CF_SPACE} - make -s CF_APP=notifications-api generate-manifest > ${CF_MANIFEST_PATH} - - cf push notifications-api --no-route -f ${CF_MANIFEST_PATH} - rm ${CF_MANIFEST_PATH} - - cf run-task notifications-api --command="flask db upgrade" --name api_db_migration - .PHONY: cf-check-api-db-migration-task cf-check-api-db-migration-task: ## Get the status for the last notifications-api task @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" -.PHONY: cf-rollback -cf-rollback: ## Rollbacks the app to the previous release - $(if ${CF_APP},,$(error Must specify CF_APP)) - rm ${CF_MANIFEST_PATH} - cf cancel-deployment ${CF_APP} +# .PHONY: cf-rollback +# cf-rollback: ## Rollbacks the app to the previous release +# $(if ${CF_APP},,$(error Must specify CF_APP)) +# rm ${CF_MANIFEST_PATH} +# cf cancel-deployment ${CF_APP} .PHONY: check-if-migrations-to-run check-if-migrations-to-run: @echo $(shell python3 scripts/check_if_new_migration.py) -.PHONY: cf-deploy-failwhale -cf-deploy-failwhale: - $(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`)) - cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml +# .PHONY: cf-deploy-failwhale +# cf-deploy-failwhale: +# $(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`)) +# cd ./paas-failwhale; cf push notify-api-failwhale -f manifest.yml -.PHONY: enable-failwhale -enable-failwhale: ## Enable the failwhale app and disable api - $(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`)) - # make sure failwhale is running first - cf start notify-api-failwhale +# .PHONY: enable-failwhale +# enable-failwhale: ## Enable the failwhale app and disable api +# $(if ${DNS_NAME},,$(error Must target space, eg `make preview enable-failwhale`)) +# # make sure failwhale is running first +# cf start notify-api-failwhale - cf map-route notify-api-failwhale ${DNS_NAME} --hostname api - cf unmap-route notify-api ${DNS_NAME} --hostname api - @echo "Failwhale is enabled" +# cf map-route notify-api-failwhale ${DNS_NAME} --hostname api +# cf unmap-route notify-api ${DNS_NAME} --hostname api +# @echo "Failwhale is enabled" -.PHONY: disable-failwhale -disable-failwhale: ## Disable the failwhale app and enable api - $(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`)) +# .PHONY: disable-failwhale +# disable-failwhale: ## Disable the failwhale app and enable api +# $(if ${DNS_NAME},,$(error Must target space, eg `make preview disable-failwhale`)) - cf map-route notify-api ${DNS_NAME} --hostname api - cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api - cf stop notify-api-failwhale - @echo "Failwhale is disabled" +# cf map-route notify-api ${DNS_NAME} --hostname api +# cf unmap-route notify-api-failwhale ${DNS_NAME} --hostname api +# cf stop notify-api-failwhale +# @echo "Failwhale is disabled" From 2ce19fd502ac49c4c0b9d2b9ee3967ced7de9b4d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:04:38 -0400 Subject: [PATCH 45/65] add env check before purging data in command --- app/commands.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/commands.py b/app/commands.py index 0515a755f..7f85718e8 100644 --- a/app/commands.py +++ b/app/commands.py @@ -113,6 +113,10 @@ def purge_functional_test_data(user_email_prefix): users, services, etc. Give an email prefix. Probably "notify-tests-preview". """ + if os.getenv('NOTIFY_ENVIRONMENT', '') not in ['development', 'test']: + current_app.logger.error('Can only be run in development') + return + users = User.query.filter(User.email_address.like("{}%".format(user_email_prefix))).all() for usr in users: # Make sure the full email includes a uuid in it From a45e02d6e5b1c8da52dc03d2ab370ed7467312f8 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:05:23 -0400 Subject: [PATCH 46/65] restructure readme & docs --- README.md | 182 ++++++++++++------------------------ docs/database-management.md | 55 +++++++++++ docs/infra-onboarding.md | 7 ++ docs/infra-setup.md | 20 ++++ docs/one-off-tasks.md | 22 +++++ docs/testing.md | 31 ++++++ 6 files changed, 193 insertions(+), 124 deletions(-) create mode 100644 docs/database-management.md create mode 100644 docs/infra-onboarding.md create mode 100644 docs/infra-setup.md create mode 100644 docs/one-off-tasks.md create mode 100644 docs/testing.md diff --git a/README.md b/README.md index f23ed0c83..cb0a95a21 100644 --- a/README.md +++ b/README.md @@ -1,153 +1,87 @@ # US Notify API -Cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers! +This project is the core of [Notify](https://notifications-admin.app.cloud.gov/). It's cloned from the brilliant work of the team at [GOV.UK Notify](https://github.com/alphagov/notifications-api), cheers! -Contains: +This repo contains: -- the public-facing REST API for US Notify, which teams can integrate with using [our clients](https://www.notifications.service.gov.uk/documentation) [DOCS ARE STILL UK] -- an internal-only REST API built using Flask to manage services, users, templates, etc (this is what the [admin app](http://github.com/18F/notifications-admin) talks to) -- asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc +- A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation) +- An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to) +- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc -## QUICKSTART ---- -If you are the first on your team to deploy, set up AWS SES/SNS as instructed in the AWS setup section below. +## Local setup -Create .env file as described in the .env section below. +### Direct installation -Install VS Code -Open VS Code and install the Remote-Containers plug-in from Microsoft. +1. Set up Postgres && Redis -Make sure your docker daemon is running (on OS X, this is typically accomplished by opening the Docker Desktop app) -Also make sure there is NOT a Postgres daemon running on port 5432. +1. Install dependencies into a virtual environment -Create the external docker network: + ``` + pipenv install --with dev + createdb notification_api + flask db upgrade + ``` -`docker network create notify-network` +1. Create the .env file -Using the command palette (shift+cmd+p), search and select β€œRemote Containers: Open Folder in Container...” -When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window (replacing the current one). + ``` + cp sample.env .env + # follow the instructions in .env + ``` -After this page loads, hit "show logs” in bottom-right. The first time this runs it will need to build the Docker image, which will likely take several minutes. +1. Run Flask -Select View->Open View..., then search/select β€œports”. Await a green dot on the port view, then open a new terminal and run the web server: -`make run-flask` + ``` + pipenv run make run-flask + ``` -Open another terminal and run the background tasks: -`make run-celery` +1. Run Celery -Confirm that everything is working by hitting localhost:6011 and it responds with a 200 OK. + ``` + pipenv run make run-celery + ``` ---- -## Setting Up -### `.env` file +### VS Code && Docker installation -Create and edit a .env file, based on sample.env. +If you're working in VS Code, you can also leverage Docker for a containerized dev environment + +1. Create .env file as described in the .env section below. + +1. Install the Remote-Containers plug-in in VS Code + +1. With Docker running, create the network: + + `docker network create notify-network` + +1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select β€œRemote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window, replacing the current one. + +1. Wait a few minutes while things happen + +1. Open a VS Code terminal and run the Flask application: + + `make run-flask` + +1. Open another VS Code terminal and run Celery: + + `make run-celery` NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container. -Things to change: +## Deeper documentation -- If you're not the first to deploy, only replace the aws creds, get these from team lead -- Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) -- Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values -- Set up AWS SES and SNS as indicated in next section (AWS Setup), fill in missing AWS env vars +### Infrastructure -### AWS Setup +- [Checklist for onboarding to all of the things](./docs/infra-onboarding.md) +- [Setting up the initial infrastructure using AWS](./docs/infra-setup.md) +- [Database management](./docs/database-management.md) -**Steps to prepare SES** +### Common dev work -1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. -2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. +- [Testing](./docs/testing.md) +- [Running one-off tasks](./docs/one-off-tasks.md) -TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures - -**Steps to prepare SNS** - -1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" -2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save -3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" -4. Under "Number Settings" choose "Request Phone Number" -5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" -6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" -7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) - -At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! πŸŽ‰ - -### Secrets Detection - -``` -brew install detect-secrets # or pip install detect-secrets -detect-secrets scan -#review output of above, make sure none of the baseline entries are sensitive -detect-secrets scan > .secrets.baseline -#creates the baseline file -``` - -Ideally, you'll install `detect-secrets` so that it's accessible from any environment from which you _might_ commit. You can use `brew install` to make it available globally. You could also install via `pip install` inside a virtual environment, if you're sure you'll _only_ commit from that environment. - -If you open .git/hooks/pre-commit you should see a simple bash script that runs the command below, reads the output and aborts before committing if detect-secrets finds a secret. You should be able to test it by staging a file with any high-entropy string like `"bblfwk3u4bt484+afw4avev5ae+afr4?/fa"` (it also has other ways to detect secrets, this is just the most straightforward to test). - -You can permit exceptions by adding an inline comment containing `pragma: allowlist secret` - -The command that is actually run by the pre-commit hook is: `git diff --staged --name-only -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline` - -You can also run against all tracked files staged or not: `git ls-files -z | xargs -0 detect-secrets-hook --baseline .secrets.baseline` - -### Postgres - -Local postgres implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml) - -### Redis - -Local redis implementation is handled by [docker compose](https://github.com/18F/notifications-api/blob/main/docker-compose.devcontainer.yml) - -## To test the application - -``` -# install dependencies, etc. -make bootstrap - -make test -``` - -## To run a local OWASP scan - -1. Run `make run-flask` from within the dev container. -2. On your host machine run: - -``` -docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf -``` - -## To run scheduled tasks - -``` -# After scheduling some tasks, open a third terminal in your running devcontainer and run celery beat -make run-celery-beat -``` - -## To run one off tasks (Ignore for Quick Start) - -Tasks are run through the `flask` command - run `flask --help` for more information. There are two sections we need to -care about: `flask db` contains alembic migration commands, and `flask command` contains all of our custom commands. For -example, to purge all dynamically generated functional test data, do the following: - -Local (from inside the devcontainer) - -``` -flask command purge_functional_test_data -u -``` - -Remote - -``` -cf run-task notify-api "flask command purge_functional_test_data -u " -``` - -All commands and command options have a --help command if you need more information. - -## Further documentation [DEPRECATED] +## UK docs that may still be helpful - [Writing public APIs](docs/writing-public-apis.md) - [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies) diff --git a/docs/database-management.md b/docs/database-management.md new file mode 100644 index 000000000..589df97fd --- /dev/null +++ b/docs/database-management.md @@ -0,0 +1,55 @@ +# Database management + +## Initial state + +In Notify, several aspects of the system are loaded into the database via migration. This means that +application setup requires loading and overwriting historical data in order to arrive at the current +configuration. + +[Here are notes](https://docs.google.com/document/d/1ZgiUtJFvRBKBxB1ehiry2Dup0Q5iIwbdCU5spuqUFTo/edit#) +about what is loaded into which tables, and some plans for how we might manage that in the future. + +Flask does not seem to have a great way to squash migrations, but rather wants you to recreate them +from the DB structure. This means it's easy to recreate the tables, but hard to recreate the initial data. + +## Migrations + +Create a migration: + +``` +flask db migrate +``` + +Trim any auto-generated stuff down to what you want, and manually rename it to be in numerical order. +We should only have one migration branch. + +Running migrations locally: + +``` +flask db upgrade +``` + +This should happen automatically on cloud.gov, but if you need to run a one-off migration for some reason: + +``` +cf run-task notifications-api-staging --commmand "flask db upgrade" --name db-upgrade +``` + +## Purging user data + +There is a Flask command to wipe user-created data (users, services, etc.). + +The command should stop itself if it's run in a production environment, but, you know, please don't run it +in a production environment. + +Running locally: + +``` +flask command purge_functional_test_data -u +``` + +Running on cloud.gov: + +``` +cf run-task notify-api "flask command purge_functional_test_data -u " +``` diff --git a/docs/infra-onboarding.md b/docs/infra-onboarding.md new file mode 100644 index 000000000..6d7789dfe --- /dev/null +++ b/docs/infra-onboarding.md @@ -0,0 +1,7 @@ +# Infrastructure onboarding + +- [ ] Join [the GSA GitHub org](https://github.com/GSA/GitHub-Administration#join-the-gsa-organization) +- [ ] Get permissions for the repos +- [ ] Get access to the cloud.gov org && space +- [ ] Get access to AWS, if necessary +- [ ] Pull down creds from cloud.gov and create the local .env file \ No newline at end of file diff --git a/docs/infra-setup.md b/docs/infra-setup.md new file mode 100644 index 000000000..9bd83f058 --- /dev/null +++ b/docs/infra-setup.md @@ -0,0 +1,20 @@ +# Setting up the infrastructure + +## Steps to prepare SES + +1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. +2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. + +TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures + +## Steps to prepare SNS + +1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" +2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save +3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" +4. Under "Number Settings" choose "Request Phone Number" +5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" +6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" +7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) + +At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! πŸŽ‰ \ No newline at end of file diff --git a/docs/one-off-tasks.md b/docs/one-off-tasks.md new file mode 100644 index 000000000..edbfcefea --- /dev/null +++ b/docs/one-off-tasks.md @@ -0,0 +1,22 @@ +# One-off tasks + +For these, we're using Flask commands, which live in [`/app/commands.py`](../app/commands.py). + +This includes things that might be one-time operations! Using a command allows the operation to be tested, +both with `pytest` and with trial runs. + +To run a command on cloud.gov, use this format: + +``` +cf run-task CLOUD-GOV-SPACE --commmand "YOUR COMMAND HERE" --name YOUR-COMMAND +``` + +[Here's more documentation](https://docs.cloudfoundry.org/devguide/using-tasks.html) about Cloud Foundry tasks. + +## Celery scheduled tasks + +After scheduling some tasks, run celery beat to get them moving: + +``` +make run-celery-beat +``` diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..2294c52cf --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,31 @@ +# Testing + +``` +# install dependencies, etc. +make bootstrap + +make test +``` + +This will run: +- flake8 for code styling +- isort for import styling +- pytest for the test suite + +On GitHub, in addition to these tests, we run: +- bandit for code security +- pip-audit for dependency vulnerabilities +- OWASP for dynamic scanning + +## CI testing + +We're using GitHub Actions. See [/.github](../.github/) for the configuration. + +## To run a local OWASP scan + +1. Run `make run-flask` from within the dev container. +2. On your host machine run: + +``` +docker run -v $(pwd):/zap/wrk/:rw --network="notify-network" -t owasp/zap2docker-weekly zap-api-scan.py -t http://dev:6011/_status -f openapi -c zap.conf +``` \ No newline at end of file From 5f4d8ee3effc9ab82639670eb952c7fa7741f31b Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:05:54 -0400 Subject: [PATCH 47/65] put instructions directly in .env file --- sample.env | 76 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/sample.env b/sample.env index 78761beac..d8694b5c1 100644 --- a/sample.env +++ b/sample.env @@ -1,5 +1,47 @@ +# STEPS TO SET UP +# +# 1. Pull down AWS creds from cloud.gov using `cf env`, then update AWS section +# +# 2. Uncomment either the Docker setup or the direct setup +# +# 3. Comment out the other setup +# +# 4. Replace `NOTIFY_EMAIL_DOMAIN` with the domain your emails will come from (i.e. the "origination email" in your SES project) +# +# 5. Replace `SECRET_KEY` and `DANGEROUS_SALT` with high-entropy secret values +# + # ## REBUILD THE DEVCONTAINER WHEN YOU MODIFY .ENV ### +############################################################# + +# AWS +AWS_REGION=us-west-2 +AWS_ACCESS_KEY_ID="don't write secrets to the sample file" +AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" +AWS_PINPOINT_REGION=us-west-2 +AWS_US_TOLL_FREE_NUMBER=+18446120782 + +############################################################# + +# Local Docker setup, all overwritten in cloud.gov +ADMIN_BASE_URL=http://admin:6012 +API_HOST_NAME=http://dev:6011 +REDIS_URL=redis://redis:6380 +REDIS_ENABLED=1 +SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api +SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api + +# Local direct setup, all overwritten in cloud.gov +# ADMIN_BASE_URL=http://localhost:6012 +# API_HOST_NAME=http://localhost:6011 +# REDIS_URL=redis://localhost:6379 +# REDIS_ENABLED=1 +# SQLALCHEMY_DATABASE_URI=postgresql://localhost:5432/notification_api +# SQLALCHEMY_DATABASE_TEST_URI=postgresql://localhost:5432/test_notification_api + +############################################################# + # Debug DEBUG=True ANTIVIRUS_ENABLED=0 @@ -10,10 +52,7 @@ NOTIFY_APP_NAME=api NOTIFY_EMAIL_DOMAIN=dispostable.com NOTIFY_LOG_PATH=/workspace/logs/app.log -# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API -ADMIN_CLIENT_ID=notify-admin -ADMIN_CLIENT_SECRET=dev-notify-secret-key -GOVUK_ALERTS_CLIENT_ID=govuk-alerts +############################################################# # Flask FLASK_APP=application.py @@ -22,28 +61,7 @@ WERKZEUG_DEBUG_PIN=off SECRET_KEY=dev-notify-secret-key DANGEROUS_SALT=dev-notify-salt -# URL of admin app, this is overriden on cloudfoundry -ADMIN_BASE_URL=http://admin:6012 - -# URL of api app, this is overriden on cloudfoundry -API_HOST_NAME=http://dev:6011 - -# URL of redis instance, this is overriden on cloudfoundry -REDIS_URL=redis://redis:6380 -REDIS_ENABLED=1 - -# DB connection string for local docker, overriden on remote with vcap env vars -SQLALCHEMY_DATABASE_URI=postgresql://postgres:chummy@db:5432/notification_api - -# For testing in local docker -SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notification_api - -# DB connection string for local non-docker connection -# SQLALCHEMY_DATABASE_URI=postgresql://user:password@localhost:5432/notification_api - -# AWS -AWS_REGION=us-west-2 -AWS_ACCESS_KEY_ID="don't write secrets to the sample file" -AWS_SECRET_ACCESS_KEY="don't write secrets to the sample file" -AWS_PINPOINT_REGION=us-west-2 -AWS_US_TOLL_FREE_NUMBER=+18446120782 +# secrets that internal apps, such as the admin app or document download, must use to authenticate with the API +ADMIN_CLIENT_ID=notify-admin +ADMIN_CLIENT_SECRET=dev-notify-secret-key +GOVUK_ALERTS_CLIENT_ID=govuk-alerts From 3e5e2f5017d6bd16b7240f06b9899a4038f1531e Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 14:06:16 -0400 Subject: [PATCH 48/65] create pipenv files --- Pipfile | 80 ++ Pipfile.lock | 1903 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.in | 4 +- 3 files changed, 1985 insertions(+), 2 deletions(-) create mode 100644 Pipfile create mode 100644 Pipfile.lock diff --git a/Pipfile b/Pipfile new file mode 100644 index 000000000..c1ae3089f --- /dev/null +++ b/Pipfile @@ -0,0 +1,80 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +alembic = "==1.7.7" +amqp = "==5.1.1" +arrow = "==1.2.2" +asn1crypto = "==1.5.1" +async-timeout = "==4.0.2" +attrs = "==21.4.0" +awscli = "==1.24.8" +bcrypt = "==3.2.2" +beautifulsoup4 = "==4.11.1" +billiard = "==3.6.4.0" +bleach = "==4.1.0" +blinker = "==1.4" +boto3 = "==1.23.8" +botocore = "==1.26.8" +cachetools = "==5.1.0" +celery = {version = "==5.2.7", extras = ["redis"]} +certifi = "==2022.5.18.1" +cffi = "==1.15.0" +charset-normalizer = "==2.0.12" +click = "==8.1.3" +click-datetime = "==0.2" +click-didyoumean = "==0.3.0" +click-plugins = "==1.1.1" +click-repl = "==0.2.0" +colorama = "==0.4.4" +defusedxml = "==0.7.1" +deprecated = "==1.2.13" +dnspython = "==2.2.1" +docopt = "==0.6.2" +docutils = "==0.16" +eventlet = "==0.33.1" +flask = "~=2.1.2" +flask-bcrypt = "==1.0.1" +flask-marshmallow = "==0.14.0" +flask-migrate = "==3.1.0" +flask-redis = "==0.4.0" +flask-sqlalchemy = {version = "==2.5.1", ref = "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", git = "https://github.com/pallets-eco/flask-sqlalchemy.git"} +gunicorn = {version = "==20.1.0", extras = ["eventlet"], ref = "1299ea9e967a61ae2edebe191082fd169b864c64", git = "https://github.com/benoitc/gunicorn.git"} +iso8601 = "==1.0.2" +itsdangerous = "==2.1.2" +jsonschema = {version = "==4.5.1", extras = ["format"]} +lxml = "==4.9.1" +marshmallow = "==3.15.0" +marshmallow-sqlalchemy = "==0.28.1" +notifications-python-client = "==6.3.0" +notifications-utils = {git = "https://github.com/GSA/notifications-utils.git"} +oscrypto = "==1.3.0" +psycopg2-binary = "==2.9.3" +pyjwt = "==2.4.0" +python-dotenv = "==0.20.0" +sqlalchemy = "==1.4.40" +werkzeug = "~=2.1.1" +# PaaS packages +awscli-cwlogs = "==1.4.6" +# gds metrics packages +prometheus-client = "==0.14.1" +gds-metrics = {ref = "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72", git = "https://github.com/alphagov/gds_metrics_python.git"} + +[dev-packages] +flake8 = "==4.0.1" +flake8-bugbear = "==22.4.25" +isort = "==5.10.1" +moto = "==3.1.9" +pytest = "==7.1.2" +pytest-env = "==0.6.2" +pytest-mock = "==3.7.0" +pytest-cov = "==3.0.0" +pytest-xdist = "==2.5.0" +freezegun = "==1.2.1" +requests-mock = "==1.9.3" +jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} + +[requires] +python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 000000000..8cd0bf67e --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,1903 @@ +{ + "_meta": { + "hash": { + "sha256": "eca9a39871c8db3e82fb384c39afc89fdc3c145c87653f5090927e578618ce11" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "alembic": { + "hashes": [ + "sha256:29be0856ec7591c39f4e1cb10f198045d890e6e2274cf8da80cb5e721a09642b", + "sha256:4961248173ead7ce8a21efb3de378f13b8398e6630fab0eb258dc74a8af24c58" + ], + "index": "pypi", + "version": "==1.7.7" + }, + "amqp": { + "hashes": [ + "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2", + "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359" + ], + "index": "pypi", + "version": "==5.1.1" + }, + "arrow": { + "hashes": [ + "sha256:05caf1fd3d9a11a1135b2b6f09887421153b94558e5ef4d090b567b47173ac2b", + "sha256:d622c46ca681b5b3e3574fcb60a04e5cc81b9625112d5fb2b44220c36c892177" + ], + "index": "pypi", + "version": "==1.2.2" + }, + "asn1crypto": { + "hashes": [ + "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", + "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67" + ], + "index": "pypi", + "version": "==1.5.1" + }, + "async-timeout": { + "hashes": [ + "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15", + "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c" + ], + "index": "pypi", + "version": "==4.0.2" + }, + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "index": "pypi", + "version": "==21.4.0" + }, + "awscli": { + "hashes": [ + "sha256:65d9414ead4f4027232bc889d5b2c8e4f205415ed877e1f9125ea238d89d025e", + "sha256:a895e378ebbf407b1dfc31205918c1e8521948d770e79006dad1316bf4987de0" + ], + "index": "pypi", + "version": "==1.24.8" + }, + "awscli-cwlogs": { + "hashes": [ + "sha256:44d2fe77d109b7b630fb8f6c06760ff6c5ec9861be16413fd5b977f5a4971f83" + ], + "index": "pypi", + "version": "==1.4.6" + }, + "bcrypt": { + "hashes": [ + "sha256:2b02d6bfc6336d1094276f3f588aa1225a598e27f8e3388f4db9948cb707b521", + "sha256:433c410c2177057705da2a9f2cd01dd157493b2a7ac14c8593a16b3dab6b6bfb", + "sha256:4e029cef560967fb0cf4a802bcf4d562d3d6b4b1bf81de5ec1abbe0f1adb027e", + "sha256:61bae49580dce88095d669226d5076d0b9d927754cedbdf76c6c9f5099ad6f26", + "sha256:6d2cb9d969bfca5bc08e45864137276e4c3d3d7de2b162171def3d188bf9d34a", + "sha256:7180d98a96f00b1050e93f5b0f556e658605dd9f524d0b0e68ae7944673f525e", + "sha256:7d9ba2e41e330d2af4af6b1b6ec9e6128e91343d0b4afb9282e54e5508f31baa", + "sha256:7ff2069240c6bbe49109fe84ca80508773a904f5a8cb960e02a977f7f519b129", + "sha256:88273d806ab3a50d06bc6a2fc7c87d737dd669b76ad955f449c43095389bc8fb", + "sha256:a2c46100e315c3a5b90fdc53e429c006c5f962529bc27e1dfd656292c20ccc40", + "sha256:cd43303d6b8a165c29ec6756afd169faba9396a9472cdff753fe9f19b96ce2fa" + ], + "index": "pypi", + "version": "==3.2.2" + }, + "beautifulsoup4": { + "hashes": [ + "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30", + "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693" + ], + "index": "pypi", + "version": "==4.11.1" + }, + "billiard": { + "hashes": [ + "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", + "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" + ], + "index": "pypi", + "version": "==3.6.4.0" + }, + "bleach": { + "hashes": [ + "sha256:0900d8b37eba61a802ee40ac0061f8c2b5dee29c1927dd1d233e075ebf5a71da", + "sha256:4d2651ab93271d1129ac9cbc679f524565cc8a1b791909c4a51eac4446a15994" + ], + "index": "pypi", + "version": "==4.1.0" + }, + "blinker": { + "hashes": [ + "sha256:471aee25f3992bd325afa3772f1063dbdbbca947a041b8b89466dc00d606f8b6" + ], + "index": "pypi", + "version": "==1.4" + }, + "boto3": { + "hashes": [ + "sha256:15733c2bbedce7a36fcf1749560c72c3ee90785aa6302a98658c7bffdcbe1f2a", + "sha256:ea8ebcea4ccb70d1cf57526d9eec6012c76796f28ada3e9cc1d89178683d8107" + ], + "index": "pypi", + "version": "==1.23.8" + }, + "botocore": { + "hashes": [ + "sha256:620851daf1245af5bc28137aa821375bac964aa0eddc482437c783fe01e298fc", + "sha256:e786722cb14de7319331cc55e9092174de66a768559700ef656d05ff41b3e24f" + ], + "index": "pypi", + "version": "==1.26.8" + }, + "cachetools": { + "hashes": [ + "sha256:4ebbd38701cdfd3603d1f751d851ed248ab4570929f2d8a7ce69e30c420b141c", + "sha256:8b3b8fa53f564762e5b221e9896798951e7f915513abf2ba072ce0f07f3f5a98" + ], + "index": "pypi", + "version": "==5.1.0" + }, + "celery": { + "extras": [ + "redis" + ], + "hashes": [ + "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14", + "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d" + ], + "index": "pypi", + "version": "==5.2.7" + }, + "certifi": { + "hashes": [ + "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", + "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + ], + "index": "pypi", + "version": "==2022.5.18.1" + }, + "cffi": { + "hashes": [ + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "index": "pypi", + "version": "==1.15.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + ], + "index": "pypi", + "version": "==2.0.12" + }, + "click": { + "hashes": [ + "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", + "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" + ], + "index": "pypi", + "version": "==8.1.3" + }, + "click-datetime": { + "hashes": [ + "sha256:7256ca518e648ada8e2550239ab328de125906e5b7199a5bd5bcbb4dfe28f946", + "sha256:c562ad24b3711784a655a49141b4a87933a78608fe66296259acae95fda5e115" + ], + "index": "pypi", + "version": "==0.2" + }, + "click-didyoumean": { + "hashes": [ + "sha256:a0713dc7a1de3f06bc0df5a9567ad19ead2d3d5689b434768a6145bff77c0667", + "sha256:f184f0d851d96b6d29297354ed981b7dd71df7ff500d82fa6d11f0856bee8035" + ], + "index": "pypi", + "version": "==0.3.0" + }, + "click-plugins": { + "hashes": [ + "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b", + "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8" + ], + "index": "pypi", + "version": "==1.1.1" + }, + "click-repl": { + "hashes": [ + "sha256:94b3fbbc9406a236f176e0506524b2937e4b23b6f4c0c0b2a0a83f8a64e9194b", + "sha256:cd12f68d745bf6151210790540b4cb064c7b13e571bc64b6957d98d120dacfd8" + ], + "index": "pypi", + "version": "==0.2.0" + }, + "colorama": { + "hashes": [ + "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b", + "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2" + ], + "index": "pypi", + "version": "==0.4.4" + }, + "defusedxml": { + "hashes": [ + "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", + "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61" + ], + "index": "pypi", + "version": "==0.7.1" + }, + "deprecated": { + "hashes": [ + "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d", + "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d" + ], + "index": "pypi", + "version": "==1.2.13" + }, + "dnspython": { + "hashes": [ + "sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e", + "sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f" + ], + "index": "pypi", + "version": "==2.2.1" + }, + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "index": "pypi", + "version": "==0.6.2" + }, + "docutils": { + "hashes": [ + "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", + "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" + ], + "index": "pypi", + "version": "==0.16" + }, + "eventlet": { + "hashes": [ + "sha256:a085922698e5029f820cf311a648ac324d73cec0e4792877609d978a4b5bbf31", + "sha256:afbe17f06a58491e9aebd7a4a03e70b0b63fd4cf76d8307bae07f280479b1515" + ], + "index": "pypi", + "version": "==0.33.1" + }, + "flask": { + "hashes": [ + "sha256:15972e5017df0575c3d6c090ba168b6db90259e620ac8d7ea813a396bad5b6cb", + "sha256:9013281a7402ad527f8fd56375164f3aa021ecfaff89bfe3825346c24f87e04c" + ], + "index": "pypi", + "version": "==2.1.3" + }, + "flask-bcrypt": { + "hashes": [ + "sha256:062fd991dc9118d05ac0583675507b9fe4670e44416c97e0e6819d03d01f808a", + "sha256:f07b66b811417ea64eb188ae6455b0b708a793d966e1a80ceec4a23bc42a4369" + ], + "index": "pypi", + "version": "==1.0.1" + }, + "flask-marshmallow": { + "hashes": [ + "sha256:2adcd782b5a4a6c5ae3c96701f320d8ca6997995a52b2661093c56cc3ed24754", + "sha256:bd01a6372cbe50e36f205cfff0fc5dab0b7b662c4c8b2c4fc06a3151b2950950" + ], + "index": "pypi", + "version": "==0.14.0" + }, + "flask-migrate": { + "hashes": [ + "sha256:57d6060839e3a7f150eaab6fe4e726d9e3e7cffe2150fb223d73f92421c6d1d9", + "sha256:a6498706241aba6be7a251078de9cf166d74307bca41a4ca3e403c9d39e2f897" + ], + "index": "pypi", + "version": "==3.1.0" + }, + "flask-redis": { + "hashes": [ + "sha256:8d79eef4eb1217095edab603acc52f935b983ae4b7655ee7c82c0dfd87315d17", + "sha256:e1fccc11e7ea35c2a4d68c0b9aa58226a098e45e834d615c7b6c4928b01ddd6c" + ], + "index": "pypi", + "version": "==0.4.0" + }, + "flask-sqlalchemy": { + "hashes": [ + "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912", + "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.5.1" + }, + "flask-sqlalchemy==2-5-1": { + "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", + "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa" + }, + "flask-sqlalchemy==2.5.1": { + "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", + "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa" + }, + "fqdn": { + "hashes": [ + "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", + "sha256:3a179af3761e4df6eb2e026ff9e1a3033d3587bf980a0b1b2e1e5d08d7358014" + ], + "version": "==1.5.1" + }, + "gds-metrics": { + "git": "https://github.com/alphagov/gds_metrics_python.git", + "ref": "6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72" + }, + "geojson": { + "hashes": [ + "sha256:6e4bb7ace4226a45d9c8c8b1348b3fc43540658359f93c3f7e03efa9f15f658a", + "sha256:ccbd13368dd728f4e4f13ffe6aaf725b6e802c692ba0dde628be475040c534ba" + ], + "version": "==2.5.0" + }, + "govuk-bank-holidays": { + "hashes": [ + "sha256:6c993f59bff512740066ae5891a9ce3155e94291560d1d5169fea6e634111a46", + "sha256:9e214828802bbed303f01dacee3facc2296a8bcac9c814654ac7300b37381f78" + ], + "version": "==0.11" + }, + "greenlet": { + "hashes": [ + "sha256:0120a879aa2b1ac5118bce959ea2492ba18783f65ea15821680a256dfad04754", + "sha256:025b8de2273d2809f027d347aa2541651d2e15d593bbce0d5f502ca438c54136", + "sha256:05ae7383f968bba4211b1fbfc90158f8e3da86804878442b4fb6c16ccbcaa519", + "sha256:0914f02fcaa8f84f13b2df4a81645d9e82de21ed95633765dd5cc4d3af9d7403", + "sha256:0971d37ae0eaf42344e8610d340aa0ad3d06cd2eee381891a10fe771879791f9", + "sha256:0a954002064ee919b444b19c1185e8cce307a1f20600f47d6f4b6d336972c809", + "sha256:0aa1845944e62f358d63fcc911ad3b415f585612946b8edc824825929b40e59e", + "sha256:104f29dd822be678ef6b16bf0035dcd43206a8a48668a6cae4d2fe9c7a7abdeb", + "sha256:11fc7692d95cc7a6a8447bb160d98671ab291e0a8ea90572d582d57361360f05", + "sha256:17a69967561269b691747e7f436d75a4def47e5efcbc3c573180fc828e176d80", + "sha256:2794eef1b04b5ba8948c72cc606aab62ac4b0c538b14806d9c0d88afd0576d6b", + "sha256:2c6e942ca9835c0b97814d14f78da453241837419e0d26f7403058e8db3e38f8", + "sha256:2ccdc818cc106cc238ff7eba0d71b9c77be868fdca31d6c3b1347a54c9b187b2", + "sha256:325f272eb997916b4a3fc1fea7313a8adb760934c2140ce13a2117e1b0a8095d", + "sha256:39464518a2abe9c505a727af7c0b4efff2cf242aa168be5f0daa47649f4d7ca8", + "sha256:3a24f3213579dc8459e485e333330a921f579543a5214dbc935bc0763474ece3", + "sha256:3aeac044c324c1a4027dca0cde550bd83a0c0fbff7ef2c98df9e718a5086c194", + "sha256:3c22998bfef3fcc1b15694818fc9b1b87c6cc8398198b96b6d355a7bcb8c934e", + "sha256:467b73ce5dcd89e381292fb4314aede9b12906c18fab903f995b86034d96d5c8", + "sha256:4a8b58232f5b72973350c2b917ea3df0bebd07c3c82a0a0e34775fc2c1f857e9", + "sha256:4f74aa0092602da2069df0bc6553919a15169d77bcdab52a21f8c5242898f519", + "sha256:5662492df0588a51d5690f6578f3bbbd803e7f8d99a99f3bf6128a401be9c269", + "sha256:5c2d21c2b768d8c86ad935e404cc78c30d53dea009609c3ef3a9d49970c864b5", + "sha256:5edf75e7fcfa9725064ae0d8407c849456553a181ebefedb7606bac19aa1478b", + "sha256:60839ab4ea7de6139a3be35b77e22e0398c270020050458b3d25db4c7c394df5", + "sha256:62723e7eb85fa52e536e516ee2ac91433c7bb60d51099293671815ff49ed1c21", + "sha256:64e10f303ea354500c927da5b59c3802196a07468332d292aef9ddaca08d03dd", + "sha256:66aa4e9a726b70bcbfcc446b7ba89c8cec40f405e51422c39f42dfa206a96a05", + "sha256:695d0d8b5ae42c800f1763c9fce9d7b94ae3b878919379150ee5ba458a460d57", + "sha256:70048d7b2c07c5eadf8393e6398595591df5f59a2f26abc2f81abca09610492f", + "sha256:7afa706510ab079fd6d039cc6e369d4535a48e202d042c32e2097f030a16450f", + "sha256:7cf37343e43404699d58808e51f347f57efd3010cc7cee134cdb9141bd1ad9ea", + "sha256:8149a6865b14c33be7ae760bcdb73548bb01e8e47ae15e013bf7ef9290ca309a", + "sha256:814f26b864ed2230d3a7efe0336f5766ad012f94aad6ba43a7c54ca88dd77cba", + "sha256:82a38d7d2077128a017094aff334e67e26194f46bd709f9dcdacbf3835d47ef5", + "sha256:83a7a6560df073ec9de2b7cb685b199dfd12519bc0020c62db9d1bb522f989fa", + "sha256:8415239c68b2ec9de10a5adf1130ee9cb0ebd3e19573c55ba160ff0ca809e012", + "sha256:88720794390002b0c8fa29e9602b395093a9a766b229a847e8d88349e418b28a", + "sha256:890f633dc8cb307761ec566bc0b4e350a93ddd77dc172839be122be12bae3e10", + "sha256:8926a78192b8b73c936f3e87929931455a6a6c6c385448a07b9f7d1072c19ff3", + "sha256:8c0581077cf2734569f3e500fab09c0ff6a2ab99b1afcacbad09b3c2843ae743", + "sha256:8fda1139d87ce5f7bd80e80e54f9f2c6fe2f47983f1a6f128c47bf310197deb6", + "sha256:91a84faf718e6f8b888ca63d0b2d6d185c8e2a198d2a7322d75c303e7097c8b7", + "sha256:924df1e7e5db27d19b1359dc7d052a917529c95ba5b8b62f4af611176da7c8ad", + "sha256:949c9061b8c6d3e6e439466a9be1e787208dec6246f4ec5fffe9677b4c19fcc3", + "sha256:9649891ab4153f217f319914455ccf0b86986b55fc0573ce803eb998ad7d6854", + "sha256:96656c5f7c95fc02c36d4f6ef32f4e94bb0b6b36e6a002c21c39785a4eec5f5d", + "sha256:a812df7282a8fc717eafd487fccc5ba40ea83bb5b13eb3c90c446d88dbdfd2be", + "sha256:a8d24eb5cb67996fb84633fdc96dbc04f2d8b12bfcb20ab3222d6be271616b67", + "sha256:bef49c07fcb411c942da6ee7d7ea37430f830c482bf6e4b72d92fd506dd3a427", + "sha256:bffba15cff4802ff493d6edcf20d7f94ab1c2aee7cfc1e1c7627c05f1102eee8", + "sha256:c0643250dd0756f4960633f5359884f609a234d4066686754e834073d84e9b51", + "sha256:c6f90234e4438062d6d09f7d667f79edcc7c5e354ba3a145ff98176f974b8132", + "sha256:c8c9301e3274276d3d20ab6335aa7c5d9e5da2009cccb01127bddb5c951f8870", + "sha256:c8ece5d1a99a2adcb38f69af2f07d96fb615415d32820108cd340361f590d128", + "sha256:cb863057bed786f6622982fb8b2c122c68e6e9eddccaa9fa98fd937e45ee6c4f", + "sha256:ccbe7129a282ec5797df0451ca1802f11578be018a32979131065565da89b392", + "sha256:d25cdedd72aa2271b984af54294e9527306966ec18963fd032cc851a725ddc1b", + "sha256:d75afcbb214d429dacdf75e03a1d6d6c5bd1fa9c35e360df8ea5b6270fb2211c", + "sha256:d7815e1519a8361c5ea2a7a5864945906f8e386fa1bc26797b4d443ab11a4589", + "sha256:eb6ac495dccb1520667cfea50d89e26f9ffb49fa28496dea2b95720d8b45eb54", + "sha256:ec615d2912b9ad807afd3be80bf32711c0ff9c2b00aa004a45fd5d5dde7853d9", + "sha256:f5e09dc5c6e1796969fd4b775ea1417d70e49a5df29aaa8e5d10675d9e11872c", + "sha256:f6661b58412879a2aa099abb26d3c93e91dedaba55a6394d1fb1512a77e85de9", + "sha256:f7d20c3267385236b4ce54575cc8e9f43e7673fc761b069c820097092e318e3b", + "sha256:fe7c51f8a2ab616cb34bc33d810c887e89117771028e1e3d3b77ca25ddeace04" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.1.3.post0" + }, + "gunicorn[eventlet]==20-1-0": { + "git": "https://github.com/benoitc/gunicorn.git", + "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" + }, + "gunicorn[eventlet]==20.1.0": { + "git": "https://github.com/benoitc/gunicorn.git", + "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" + }, + "idna": { + "hashes": [ + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + ], + "version": "==3.4" + }, + "importlib-metadata": { + "hashes": [ + "sha256:da31db32b304314d044d3c12c79bd59e307889b287ad12ff387b3500835fc2ab", + "sha256:ddb0e35065e8938f867ed4928d0ae5bf2a53b7773871bfe6bcc7e4fcdc7dea43" + ], + "markers": "python_version < '3.10'", + "version": "==5.0.0" + }, + "iso8601": { + "hashes": [ + "sha256:27f503220e6845d9db954fb212b95b0362d8b7e6c1b2326a87061c3de93594b1", + "sha256:d7bc01b1c2a43b259570bb307f057abc578786ea734ba2b87b836c5efc5bd443" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "isoduration": { + "hashes": [ + "sha256:ac2f9015137935279eac671f94f89eb00584f940f5dc49462a0c4ee692ba1bd9", + "sha256:b2904c2a4228c3d44f409c8ae8e2370eb21a26f7ac2ec5446df141dde3452042" + ], + "version": "==20.11.0" + }, + "itsdangerous": { + "hashes": [ + "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", + "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" + ], + "index": "pypi", + "version": "==2.1.2" + }, + "jinja2": { + "hashes": [ + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "jsonpointer": { + "hashes": [ + "sha256:51801e558539b4e9cd268638c078c6c5746c9ac96bc38152d443400e4f3793e9", + "sha256:97cba51526c829282218feb99dab1b1e6bdf8efd1c43dc9d57be093c0d69c99a" + ], + "version": "==2.3" + }, + "jsonschema": { + "extras": [ + "format" + ], + "hashes": [ + "sha256:71b5e39324422543546572954ce71c67728922c104902cb7ce252e522235b33f", + "sha256:7c6d882619340c3347a1bf7315e147e6d3dae439033ae6383d6acb908c101dfc" + ], + "index": "pypi", + "version": "==4.5.1" + }, + "kombu": { + "hashes": [ + "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610", + "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4" + ], + "markers": "python_version >= '3.7'", + "version": "==5.2.4" + }, + "lxml": { + "hashes": [ + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" + ], + "index": "pypi", + "version": "==4.9.1" + }, + "mako": { + "hashes": [ + "sha256:7fde96466fcfeedb0eed94f187f20b23d85e4cb41444be0e542e2c8c65c396cd", + "sha256:c413a086e38cd885088d5e165305ee8eed04e8b3f8f62df343480da0a385735f" + ], + "markers": "python_version >= '3.7'", + "version": "==1.2.3" + }, + "markupsafe": { + "hashes": [ + "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", + "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88", + "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5", + "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7", + "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a", + "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603", + "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1", + "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135", + "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247", + "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6", + "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601", + "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77", + "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02", + "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e", + "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63", + "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f", + "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980", + "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b", + "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812", + "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff", + "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96", + "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1", + "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925", + "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a", + "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6", + "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e", + "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f", + "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4", + "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f", + "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3", + "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c", + "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a", + "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417", + "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a", + "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a", + "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37", + "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452", + "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933", + "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", + "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.1" + }, + "marshmallow": { + "hashes": [ + "sha256:2aaaab4f01ef4f5a011a21319af9fce17ab13bf28a026d1252adab0e035648d5", + "sha256:ff79885ed43b579782f48c251d262e062bce49c65c52412458769a4fb57ac30f" + ], + "index": "pypi", + "version": "==3.15.0" + }, + "marshmallow-sqlalchemy": { + "hashes": [ + "sha256:aa376747296780a56355e3067b9c8bf43a2a1c44ff985de82b3a5d9e161ca2b8", + "sha256:dbb061c19375eca3a7d18358d2ca8bbaee825fc3000a3f114e2698282362b536" + ], + "index": "pypi", + "version": "==0.28.1" + }, + "mistune": { + "hashes": [ + "sha256:59a3429db53c50b5c6bcc8a07f8848cb00d7dc8bdb431a4ab41920d201d4756e", + "sha256:88a1051873018da288eee8538d476dffe1262495144b33ecb586c4ab266bb8d4" + ], + "version": "==0.8.4" + }, + "notifications-python-client": { + "hashes": [ + "sha256:47c803fcc8b4098d069b92547bb52607b558cec25c19e2697a74faab2e5ef4c0" + ], + "index": "pypi", + "version": "==6.3.0" + }, + "notifications-utils": { + "git": "https://github.com/GSA/notifications-utils.git", + "ref": "90c12da575f4e481452d4fcd2a594204b0c28249" + }, + "orderedset": { + "hashes": [ + "sha256:b2f5ccfb5a86e7b3b3ddf18b29779cc18b24653abf9d6da4bebecf33780a6e29" + ], + "version": "==2.0.3" + }, + "oscrypto": { + "hashes": [ + "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085", + "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4" + ], + "index": "pypi", + "version": "==1.3.0" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "phonenumbers": { + "hashes": [ + "sha256:057d1966962fb86b3dc447bfac2c8e25ceed774509e49b180926a13a99910318", + "sha256:0b234c4a9519fac18d00b3c542f5b429513ea69372d4f95fbd0f716f5e2a89b5" + ], + "version": "==8.12.57" + }, + "prometheus-client": { + "hashes": [ + "sha256:522fded625282822a89e2773452f42df14b5a8e84a86433e3f8a189c1d54dc01", + "sha256:5459c427624961076277fdc6dc50540e2bacb98eebde99886e59ec55ed92093a" + ], + "index": "pypi", + "version": "==0.14.1" + }, + "prompt-toolkit": { + "hashes": [ + "sha256:9696f386133df0fc8ca5af4895afe5d78f5fcfe5258111c2a79a1c3e41ffa96d", + "sha256:9ada952c9d1787f52ff6d5f3484d0b4df8952787c087edf6a1f7c2cb1ea88148" + ], + "markers": "python_full_version >= '3.6.2'", + "version": "==3.0.31" + }, + "psycopg2-binary": { + "hashes": [ + "sha256:01310cf4cf26db9aea5158c217caa92d291f0500051a6469ac52166e1a16f5b7", + "sha256:083a55275f09a62b8ca4902dd11f4b33075b743cf0d360419e2051a8a5d5ff76", + "sha256:090f3348c0ab2cceb6dfbe6bf721ef61262ddf518cd6cc6ecc7d334996d64efa", + "sha256:0a29729145aaaf1ad8bafe663131890e2111f13416b60e460dae0a96af5905c9", + "sha256:0c9d5450c566c80c396b7402895c4369a410cab5a82707b11aee1e624da7d004", + "sha256:10bb90fb4d523a2aa67773d4ff2b833ec00857f5912bafcfd5f5414e45280fb1", + "sha256:12b11322ea00ad8db8c46f18b7dfc47ae215e4df55b46c67a94b4effbaec7094", + "sha256:152f09f57417b831418304c7f30d727dc83a12761627bb826951692cc6491e57", + "sha256:15803fa813ea05bef089fa78835118b5434204f3a17cb9f1e5dbfd0b9deea5af", + "sha256:15c4e4cfa45f5a60599d9cec5f46cd7b1b29d86a6390ec23e8eebaae84e64554", + "sha256:183a517a3a63503f70f808b58bfbf962f23d73b6dccddae5aa56152ef2bcb232", + "sha256:1f14c8b0942714eb3c74e1e71700cbbcb415acbc311c730370e70c578a44a25c", + "sha256:1f6b813106a3abdf7b03640d36e24669234120c72e91d5cbaeb87c5f7c36c65b", + "sha256:280b0bb5cbfe8039205c7981cceb006156a675362a00fe29b16fbc264e242834", + "sha256:2d872e3c9d5d075a2e104540965a1cf898b52274a5923936e5bfddb58c59c7c2", + "sha256:2f2534ab7dc7e776a263b463a16e189eb30e85ec9bbe1bff9e78dae802608932", + "sha256:2f9ffd643bc7349eeb664eba8864d9e01f057880f510e4681ba40a6532f93c71", + "sha256:3303f8807f342641851578ee7ed1f3efc9802d00a6f83c101d21c608cb864460", + "sha256:35168209c9d51b145e459e05c31a9eaeffa9a6b0fd61689b48e07464ffd1a83e", + "sha256:3a79d622f5206d695d7824cbf609a4f5b88ea6d6dab5f7c147fc6d333a8787e4", + "sha256:404224e5fef3b193f892abdbf8961ce20e0b6642886cfe1fe1923f41aaa75c9d", + "sha256:46f0e0a6b5fa5851bbd9ab1bc805eef362d3a230fbdfbc209f4a236d0a7a990d", + "sha256:47133f3f872faf28c1e87d4357220e809dfd3fa7c64295a4a148bcd1e6e34ec9", + "sha256:526ea0378246d9b080148f2d6681229f4b5964543c170dd10bf4faaab6e0d27f", + "sha256:53293533fcbb94c202b7c800a12c873cfe24599656b341f56e71dd2b557be063", + "sha256:539b28661b71da7c0e428692438efbcd048ca21ea81af618d845e06ebfd29478", + "sha256:57804fc02ca3ce0dbfbef35c4b3a4a774da66d66ea20f4bda601294ad2ea6092", + "sha256:63638d875be8c2784cfc952c9ac34e2b50e43f9f0a0660b65e2a87d656b3116c", + "sha256:6472a178e291b59e7f16ab49ec8b4f3bdada0a879c68d3817ff0963e722a82ce", + "sha256:68641a34023d306be959101b345732360fc2ea4938982309b786f7be1b43a4a1", + "sha256:6e82d38390a03da28c7985b394ec3f56873174e2c88130e6966cb1c946508e65", + "sha256:761df5313dc15da1502b21453642d7599d26be88bff659382f8f9747c7ebea4e", + "sha256:7af0dd86ddb2f8af5da57a976d27cd2cd15510518d582b478fbb2292428710b4", + "sha256:7b1e9b80afca7b7a386ef087db614faebbf8839b7f4db5eb107d0f1a53225029", + "sha256:874a52ecab70af13e899f7847b3e074eeb16ebac5615665db33bce8a1009cf33", + "sha256:887dd9aac71765ac0d0bac1d0d4b4f2c99d5f5c1382d8b770404f0f3d0ce8a39", + "sha256:8b344adbb9a862de0c635f4f0425b7958bf5a4b927c8594e6e8d261775796d53", + "sha256:8fc53f9af09426a61db9ba357865c77f26076d48669f2e1bb24d85a22fb52307", + "sha256:91920527dea30175cc02a1099f331aa8c1ba39bf8b7762b7b56cbf54bc5cce42", + "sha256:93cd1967a18aa0edd4b95b1dfd554cf15af657cb606280996d393dadc88c3c35", + "sha256:99485cab9ba0fa9b84f1f9e1fef106f44a46ef6afdeec8885e0b88d0772b49e8", + "sha256:9d29409b625a143649d03d0fd7b57e4b92e0ecad9726ba682244b73be91d2fdb", + "sha256:a29b3ca4ec9defec6d42bf5feb36bb5817ba3c0230dd83b4edf4bf02684cd0ae", + "sha256:a9e1f75f96ea388fbcef36c70640c4efbe4650658f3d6a2967b4cc70e907352e", + "sha256:accfe7e982411da3178ec690baaceaad3c278652998b2c45828aaac66cd8285f", + "sha256:adf20d9a67e0b6393eac162eb81fb10bc9130a80540f4df7e7355c2dd4af9fba", + "sha256:af9813db73395fb1fc211bac696faea4ca9ef53f32dc0cfa27e4e7cf766dcf24", + "sha256:b1c8068513f5b158cf7e29c43a77eb34b407db29aca749d3eb9293ee0d3103ca", + "sha256:b3a24a1982ae56461cc24f6680604fffa2c1b818e9dc55680da038792e004d18", + "sha256:bda845b664bb6c91446ca9609fc69f7db6c334ec5e4adc87571c34e4f47b7ddb", + "sha256:c381bda330ddf2fccbafab789d83ebc6c53db126e4383e73794c74eedce855ef", + "sha256:c3ae8e75eb7160851e59adc77b3a19a976e50622e44fd4fd47b8b18208189d42", + "sha256:d1c1b569ecafe3a69380a94e6ae09a4789bbb23666f3d3a08d06bbd2451f5ef1", + "sha256:def68d7c21984b0f8218e8a15d514f714d96904265164f75f8d3a70f9c295667", + "sha256:dffc08ca91c9ac09008870c9eb77b00a46b3378719584059c034b8945e26b272", + "sha256:e3699852e22aa68c10de06524a3721ade969abf382da95884e6a10ff798f9281", + "sha256:e6aa71ae45f952a2205377773e76f4e3f27951df38e69a4c95440c779e013560", + "sha256:e847774f8ffd5b398a75bc1c18fbb56564cda3d629fe68fd81971fece2d3c67e", + "sha256:ffb7a888a047696e7f8240d649b43fb3644f14f0ee229077e7f6b9f9081635bd" + ], + "index": "pypi", + "version": "==2.9.3" + }, + "pyasn1": { + "hashes": [ + "sha256:014c0e9976956a08139dc0712ae195324a75e142284d5f87f1a87ee1b068a359", + "sha256:03840c999ba71680a131cfaee6fab142e1ed9bbd9c693e285cc6aca0d555e576", + "sha256:0458773cfe65b153891ac249bcf1b5f8f320b7c2ce462151f8fa74de8934becf", + "sha256:08c3c53b75eaa48d71cf8c710312316392ed40899cb34710d092e96745a358b7", + "sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d", + "sha256:5c9414dcfede6e441f7e8f81b43b34e834731003427e5b09e4e00e3172a10f00", + "sha256:6e7545f1a61025a4e58bb336952c5061697da694db1cae97b116e9c46abcf7c8", + "sha256:78fa6da68ed2727915c4767bb386ab32cdba863caa7dbe473eaae45f9959da86", + "sha256:7ab8a544af125fb704feadb008c99a88805126fb525280b2270bb25cc1d78a12", + "sha256:99fcc3c8d804d1bc6d9a099921e39d827026409a58f2a720dcdb89374ea0c776", + "sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba", + "sha256:e89bf84b5437b532b0803ba5c9a5e054d21fec423a89952a74f87fa2c9b7bce2", + "sha256:fec3e9d8e36808a28efb59b489e4528c10ad0f480e57dcc32b4de5c9d8c9fdf3" + ], + "version": "==0.4.8" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pyjwt": { + "hashes": [ + "sha256:72d1d253f32dbd4f5c88eaf1fdc62f3a19f676ccbadb9dbc5d07e951b2b26daf", + "sha256:d42908208c699b3b973cbeb01a969ba6a96c821eefb1c5bfe4c390c01d67abba" + ], + "index": "pypi", + "version": "==2.4.0" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pypdf2": { + "hashes": [ + "sha256:3c7badd512c21711eb1789c2eadbf96279289c0f94452ee54a86473bfbefd732", + "sha256:7291a552ead2e7c2d556cce03bf71842fbbab478fcba13ae75ab1d59746b4dcb" + ], + "markers": "python_version >= '3.6'", + "version": "==2.11.1" + }, + "pyproj": { + "hashes": [ + "sha256:0fff9c3a991508f16027be27d153f6c5583d03799443639d13c681e60f49e2d7", + "sha256:12f62c20656ac9b6076ebb213e9a635d52f4f01fef95310121d337e62e910cb6", + "sha256:14ad113b5753c6057f9b2f3c85a6497cef7fa237c4328f2943c0223e98c1dde6", + "sha256:1f9c100fd0fd80edbc7e4daa303600a8cbef6f0de43d005617acb38276b88dc0", + "sha256:221d8939685e0c43ee594c9f04b6a73a10e8e1cc0e85f28be0b4eb2f1bc8777d", + "sha256:25a36e297f3e0524694d40259e3e895edc1a47492a0e30608268ffc1328e3f5d", + "sha256:2cb8592259ea54e7557523b079d3f2304081680bdb48bfbf0fd879ee6156129c", + "sha256:3b85acf09e5a9e35cd9ee72989793adb7089b4e611be02a43d3d0bda50ad116b", + "sha256:45554f47d1a12a84b0620e4abc08a2a1b5d9f273a4759eaef75e74788ec7162a", + "sha256:4688b4cd62cbd86b5e855f9e27d90fbb53f2b4c2ea1cd394a46919e1a4151b89", + "sha256:47ad53452ae1dc8b0bf1df920a210bb5616989085aa646592f8681f1d741a754", + "sha256:48787962232109bad8b72e27949037a9b03591228a6955f25dbe451233e8648a", + "sha256:4a23d84c5ffc383c7d9f0bde3a06fc1f6697b1b96725597f8f01e7b4bef0a2b5", + "sha256:4e161114bc92701647a83c4bbce79489984f12d980cabb365516e953d1450885", + "sha256:4fd425ee8b6781c249c7adb7daa2e6c41ce573afabe4f380f5eecd913b56a3be", + "sha256:52e54796e2d9554a5eb8f11df4748af1fbbc47f76aa234d6faf09216a84554c5", + "sha256:5816807ca0bdc7256558770c6206a6783a3f02bcf844f94ee245f197bb5f7285", + "sha256:65a0bcdbad95b3c00b419e5d75b1f7e450ec17349b5ea16bf7438ac1d50a12a2", + "sha256:77d5f519f3cdb94b026ecca626f78db4f041afe201cf082079c8c0092a30b087", + "sha256:82200b4569d68b421c079d2973475b58d5959306fe758b43366e79fe96facfe5", + "sha256:954b068136518b3174d0a99448056e97af62b63392a95c420894f7de2229dae6", + "sha256:9a496d9057b2128db9d733e66b206f2d5954bbae6b800d412f562d780561478c", + "sha256:a454a7c4423faa2a14e939d08ef293ee347fa529c9df79022b0585a6e1d8310c", + "sha256:a708445927ace9857f52c3ba67d2915da7b41a8fdcd9b8f99a4c9ed60a75eb33", + "sha256:aa5171f700f174777a9e9ed8f4655583243967c0f9cf2c90e3f54e54ff740134", + "sha256:ccb4b70ad25218027f77e0c8934d10f9b7cdf91d5e64080147743d58fddbc3c0", + "sha256:d94afed99f31673d3d19fe750283621e193e2a53ca9e0443bf9d092c3905833b", + "sha256:e7e609903572a56cca758bbaee5c1663c3e829ddce5eec4f368e68277e37022b", + "sha256:f343725566267a296b09ee7e591894f1fdc90f84f8ad5ec476aeb53bd4479c07", + "sha256:f80adda8c54b84271a93829477a01aa57bc178c834362e9f74e1de1b5033c74c" + ], + "markers": "python_version >= '3.8'", + "version": "==3.4.0" + }, + "pyrsistent": { + "hashes": [ + "sha256:0e3e1fcc45199df76053026a51cc59ab2ea3fc7c094c6627e93b7b44cdae2c8c", + "sha256:1b34eedd6812bf4d33814fca1b66005805d3640ce53140ab8bbb1e2651b0d9bc", + "sha256:4ed6784ceac462a7d6fcb7e9b663e93b9a6fb373b7f43594f9ff68875788e01e", + "sha256:5d45866ececf4a5fff8742c25722da6d4c9e180daa7b405dc0a2a2790d668c26", + "sha256:636ce2dc235046ccd3d8c56a7ad54e99d5c1cd0ef07d9ae847306c91d11b5fec", + "sha256:6455fc599df93d1f60e1c5c4fe471499f08d190d57eca040c0ea182301321286", + "sha256:6bc66318fb7ee012071b2792024564973ecc80e9522842eb4e17743604b5e045", + "sha256:7bfe2388663fd18bd8ce7db2c91c7400bf3e1a9e8bd7d63bf7e77d39051b85ec", + "sha256:7ec335fc998faa4febe75cc5268a9eac0478b3f681602c1f27befaf2a1abe1d8", + "sha256:914474c9f1d93080338ace89cb2acee74f4f666fb0424896fcfb8d86058bf17c", + "sha256:b568f35ad53a7b07ed9b1b2bae09eb15cdd671a5ba5d2c66caee40dbf91c68ca", + "sha256:cdfd2c361b8a8e5d9499b9082b501c452ade8bbf42aef97ea04854f4a3f43b22", + "sha256:d1b96547410f76078eaf66d282ddca2e4baae8964364abb4f4dcdde855cd123a", + "sha256:d4d61f8b993a7255ba714df3aca52700f8125289f84f704cf80916517c46eb96", + "sha256:d7a096646eab884bf8bed965bad63ea327e0d0c38989fc83c5ea7b8a87037bfc", + "sha256:df46c854f490f81210870e509818b729db4488e1f30f2a1ce1698b2295a878d1", + "sha256:e24a828f57e0c337c8d8bb9f6b12f09dfdf0273da25fda9e314f0b684b415a07", + "sha256:e4f3149fd5eb9b285d6bfb54d2e5173f6a116fe19172686797c056672689daf6", + "sha256:e92a52c166426efbe0d1ec1332ee9119b6d32fc1f0bbfd55d5c1088070e7fc1b", + "sha256:f87cc2863ef33c709e237d4b5f4502a62a00fab450c9e020892e8e2ede5847f5", + "sha256:fd8da6d0124efa2f67d86fa70c851022f87c98e205f0594e1fae044e7119a5a6" + ], + "markers": "python_version >= '3.7'", + "version": "==0.18.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "python-dotenv": { + "hashes": [ + "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f", + "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938" + ], + "index": "pypi", + "version": "==0.20.0" + }, + "python-json-logger": { + "hashes": [ + "sha256:3b03487b14eb9e4f77e4fc2a023358b5394b82fd89cecf5586259baed57d8c6f", + "sha256:764d762175f99fcc4630bd4853b09632acb60a6224acb27ce08cd70f0b1b81bd" + ], + "markers": "python_version >= '3.5'", + "version": "==2.0.4" + }, + "pytz": { + "hashes": [ + "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", + "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + ], + "version": "==2022.5" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==5.4.1" + }, + "redis": { + "hashes": [ + "sha256:a52d5694c9eb4292770084fa8c863f79367ca19884b329ab574d5cb2036b3e54", + "sha256:ddf27071df4adf3821c4f2ca59d67525c3a82e5f268bed97b813cb4fabf87880" + ], + "version": "==4.3.4" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", + "version": "==2.28.1" + }, + "rfc3339-validator": { + "hashes": [ + "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", + "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa" + ], + "version": "==0.1.4" + }, + "rfc3987": { + "hashes": [ + "sha256:10702b1e51e5658843460b189b185c0366d2cf4cff716f13111b0ea9fd2dce53", + "sha256:d3c4d257a560d544e9826b38bc81db676890c79ab9d7ac92b39c7a253d5ca733" + ], + "version": "==1.3.8" + }, + "rsa": { + "hashes": [ + "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", + "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" + ], + "markers": "python_version >= '3.5' and python_full_version < '4.0.0'", + "version": "==4.7.2" + }, + "s3transfer": { + "hashes": [ + "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971", + "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed" + ], + "markers": "python_version >= '3.6'", + "version": "==0.5.2" + }, + "setuptools": { + "hashes": [ + "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17", + "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356" + ], + "markers": "python_version >= '3.7'", + "version": "==65.5.0" + }, + "shapely": { + "hashes": [ + "sha256:02dd5d7dc6e46515d88874134dc8fcdc65826bca93c3eecee59d1910c42c1b17", + "sha256:0b4ee3132ee90f07d63db3aea316c4c065ed7a26231458dda0874414a09d6ba3", + "sha256:0d885cb0cf670c1c834df3f371de8726efdf711f18e2a75da5cfa82843a7ab65", + "sha256:147066da0be41b147a61f8eb805dea3b13709dbc873a431ccd7306e24d712bc0", + "sha256:21776184516a16bf82a0c3d6d6a312b3cd15a4cabafc61ee01cf2714a82e8396", + "sha256:2e0a8c2e55f1be1312b51c92b06462ea89e6bb703fab4b114e7a846d941cfc40", + "sha256:2fd15397638df291c427a53d641d3e6fd60458128029c8c4f487190473a69a91", + "sha256:3480657460e939f45a7d359ef0e172a081f249312557fe9aa78c4fd3a362d993", + "sha256:370b574c78dc5af3a198a6da5d9b3d7c04654bd2ef7e80e80a3a0992dfb2d9cd", + "sha256:38f0fbbcb8ca20c16451c966c1f527cc43968e121c8a048af19ed3e339a921cd", + "sha256:4728666fff8cccc65a07448cae72c75a8773fea061c3f4f139c44adc429b18c3", + "sha256:48dcfffb9e225c0481120f4bdf622131c8c95f342b00b158cdbe220edbbe20b6", + "sha256:532a55ee2a6c52d23d6f7d1567c8f0473635f3b270262c44e1b0c88096827e22", + "sha256:5d7f85c2d35d39ff53c9216bc76b7641c52326f7e09aaad1789a3611a0f812f2", + "sha256:65b21243d8f6bcd421210daf1fabb9de84de2c04353c5b026173b88d17c1a581", + "sha256:66bdac74fbd1d3458fa787191a90fa0ae610f09e2a5ec398c36f968cc0ed743f", + "sha256:6d388c0c1bd878ed1af4583695690aa52234b02ed35f93a1c8486ff52a555838", + "sha256:6fe855e7d45685926b6ba00aaeb5eba5862611f7465775dacd527e081a8ced6d", + "sha256:753ed0e21ab108bd4282405b9b659f2e985e8502b1a72b978eaa51d3496dee19", + "sha256:783bad5f48e2708a0e2f695a34ed382e4162c795cb2f0368b39528ac1d6db7ed", + "sha256:78fb9d929b8ee15cfd424b6c10879ce1907f24e05fb83310fc47d2cd27088e40", + "sha256:84010db15eb364a52b74ea8804ef92a6a930dfc1981d17a369444b6ddec66efd", + "sha256:8d086591f744be483b34628b391d741e46f2645fe37594319e0a673cc2c26bcf", + "sha256:8e59817b0fe63d34baedaabba8c393c0090f061917d18fc0bcc2f621937a8f73", + "sha256:99a2f0da0109e81e0c101a2b4cd8412f73f5f299e7b5b2deaf64cd2a100ac118", + "sha256:99ab0ddc05e44acabdbe657c599fdb9b2d82e86c5493bdae216c0c4018a82dee", + "sha256:a23ef3882d6aa203dd3623a3d55d698f59bfbd9f8a3bfed52c2da05a7f0f8640", + "sha256:a354199219c8d836f280b88f2c5102c81bb044ccea45bd361dc38a79f3873714", + "sha256:a74631e511153366c6dbe3229fa93f877e3c87ea8369cd00f1d38c76b0ed9ace", + "sha256:ab38f7b5196ace05725e407cb8cab9ff66edb8e6f7bb36a398e8f73f52a7aaa2", + "sha256:adcf8a11b98af9375e32bff91de184f33a68dc48b9cb9becad4f132fa25cfa3c", + "sha256:b65f5d530ba91e49ffc7c589255e878d2506a8b96ffce69d3b7c4500a9a9eaf8", + "sha256:be9423d5a3577ac2e92c7e758bd8a2b205f5e51a012177a590bc46fc51eb4834", + "sha256:c2822111ddc5bcfb116e6c663e403579d0fe3f147d2a97426011a191c43a7458", + "sha256:c6a9a4a31cd6e86d0fbe8473ceed83d4fe760b19d949fb557ef668defafea0f6", + "sha256:d048f93e42ba578b82758c15d8ae037d08e69d91d9872bca5a1895b118f4e2b0", + "sha256:e9c30b311de2513555ab02464ebb76115d242842b29c412f5a9aa0cac57be9f6", + "sha256:ec14ceca36f67cb48b34d02d7f65a9acae15cd72b48e303531893ba4a960f3ea", + "sha256:ef3be705c3eac282a28058e6c6e5503419b250f482320df2172abcbea642c831" + ], + "markers": "python_version >= '3.6'", + "version": "==1.8.5.post1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "smartypants": { + "hashes": [ + "sha256:8db97f7cbdf08d15b158a86037cd9e116b4cf37703d24e0419a0d64ca5808f0d" + ], + "version": "==2.0.1" + }, + "soupsieve": { + "hashes": [ + "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759", + "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d" + ], + "markers": "python_version >= '3.6'", + "version": "==2.3.2.post1" + }, + "sqlalchemy": { + "hashes": [ + "sha256:00dd998b43b282c71de46b061627b5edb9332510eb1edfc5017b9e4356ed44ea", + "sha256:08b47c971327e733ffd6bae2d4f50a7b761793efe69d41067fcba86282819eea", + "sha256:0992f3cc640ec0f88f721e426da884c34ff0a60eb73d3d64172e23dfadfc8a0b", + "sha256:0c956a5d1adb49a35d78ef0fae26717afc48a36262359bb5b0cbd7a3a247c26f", + "sha256:1ab08141d93de83559f6a7d9a962830f918623a885b3759ec2b9d1a531ff28fe", + "sha256:1cf03d37819dc17a388d313919daf32058d19ba1e592efdf14ce8cbd997e6023", + "sha256:2026632051a93997cf8f6fda14360f99230be1725b7ab2ef15be205a4b8a5430", + "sha256:23b693876ac7963b6bc7b1a5f3a2642f38d2624af834faad5933913928089d1b", + "sha256:26ee4dbac5dd7abf18bf3cd8f04e51f72c339caf702f68172d308888cd26c6c9", + "sha256:28b1791a30d62fc104070965f1a2866699c45bbf5adc0be0cf5f22935edcac58", + "sha256:2b64955850a14b9d481c17becf0d3f62fb1bb31ac2c45c2caf5ad06d9e811187", + "sha256:2cf50611ef4221ad587fb7a1708e61ff72966f84330c6317642e08d6db4138fd", + "sha256:44a660506080cc975e1dfa5776fe5f6315ddc626a77b50bf0eee18b0389ea265", + "sha256:4ec440990ab00650d0c7ea2c75bc225087afdd7ddcb248e3d934def4dff62762", + "sha256:63ad778f4e80913fb171247e4fa82123d0068615ae1d51a9791fc4284cb81748", + "sha256:69deec3a94de10062080d91e1ba69595efeafeafe68b996426dec9720031fb25", + "sha256:6b70d02bbe1adbbf715d2249cacf9ac17c6f8d22dfcb3f1a4fbc5bf64364da8a", + "sha256:885e11638946472b4a0a7db8e6df604b2cf64d23dc40eedc3806d869fcb18fae", + "sha256:959bf4390766a8696aa01285016c766b4eb676f712878aac5fce956dd49695d9", + "sha256:9ced2450c9fd016f9232d976661623e54c450679eeefc7aa48a3d29924a63189", + "sha256:a0b9e3d81f86ba04007f0349e373a5b8c81ec2047aadb8d669caf8c54a092461", + "sha256:a62c0ecbb9976550f26f7bf75569f425e661e7249349487f1483115e5fc893a6", + "sha256:b07fc38e6392a65935dc8b486229679142b2ea33c94059366b4d8b56f1e35a97", + "sha256:b41b87b929118838bafc4bb18cf3c5cd1b3be4b61cd9042e75174df79e8ac7a2", + "sha256:b7ccdca6cd167611f4a62a8c2c0c4285c2535640d77108f782ce3f3cccb70f3a", + "sha256:b7ff0a8bf0aec1908b92b8dfa1246128bf4f94adbdd3da6730e9c542e112542d", + "sha256:bb342c0e25cc8f78a0e7c692da3b984f072666b316fbbec2a0e371cb4dfef5f0", + "sha256:bf073c619b5a7f7cd731507d0fdc7329bee14b247a63b0419929e4acd24afea8", + "sha256:c8d974c991eef0cd29418a5957ae544559dc326685a6f26b3a914c87759bf2f4", + "sha256:c9d0f1a9538cc5e75f2ea0cb6c3d70155a1b7f18092c052e0d84105622a41b63", + "sha256:cdee4d475e35684d210dc6b430ff8ca2ed0636378ac19b457e2f6f350d1f5acc", + "sha256:cfa8ab4ba0c97ab6bcae1f0948497d14c11b6c6ecd1b32b8a79546a0823d8211", + "sha256:d259fa08e4b3ed952c01711268bcf6cd2442b0c54866d64aece122f83da77c6d", + "sha256:f2aa85aebc0ef6b342d5d3542f969caa8c6a63c8d36cf5098769158a9fa2123c", + "sha256:fa9e0d7832b7511b3b3fd0e67fac85ff11fd752834c143ca2364c9b778c0485a", + "sha256:fb4edb6c354eac0fcc07cb91797e142f702532dbb16c1d62839d6eec35f814cf" + ], + "index": "pypi", + "version": "==1.4.40" + }, + "statsd": { + "hashes": [ + "sha256:c610fb80347fca0ef62666d241bce64184bd7cc1efe582f9690e045c25535eaa", + "sha256:e3e6db4c246f7c59003e51c9720a51a7f39a396541cb9b147ff4b14d15b5dd1f" + ], + "version": "==3.3.0" + }, + "typing-extensions": { + "hashes": [ + "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", + "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" + ], + "markers": "python_version < '3.10'", + "version": "==4.4.0" + }, + "uri-template": { + "hashes": [ + "sha256:934e4d09d108b70eb8a24410af8615294d09d279ce0e7cbcdaef1bd21f932b06", + "sha256:f1699c77b73b925cf4937eae31ab282a86dc885c333f2e942513f08f691fc7db" + ], + "version": "==1.2.0" + }, + "urllib3": { + "hashes": [ + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", + "version": "==1.26.12" + }, + "vine": { + "hashes": [ + "sha256:4c9dceab6f76ed92105027c49c823800dd33cacce13bdedc5b914e3514b7fb30", + "sha256:7d3b1624a953da82ef63462013bbd271d3eb75751489f9807598e8f340bd637e" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, + "wcwidth": { + "hashes": [ + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" + ], + "version": "==0.2.5" + }, + "webcolors": { + "hashes": [ + "sha256:16d043d3a08fd6a1b1b7e3e9e62640d09790dce80d2bdd4792a175b35fe794a9", + "sha256:d98743d81d498a2d3eaf165196e65481f0d2ea85281463d856b1e51b09f62dce" + ], + "version": "==1.12" + }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, + "werkzeug": { + "hashes": [ + "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", + "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255" + ], + "index": "pypi", + "version": "==2.1.2" + }, + "wrapt": { + "hashes": [ + "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3", + "sha256:01c205616a89d09827986bc4e859bcabd64f5a0662a7fe95e0d359424e0e071b", + "sha256:02b41b633c6261feff8ddd8d11c711df6842aba629fdd3da10249a53211a72c4", + "sha256:07f7a7d0f388028b2df1d916e94bbb40624c59b48ecc6cbc232546706fac74c2", + "sha256:11871514607b15cfeb87c547a49bca19fde402f32e2b1c24a632506c0a756656", + "sha256:1b376b3f4896e7930f1f772ac4b064ac12598d1c38d04907e696cc4d794b43d3", + "sha256:21ac0156c4b089b330b7666db40feee30a5d52634cc4560e1905d6529a3897ff", + "sha256:257fd78c513e0fb5cdbe058c27a0624c9884e735bbd131935fd49e9fe719d310", + "sha256:2b39d38039a1fdad98c87279b48bc5dce2c0ca0d73483b12cb72aa9609278e8a", + "sha256:2cf71233a0ed05ccdabe209c606fe0bac7379fdcf687f39b944420d2a09fdb57", + "sha256:2fe803deacd09a233e4762a1adcea5db5d31e6be577a43352936179d14d90069", + "sha256:3232822c7d98d23895ccc443bbdf57c7412c5a65996c30442ebe6ed3df335383", + "sha256:34aa51c45f28ba7f12accd624225e2b1e5a3a45206aa191f6f9aac931d9d56fe", + "sha256:36f582d0c6bc99d5f39cd3ac2a9062e57f3cf606ade29a0a0d6b323462f4dd87", + "sha256:380a85cf89e0e69b7cfbe2ea9f765f004ff419f34194018a6827ac0e3edfed4d", + "sha256:40e7bc81c9e2b2734ea4bc1aceb8a8f0ceaac7c5299bc5d69e37c44d9081d43b", + "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907", + "sha256:4fcc4649dc762cddacd193e6b55bc02edca674067f5f98166d7713b193932b7f", + "sha256:5a0f54ce2c092aaf439813735584b9537cad479575a09892b8352fea5e988dc0", + "sha256:5a9a0d155deafd9448baff28c08e150d9b24ff010e899311ddd63c45c2445e28", + "sha256:5b02d65b9ccf0ef6c34cba6cf5bf2aab1bb2f49c6090bafeecc9cd81ad4ea1c1", + "sha256:60db23fa423575eeb65ea430cee741acb7c26a1365d103f7b0f6ec412b893853", + "sha256:642c2e7a804fcf18c222e1060df25fc210b9c58db7c91416fb055897fc27e8cc", + "sha256:6a9a25751acb379b466ff6be78a315e2b439d4c94c1e99cb7266d40a537995d3", + "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3", + "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164", + "sha256:6e743de5e9c3d1b7185870f480587b75b1cb604832e380d64f9504a0535912d1", + "sha256:709fe01086a55cf79d20f741f39325018f4df051ef39fe921b1ebe780a66184c", + "sha256:7b7c050ae976e286906dd3f26009e117eb000fb2cf3533398c5ad9ccc86867b1", + "sha256:7d2872609603cb35ca513d7404a94d6d608fc13211563571117046c9d2bcc3d7", + "sha256:7ef58fb89674095bfc57c4069e95d7a31cfdc0939e2a579882ac7d55aadfd2a1", + "sha256:80bb5c256f1415f747011dc3604b59bc1f91c6e7150bd7db03b19170ee06b320", + "sha256:81b19725065dcb43df02b37e03278c011a09e49757287dca60c5aecdd5a0b8ed", + "sha256:833b58d5d0b7e5b9832869f039203389ac7cbf01765639c7309fd50ef619e0b1", + "sha256:88bd7b6bd70a5b6803c1abf6bca012f7ed963e58c68d76ee20b9d751c74a3248", + "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c", + "sha256:8c0ce1e99116d5ab21355d8ebe53d9460366704ea38ae4d9f6933188f327b456", + "sha256:8d649d616e5c6a678b26d15ece345354f7c2286acd6db868e65fcc5ff7c24a77", + "sha256:903500616422a40a98a5a3c4ff4ed9d0066f3b4c951fa286018ecdf0750194ef", + "sha256:9736af4641846491aedb3c3f56b9bc5568d92b0692303b5a305301a95dfd38b1", + "sha256:988635d122aaf2bdcef9e795435662bcd65b02f4f4c1ae37fbee7401c440b3a7", + "sha256:9cca3c2cdadb362116235fdbd411735de4328c61425b0aa9f872fd76d02c4e86", + "sha256:9e0fd32e0148dd5dea6af5fee42beb949098564cc23211a88d799e434255a1f4", + "sha256:9f3e6f9e05148ff90002b884fbc2a86bd303ae847e472f44ecc06c2cd2fcdb2d", + "sha256:a85d2b46be66a71bedde836d9e41859879cc54a2a04fad1191eb50c2066f6e9d", + "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8", + "sha256:aa31fdcc33fef9eb2552cbcbfee7773d5a6792c137b359e82879c101e98584c5", + "sha256:b014c23646a467558be7da3d6b9fa409b2c567d2110599b7cf9a0c5992b3b471", + "sha256:b21bb4c09ffabfa0e85e3a6b623e19b80e7acd709b9f91452b8297ace2a8ab00", + "sha256:b5901a312f4d14c59918c221323068fad0540e34324925c8475263841dbdfe68", + "sha256:b9b7a708dd92306328117d8c4b62e2194d00c365f18eff11a9b53c6f923b01e3", + "sha256:d1967f46ea8f2db647c786e78d8cc7e4313dbd1b0aca360592d8027b8508e24d", + "sha256:d52a25136894c63de15a35bc0bdc5adb4b0e173b9c0d07a2be9d3ca64a332735", + "sha256:d77c85fedff92cf788face9bfa3ebaa364448ebb1d765302e9af11bf449ca36d", + "sha256:d79d7d5dc8a32b7093e81e97dad755127ff77bcc899e845f41bf71747af0c569", + "sha256:dbcda74c67263139358f4d188ae5faae95c30929281bc6866d00573783c422b7", + "sha256:ddaea91abf8b0d13443f6dac52e89051a5063c7d014710dcb4d4abb2ff811a59", + "sha256:dee0ce50c6a2dd9056c20db781e9c1cfd33e77d2d569f5d1d9321c641bb903d5", + "sha256:dee60e1de1898bde3b238f18340eec6148986da0455d8ba7848d50470a7a32fb", + "sha256:e2f83e18fe2f4c9e7db597e988f72712c0c3676d337d8b101f6758107c42425b", + "sha256:e3fb1677c720409d5f671e39bac6c9e0e422584e5f518bfd50aa4cbbea02433f", + "sha256:ee2b1b1769f6707a8a445162ea16dddf74285c3964f605877a20e38545c3c462", + "sha256:ee6acae74a2b91865910eef5e7de37dc6895ad96fa23603d1d27ea69df545015", + "sha256:ef3f72c9666bba2bab70d2a8b79f2c6d2c1a42a7f7e2b0ec83bb2f9e383950af" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.14.1" + }, + "zipp": { + "hashes": [ + "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb", + "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980" + ], + "markers": "python_version >= '3.7'", + "version": "==3.9.0" + } + }, + "develop": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "index": "pypi", + "version": "==21.4.0" + }, + "boto3": { + "hashes": [ + "sha256:15733c2bbedce7a36fcf1749560c72c3ee90785aa6302a98658c7bffdcbe1f2a", + "sha256:ea8ebcea4ccb70d1cf57526d9eec6012c76796f28ada3e9cc1d89178683d8107" + ], + "index": "pypi", + "version": "==1.23.8" + }, + "botocore": { + "hashes": [ + "sha256:620851daf1245af5bc28137aa821375bac964aa0eddc482437c783fe01e298fc", + "sha256:e786722cb14de7319331cc55e9092174de66a768559700ef656d05ff41b3e24f" + ], + "index": "pypi", + "version": "==1.26.8" + }, + "certifi": { + "hashes": [ + "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", + "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + ], + "index": "pypi", + "version": "==2022.5.18.1" + }, + "cffi": { + "hashes": [ + "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3", + "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2", + "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636", + "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20", + "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728", + "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27", + "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66", + "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443", + "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0", + "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7", + "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39", + "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605", + "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a", + "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37", + "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029", + "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139", + "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc", + "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df", + "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14", + "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880", + "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2", + "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a", + "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e", + "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474", + "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024", + "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8", + "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0", + "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e", + "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a", + "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e", + "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032", + "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6", + "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e", + "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b", + "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e", + "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954", + "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962", + "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c", + "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4", + "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55", + "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962", + "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023", + "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c", + "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6", + "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8", + "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382", + "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7", + "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc", + "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997", + "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796" + ], + "index": "pypi", + "version": "==1.15.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + ], + "index": "pypi", + "version": "==2.0.12" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79", + "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a", + "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f", + "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a", + "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa", + "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398", + "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba", + "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d", + "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf", + "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b", + "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518", + "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d", + "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795", + "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2", + "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e", + "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32", + "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745", + "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b", + "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e", + "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d", + "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f", + "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660", + "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62", + "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6", + "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04", + "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c", + "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5", + "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef", + "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc", + "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae", + "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578", + "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466", + "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4", + "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91", + "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0", + "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4", + "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b", + "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe", + "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b", + "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75", + "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b", + "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c", + "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72", + "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b", + "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f", + "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e", + "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53", + "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3", + "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84", + "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987" + ], + "markers": "python_version >= '3.7'", + "version": "==6.5.0" + }, + "cryptography": { + "hashes": [ + "sha256:0297ffc478bdd237f5ca3a7dc96fc0d315670bfa099c04dc3a4a2172008a405a", + "sha256:10d1f29d6292fc95acb597bacefd5b9e812099d75a6469004fd38ba5471a977f", + "sha256:16fa61e7481f4b77ef53991075de29fc5bacb582a1244046d2e8b4bb72ef66d0", + "sha256:194044c6b89a2f9f169df475cc167f6157eb9151cc69af8a2a163481d45cc407", + "sha256:1db3d807a14931fa317f96435695d9ec386be7b84b618cc61cfa5d08b0ae33d7", + "sha256:3261725c0ef84e7592597606f6583385fed2a5ec3909f43bc475ade9729a41d6", + "sha256:3b72c360427889b40f36dc214630e688c2fe03e16c162ef0aa41da7ab1455153", + "sha256:3e3a2599e640927089f932295a9a247fc40a5bdf69b0484532f530471a382750", + "sha256:3fc26e22840b77326a764ceb5f02ca2d342305fba08f002a8c1f139540cdfaad", + "sha256:5067ee7f2bce36b11d0e334abcd1ccf8c541fc0bbdaf57cdd511fdee53e879b6", + "sha256:52e7bee800ec869b4031093875279f1ff2ed12c1e2f74923e8f49c916afd1d3b", + "sha256:64760ba5331e3f1794d0bcaabc0d0c39e8c60bf67d09c93dc0e54189dfd7cfe5", + "sha256:765fa194a0f3372d83005ab83ab35d7c5526c4e22951e46059b8ac678b44fa5a", + "sha256:79473cf8a5cbc471979bd9378c9f425384980fcf2ab6534b18ed7d0d9843987d", + "sha256:896dd3a66959d3a5ddcfc140a53391f69ff1e8f25d93f0e2e7830c6de90ceb9d", + "sha256:89ed49784ba88c221756ff4d4755dbc03b3c8d2c5103f6d6b4f83a0fb1e85294", + "sha256:ac7e48f7e7261207d750fa7e55eac2d45f720027d5703cd9007e9b37bbb59ac0", + "sha256:ad7353f6ddf285aeadfaf79e5a6829110106ff8189391704c1d8801aa0bae45a", + "sha256:b0163a849b6f315bf52815e238bc2b2346604413fa7c1601eea84bcddb5fb9ac", + "sha256:b6c9b706316d7b5a137c35e14f4103e2115b088c412140fdbd5f87c73284df61", + "sha256:c2e5856248a416767322c8668ef1845ad46ee62629266f84a8f007a317141013", + "sha256:ca9f6784ea96b55ff41708b92c3f6aeaebde4c560308e5fbbd3173fbc466e94e", + "sha256:d1a5bd52d684e49a36582193e0b89ff267704cd4025abefb9e26803adeb3e5fb", + "sha256:d3971e2749a723e9084dd507584e2a2761f78ad2c638aa31e80bc7a15c9db4f9", + "sha256:d4ef6cc305394ed669d4d9eebf10d3a101059bdcf2669c366ec1d14e4fb227bd", + "sha256:d9e69ae01f99abe6ad646947bba8941e896cb3aa805be2597a0400e0764b5818" + ], + "markers": "python_version >= '3.6'", + "version": "==38.0.1" + }, + "execnet": { + "hashes": [ + "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5", + "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.9.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "flake8-bugbear": { + "hashes": [ + "sha256:ec374101cddf65bd7a96d393847d74e58d3b98669dbf9768344c39b6290e8bd6", + "sha256:f7c080563fca75ee6b205d06b181ecba22b802babb96b0b084cc7743d6908a55" + ], + "index": "pypi", + "version": "==22.4.25" + }, + "freezegun": { + "hashes": [ + "sha256:15103a67dfa868ad809a8f508146e396be2995172d25f927e48ce51c0bf5cb09", + "sha256:b4c64efb275e6bc68dc6e771b17ffe0ff0f90b81a2a5189043550b6519926ba4" + ], + "index": "pypi", + "version": "==1.2.1" + }, + "idna": { + "hashes": [ + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + ], + "version": "==3.4" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "isort": { + "hashes": [ + "sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7", + "sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951" + ], + "index": "pypi", + "version": "==5.10.1" + }, + "jinja2": { + "hashes": [ + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.2" + }, + "jinja2-cli": { + "extras": [ + "yaml" + ], + "hashes": [ + "sha256:a16bb1454111128e206f568c95938cdef5b5a139929378f72bb8cf6179e18e50", + "sha256:b91715c79496beaddad790171e7258a87db21c1a0b6d2b15bca3ba44b74aac5d" + ], + "index": "pypi", + "version": "==0.8.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "markupsafe": { + "hashes": [ + "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", + "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88", + "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5", + "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7", + "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a", + "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603", + "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1", + "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135", + "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247", + "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6", + "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601", + "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77", + "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02", + "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e", + "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63", + "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f", + "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980", + "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b", + "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812", + "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff", + "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96", + "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1", + "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925", + "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a", + "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6", + "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e", + "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f", + "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4", + "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f", + "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3", + "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c", + "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a", + "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417", + "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a", + "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a", + "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37", + "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452", + "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933", + "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", + "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "moto": { + "hashes": [ + "sha256:8928ec168e5fd88b1127413b2fa570a80d45f25182cdad793edd208d07825269", + "sha256:ba683e70950b6579189bc12d74c1477aa036c090c6ad8b151a22f5896c005113" + ], + "index": "pypi", + "version": "==3.1.9" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pytest": { + "hashes": [ + "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", + "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" + ], + "index": "pypi", + "version": "==7.1.2" + }, + "pytest-cov": { + "hashes": [ + "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "pytest-env": { + "hashes": [ + "sha256:7e94956aef7f2764f3c147d216ce066bf6c42948bb9e293169b1b1c880a580c2" + ], + "index": "pypi", + "version": "==0.6.2" + }, + "pytest-forked": { + "hashes": [ + "sha256:8b67587c8f98cbbadfdd804539ed5455b6ed03802203485dd2f53c1422d7440e", + "sha256:bbbb6717efc886b9d64537b41fb1497cfaf3c9601276be8da2cccfea5a3c8ad8" + ], + "markers": "python_version >= '3.6'", + "version": "==1.4.0" + }, + "pytest-mock": { + "hashes": [ + "sha256:5112bd92cc9f186ee96e1a92efc84969ea494939c3aead39c50f421c4cc69534", + "sha256:6cff27cec936bf81dc5ee87f07132b807bcda51106b5ec4b90a04331cba76231" + ], + "index": "pypi", + "version": "==3.7.0" + }, + "pytest-xdist": { + "hashes": [ + "sha256:4580deca3ff04ddb2ac53eba39d76cb5dd5edeac050cb6fbc768b0dd712b4edf", + "sha256:6fe5c74fec98906deb8f2d2b616b5c782022744978e7bd4695d39c8f42d0ce65" + ], + "index": "pypi", + "version": "==2.5.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:335ab46900b1465e714b4fda4963d87363264eb662aab5e65da039c25f1f5b22", + "sha256:c4d88f472f54d615e9cd582a5004d1e5f624854a6a27a6211591c251f22a6914" + ], + "version": "==2022.5" + }, + "pyyaml": { + "hashes": [ + "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf", + "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696", + "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393", + "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77", + "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922", + "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5", + "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8", + "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10", + "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc", + "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018", + "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e", + "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253", + "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347", + "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183", + "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541", + "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb", + "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185", + "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc", + "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db", + "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa", + "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46", + "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122", + "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b", + "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63", + "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df", + "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc", + "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247", + "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6", + "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==5.4.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", + "version": "==2.28.1" + }, + "requests-mock": { + "hashes": [ + "sha256:0a2d38a117c08bb78939ec163522976ad59a6b7fdd82b709e23bb98004a44970", + "sha256:8d72abe54546c1fc9696fa1516672f1031d72a55a1d66c85184f972a24ba0eba" + ], + "index": "pypi", + "version": "==1.9.3" + }, + "responses": { + "hashes": [ + "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e", + "sha256:dcf294d204d14c436fddcc74caefdbc5764795a40ff4e6a7740ed8ddbf3294be" + ], + "markers": "python_version >= '3.7'", + "version": "==0.22.0" + }, + "s3transfer": { + "hashes": [ + "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971", + "sha256:95c58c194ce657a5f4fb0b9e60a84968c808888aed628cd98ab8771fe1db98ed" + ], + "markers": "python_version >= '3.6'", + "version": "==0.5.2" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.1" + }, + "types-toml": { + "hashes": [ + "sha256:8300fd093e5829eb9c1fba69cee38130347d4b74ddf32d0a7df650ae55c2b599", + "sha256:b7e7ea572308b1030dc86c3ba825c5210814c2825612ec679eb7814f8dd9295a" + ], + "version": "==0.10.8" + }, + "urllib3": { + "hashes": [ + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", + "version": "==1.26.12" + }, + "werkzeug": { + "hashes": [ + "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", + "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255" + ], + "index": "pypi", + "version": "==2.1.2" + }, + "xmltodict": { + "hashes": [ + "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", + "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + ], + "markers": "python_version >= '3.4'", + "version": "==0.13.0" + } + } +} diff --git a/requirements.in b/requirements.in index fc1679f3f..dbcc0f8a6 100644 --- a/requirements.in +++ b/requirements.in @@ -32,8 +32,8 @@ notifications-python-client==6.3.0 # PaaS awscli-cwlogs==1.4.6 -notifications-utils @ git+https://github.com/GSA/notifications-utils.git +notifications-utils @ git+https://github.com/GSA/notifications-utils.git#egg=notifications-utils # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains prometheus-client==0.14.1 -git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72 +git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72#egg=gds-metrics From e58d0eb552f76dedf5e6fca739611be25289e20a Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Thu, 20 Oct 2022 20:49:49 +0000 Subject: [PATCH 49/65] replace overeager deletion --- app/schemas.py | 6 ++++++ app/template/rest.py | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/app/schemas.py b/app/schemas.py index 6ea50d16f..1317ab33d 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -418,6 +418,12 @@ class TemplateSchemaNoDetail(TemplateSchema): 'version', ) + @pre_dump + def remove_content_for_non_broadcast_templates(self, template, **kwargs): + template.content = None + + return template + class TemplateHistorySchema(BaseSchema): diff --git a/app/template/rest.py b/app/template/rest.py index cf313d297..271458f76 100644 --- a/app/template/rest.py +++ b/app/template/rest.py @@ -166,7 +166,6 @@ def get_all_templates_for_service(service_id): data = template_schema.dump(templates, many=True) else: data = template_schema_no_detail.dump(templates, many=True) - print(data) return jsonify(data=data) From 9f37592b1e1aff771e3ca4acec53748ef8883977 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 21 Oct 2022 00:26:37 +0000 Subject: [PATCH 50/65] cleaner flake8 cleaning --- app/celery/process_ses_receipts_tasks.py | 12 ++++++------ app/dao/services_dao.py | 4 +--- app/inbound_sms/rest.py | 2 -- app/notifications/notifications_sms_callback.py | 6 +----- tests/app/celery/test_process_ses_receipts_tasks.py | 5 +---- 5 files changed, 9 insertions(+), 20 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 232dfafaf..7a35f874a 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -52,16 +52,16 @@ def process_ses_results(self, response): message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): current_app.logger.info( - f"notification not found for reference: {reference} \ - (while attempting update to {notification_status}). " - f"Callback may have arrived before notification was \ - persisted to the DB. Adding task to retry queue" + f"notification not found for reference: {reference}" + f"(while attempting update to {notification_status}). " + f"Callback may have arrived before notification was" + f"persisted to the DB. Adding task to retry queue" ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - f"notification not found for reference: {reference} \ - (while attempting update to {notification_status})" + f"notification not found for reference: {reference} " + f"(while attempting update to {notification_status})" ) return diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 5a1540895..ae5fb2a88 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -9,8 +9,6 @@ from sqlalchemy.sql.expression import and_, asc, case, func from app import db from app.dao.dao_utils import VersionOptions, autocommit, version_class from app.dao.date_util import get_current_financial_year -# from app.dao.email_branding_dao import dao_get_email_branding_by_name -# from app.dao.letter_branding_dao import dao_get_letter_branding_by_name from app.dao.organisation_dao import dao_get_organisation_by_email_address from app.dao.service_sms_sender_dao import insert_service_sms_sender from app.dao.service_user_dao import dao_get_service_user @@ -46,7 +44,7 @@ from app.models import ( User, VerifyCode, ) -from app.utils import ( # email_address_is_nhs, +from app.utils import ( escape_special_characters, get_archived_db_column_value, get_london_midnight_in_utc, diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py index 1bc3dec07..a0439196c 100644 --- a/app/inbound_sms/rest.py +++ b/app/inbound_sms/rest.py @@ -15,8 +15,6 @@ from app.inbound_sms.inbound_sms_schemas import ( ) from app.schema_validation import validate -# from notifications_utils.recipients import try_validate_and_format_phone_number - inbound_sms = Blueprint( 'inbound_sms', diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py index b585e3a20..4448191d2 100644 --- a/app/notifications/notifications_sms_callback.py +++ b/app/notifications/notifications_sms_callback.py @@ -1,9 +1,5 @@ -from flask import Blueprint # , json, jsonify, request +from flask import Blueprint -# from app.celery.process_sms_client_response_tasks import ( -# process_sms_client_response, -# ) -# from app.config import QueueNames from app.errors import register_errors sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms") diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index fcd258926..896ddc079 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -197,11 +197,8 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, with freeze_time('2017-11-17T12:34:03.646Z'): assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 - # the multiline indent must be the same as in the application code - # for the assertion to completely match mock_logger.assert_called_once_with( - 'notification not found for reference: ref \ - (while attempting update to delivered)' + 'notification not found for reference: ref (while attempting update to delivered)' ) From 8e2b8dd7c436cdf32f2963f5e15d71b9eccdc883 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 21 Oct 2022 13:29:52 +0000 Subject: [PATCH 51/65] keep on flakin the flake world --- app/celery/process_ses_receipts_tasks.py | 14 +++++++------- app/inbound_sms/rest.py | 1 - app/notifications/receive_notifications.py | 4 ++-- app/notifications/sns_handlers.py | 8 ++++---- .../app/celery/test_process_ses_receipts_tasks.py | 2 +- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 7a35f874a..8a0b3417f 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -30,8 +30,8 @@ def process_ses_results(self, response): notification_type = ses_message["notificationType"] # TODO remove after smoke testing on prod is implemented current_app.logger.info( - f"Attempting to process SES delivery status message \ - from SNS with type: {notification_type} and body: {ses_message}" + f"Attempting to process SES delivery status message " + f"from SNS with type: {notification_type} and body: {ses_message}" ) bounce_message = None @@ -52,16 +52,16 @@ def process_ses_results(self, response): message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None) if datetime.utcnow() - message_time < timedelta(minutes=5): current_app.logger.info( - f"notification not found for reference: {reference}" - f"(while attempting update to {notification_status}). " + f"Notification not found for reference: {reference}" + f"(while attempting update to {notification_status}). " f"Callback may have arrived before notification was" - f"persisted to the DB. Adding task to retry queue" + f"persisted to the DB. Adding task to retry queue" ) self.retry(queue=QueueNames.RETRY) else: current_app.logger.warning( - f"notification not found for reference: {reference} " - f"(while attempting update to {notification_status})" + f"Notification not found for reference: {reference} " + f"(while attempting update to {notification_status})" ) return diff --git a/app/inbound_sms/rest.py b/app/inbound_sms/rest.py index a0439196c..7a82b8920 100644 --- a/app/inbound_sms/rest.py +++ b/app/inbound_sms/rest.py @@ -15,7 +15,6 @@ from app.inbound_sms.inbound_sms_schemas import ( ) from app.schema_validation import validate - inbound_sms = Blueprint( 'inbound_sms', __name__, diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index 8b93b935e..03009a791 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -61,8 +61,8 @@ def receive_sns_sms(): # since this is an issue with our service <-> number mapping, or no inbound_sms service permission # we should still tell SNS that we received it successfully current_app.logger.warning( - f"Mapping between service and inbound number: {inbound_number} is broken, \ - or service does not have permission to receive inbound sms" + f"Mapping between service and inbound number: {inbound_number} is broken, " + f"or service does not have permission to receive inbound sms" ) return jsonify( result="success", message="SMS-SNS callback succeeded" diff --git a/app/notifications/sns_handlers.py b/app/notifications/sns_handlers.py index dcb6a7e3f..535ec12db 100644 --- a/app/notifications/sns_handlers.py +++ b/app/notifications/sns_handlers.py @@ -58,12 +58,12 @@ def sns_notification_handler(data, headers): response.raise_for_status() except Exception as e: current_app.logger.warning( - f"Attempt to raise_for_status()SubscriptionConfirmation Type \ - message files for response: {response.text} with error {e}" + f"Attempt to raise_for_status()SubscriptionConfirmation Type " + f"message files for response: {response.text} with error {e}" ) raise InvalidRequest( - "SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation \ - Type message failed", 400 + "SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation " + "Type message failed", 400 ) current_app.logger.info("SES-SNS auto-confirm subscription callback succeeded") return message diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index 896ddc079..a90f9f6f8 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -198,7 +198,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 mock_logger.assert_called_once_with( - 'notification not found for reference: ref (while attempting update to delivered)' + 'Notification not found for reference: ref (while attempting update to delivered)' ) From 4737cefb1c4e411272db4696887cc9db3d57fd4c Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 25 Oct 2022 11:52:56 -0400 Subject: [PATCH 52/65] broadcast migrations: replace older ones & add final removal --- .../versions/0322_broadcast_service_perm.py | 5 +- migrations/versions/0323_broadcast_message.py | 72 ++++++++- migrations/versions/0326_broadcast_event.py | 26 +++- .../versions/0329_purge_broadcast_data.py | 2 +- .../versions/0330_broadcast_invite_email.py | 52 ++++++- migrations/versions/0331_add_broadcast_org.py | 61 +++++++- .../versions/0332_broadcast_provider_msg.py | 24 ++- .../0333_service_broadcast_provider.py | 11 +- .../versions/0334_broadcast_message_number.py | 17 ++- .../versions/0335_broadcast_msg_content.py | 10 +- .../versions/0336_broadcast_msg_content_2.py | 20 ++- migrations/versions/0337_broadcast_msg_api.py | 9 +- .../versions/0340_stub_training_broadcasts.py | 4 +- .../0342_service_broadcast_settings.py | 29 +++- .../versions/0344_stubbed_not_nullable.py | 15 +- .../versions/0345_move_broadcast_provider.py | 22 ++- ...ast_settings_migrate_broadcast_settings.py | 30 +++- .../versions/0352_broadcast_provider_types.py | 16 +- .../0353_broadcast_provider_not_null.py | 6 +- .../versions/0354_government_channel.py | 7 +- migrations/versions/0358_operator_channel.py | 7 +- migrations/versions/0359_more_permissions.py | 34 ++++- .../versions/0362_broadcast_msg_event.py | 4 +- .../versions/0363_cancelled_by_api_key.py | 26 +++- migrations/versions/0364_drop_old_column.py | 16 +- migrations/versions/0379_remove_broadcasts.py | 144 ++++++++++++++++++ 26 files changed, 616 insertions(+), 53 deletions(-) create mode 100644 migrations/versions/0379_remove_broadcasts.py diff --git a/migrations/versions/0322_broadcast_service_perm.py b/migrations/versions/0322_broadcast_service_perm.py index 4a0385bad..2819dd8bb 100644 --- a/migrations/versions/0322_broadcast_service_perm.py +++ b/migrations/versions/0322_broadcast_service_perm.py @@ -13,8 +13,9 @@ down_revision = '0321_drop_postage_constraints' def upgrade(): - pass + op.execute("INSERT INTO service_permission_types VALUES ('broadcast')") def downgrade(): - pass \ No newline at end of file + op.execute("DELETE FROM service_permissions WHERE permission = 'broadcast'") + op.execute("DELETE FROM service_permission_types WHERE name = 'broadcast'") diff --git a/migrations/versions/0323_broadcast_message.py b/migrations/versions/0323_broadcast_message.py index fdfc50750..03aecb0c8 100644 --- a/migrations/versions/0323_broadcast_message.py +++ b/migrations/versions/0323_broadcast_message.py @@ -14,6 +14,16 @@ revision = '0323_broadcast_message' down_revision = '0322_broadcast_service_perm' +name = 'template_type' +tmp_name = 'tmp_' + name + +old_options = ('sms', 'email', 'letter') +new_options = old_options + ('broadcast',) + +new_type = sa.Enum(*new_options, name=name) +old_type = sa.Enum(*old_options, name=name) + + STATUSES = [ 'draft', 'pending-approval', @@ -26,8 +36,66 @@ STATUSES = [ def upgrade(): - pass + op.execute(f'ALTER TYPE {name} RENAME TO {tmp_name}') + new_type.create(op.get_bind()) + + for table in ['templates', 'templates_history', 'service_contact_list']: + op.execute(f'ALTER TABLE {table} ALTER COLUMN template_type TYPE {name} USING template_type::text::{name}') + + op.execute(f'DROP TYPE {tmp_name}') + + broadcast_status_type = op.create_table( + 'broadcast_status_type', + sa.Column('name', sa.String(), nullable=False), + sa.PrimaryKeyConstraint('name') + ) + op.bulk_insert(broadcast_status_type, [{'name': state} for state in STATUSES]) + + op.create_table( + 'broadcast_message', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True)), + sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('template_version', sa.Integer(), nullable=False), + sa.Column('_personalisation', sa.String()), + sa.Column('areas', postgresql.JSONB(none_as_null=True, astext_type=sa.Text())), + sa.Column('status', sa.String()), + sa.Column('starts_at', sa.DateTime()), + sa.Column('finishes_at', sa.DateTime()), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('approved_at', sa.DateTime()), + sa.Column('cancelled_at', sa.DateTime()), + sa.Column('updated_at', sa.DateTime()), + sa.Column('created_by_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('approved_by_id', postgresql.UUID(as_uuid=True)), + sa.Column('cancelled_by_id', postgresql.UUID(as_uuid=True)), + + sa.ForeignKeyConstraint(['approved_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['cancelled_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], ), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.ForeignKeyConstraint(['template_id', 'template_version'], ['templates_history.id', 'templates_history.version'], ), + sa.PrimaryKeyConstraint('id') + ) + + op.add_column('templates', sa.Column('broadcast_data', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()))) + op.add_column('templates_history', sa.Column('broadcast_data', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()))) def downgrade(): - pass + op.execute("DELETE FROM template_folder_map WHERE template_id IN (SELECT id FROM templates WHERE template_type = 'broadcast')") + op.execute("DELETE FROM template_redacted WHERE template_id IN (SELECT id FROM templates WHERE template_type = 'broadcast')") + op.execute("DELETE FROM templates WHERE template_type = 'broadcast'") + op.execute("DELETE FROM templates_history WHERE template_type = 'broadcast'") + + op.execute(f'ALTER TYPE {name} RENAME TO {tmp_name}') + old_type.create(op.get_bind()) + + for table in ['templates', 'templates_history', 'service_contact_list']: + op.execute(f'ALTER TABLE {table} ALTER COLUMN template_type TYPE {name} USING template_type::text::{name}') + op.execute(f'DROP TYPE {tmp_name}') + + op.drop_column('templates_history', 'broadcast_data') + op.drop_column('templates', 'broadcast_data') + op.drop_table('broadcast_message') + op.drop_table('broadcast_status_type') diff --git a/migrations/versions/0326_broadcast_event.py b/migrations/versions/0326_broadcast_event.py index 5ebb1a008..46cdb258f 100644 --- a/migrations/versions/0326_broadcast_event.py +++ b/migrations/versions/0326_broadcast_event.py @@ -14,8 +14,30 @@ down_revision = '0325_int_letter_rates_fix' def upgrade(): - pass + op.create_table('broadcast_event', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('broadcast_message_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('sent_at', sa.DateTime(), nullable=False), + sa.Column('message_type', sa.String(), nullable=False), + sa.Column('transmitted_content', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), nullable=True), + sa.Column('transmitted_areas', postgresql.JSONB(none_as_null=True, astext_type=sa.Text()), nullable=False), + sa.Column('transmitted_sender', sa.String(), nullable=False), + sa.Column('transmitted_starts_at', sa.DateTime(), nullable=True), + sa.Column('transmitted_finishes_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['broadcast_message_id'], ['broadcast_message.id'], ), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # this shouldn't be nullable. it defaults to `[]` in python. + op.alter_column('broadcast_message', 'areas', existing_type=postgresql.JSONB(astext_type=sa.Text()), nullable=False) + # this can't be nullable. it defaults to 'draft' in python. + op.alter_column('broadcast_message', 'status', existing_type=sa.VARCHAR(), nullable=False) + op.create_foreign_key(None, 'broadcast_message', 'broadcast_status_type', ['status'], ['name']) def downgrade(): - pass + op.drop_constraint('broadcast_message_status_fkey', 'broadcast_message', type_='foreignkey') + op.alter_column('broadcast_message', 'status', existing_type=sa.VARCHAR(), nullable=True) + op.alter_column('broadcast_message', 'areas', existing_type=postgresql.JSONB(astext_type=sa.Text()), nullable=True) + op.drop_table('broadcast_event') diff --git a/migrations/versions/0329_purge_broadcast_data.py b/migrations/versions/0329_purge_broadcast_data.py index ed3507e33..b8c698c53 100644 --- a/migrations/versions/0329_purge_broadcast_data.py +++ b/migrations/versions/0329_purge_broadcast_data.py @@ -14,7 +14,7 @@ down_revision = '0328_international_letters_perm' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.execute("TRUNCATE broadcast_event, broadcast_message;") # ### end Alembic commands ### diff --git a/migrations/versions/0330_broadcast_invite_email.py b/migrations/versions/0330_broadcast_invite_email.py index eb50bbdd4..2127f8832 100644 --- a/migrations/versions/0330_broadcast_invite_email.py +++ b/migrations/versions/0330_broadcast_invite_email.py @@ -18,9 +18,57 @@ user_id = '6af522d0-2915-4e52-83a3-3690455a5fe6' service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' template_id = '46152f7c-6901-41d5-8590-a5624d0d4359' +broadcast_invitation_template_name = 'Notify broadcast invitation email' +broadcast_invitation_subject = "((user_name)) has invited you to join ((service_name)) on GOV.UK Notify" +broadcast_invitation_content = """((user_name)) has invited you to join ((service_name)) on GOV.UK Notify. + +In an emergency, use Notify to broadcast an alert, warning the public about an imminent risk to life. + +Use this link to join the team: +((url)) + +This invitation will stop working at midnight tomorrow. This is to keep ((service_name)) secure. + +Thanks + +GOV.​UK Notify team +https://www.gov.uk/notify +""" + + def upgrade(): - pass + insert_query = """ + INSERT INTO {} + (id, name, template_type, created_at, content, archived, service_id, + subject, created_by_id, version, process_type, hidden) + VALUES + ('{}', '{}', 'email', '{}', '{}', False, '{}', '{}', '{}', 1, 'normal', False) + """ + + op.execute(insert_query.format( + 'templates_history', + template_id, + broadcast_invitation_template_name, + datetime.utcnow(), + broadcast_invitation_content, + service_id, + broadcast_invitation_subject, + user_id + )) + + op.execute(insert_query.format( + 'templates', + template_id, + broadcast_invitation_template_name, + datetime.utcnow(), + broadcast_invitation_content, + service_id, + broadcast_invitation_subject, + user_id + )) def downgrade(): - pass + op.get_bind() + op.execute("delete from templates where id = '{}'".format(template_id)) + op.execute("delete from templates_history where id = '{}'".format(template_id)) diff --git a/migrations/versions/0331_add_broadcast_org.py b/migrations/versions/0331_add_broadcast_org.py index 9ffc3e2f2..2952c10ad 100644 --- a/migrations/versions/0331_add_broadcast_org.py +++ b/migrations/versions/0331_add_broadcast_org.py @@ -18,7 +18,64 @@ organisation_id = '38e4bf69-93b0-445d-acee-53ea53fe02df' def upgrade(): - pass + # we've already done this manually on production + if environment != "production": + insert_sql = """ + INSERT INTO organisation + ( + id, + name, + active, + created_at, + agreement_signed, + crown, + organisation_type + ) + VALUES ( + :id, + :name, + :active, + current_timestamp, + :agreement_signed, + :crown, + :organisation_type + ) + """ + update_service_set_broadcast_org_sql = """ + UPDATE services + SET organisation_id = :organisation_id + WHERE id in ( + SELECT service_id + FROM service_permissions + WHERE permission = 'broadcast' + ) + """ + conn = op.get_bind() + conn.execute( + sa.text(insert_sql), + id=organisation_id, + name=f'Broadcast Services ({environment})', + active=True, + agreement_signed=None, + crown=None, + organisation_type='central', + ) + conn.execute( + sa.text(update_service_set_broadcast_org_sql), + organisation_id=organisation_id + ) + def downgrade(): - pass + update_service_remove_org_sql = """ + UPDATE services + SET organisation_id = NULL, updated_at = current_timestamp + WHERE organisation_id = :organisation_id + """ + delete_sql = """ + DELETE FROM organisation + WHERE id = :organisation_id + """ + conn = op.get_bind() + conn.execute(sa.text(update_service_remove_org_sql), organisation_id=organisation_id) + conn.execute(sa.text(delete_sql), organisation_id=organisation_id) diff --git a/migrations/versions/0332_broadcast_provider_msg.py b/migrations/versions/0332_broadcast_provider_msg.py index 0e3539e5d..088f1c9df 100644 --- a/migrations/versions/0332_broadcast_provider_msg.py +++ b/migrations/versions/0332_broadcast_provider_msg.py @@ -22,8 +22,28 @@ STATUSES = [ def upgrade(): - pass + broadcast_provider_message_status_type = op.create_table( + 'broadcast_provider_message_status_type', + sa.Column('name', sa.String(), nullable=False), + sa.PrimaryKeyConstraint('name') + ) + op.bulk_insert(broadcast_provider_message_status_type, [{'name': status} for status in STATUSES]) + + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'broadcast_provider_message', + sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('broadcast_event_id', postgresql.UUID(as_uuid=True), nullable=True), + sa.Column('provider', sa.String(), nullable=True), + sa.Column('status', sa.String(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['broadcast_event_id'], ['broadcast_event.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('broadcast_event_id', 'provider') + ) def downgrade(): - pass \ No newline at end of file + op.drop_table('broadcast_provider_message') + op.drop_table('broadcast_provider_message_status_type') diff --git a/migrations/versions/0333_service_broadcast_provider.py b/migrations/versions/0333_service_broadcast_provider.py index 2de345328..3c8d3fa94 100644 --- a/migrations/versions/0333_service_broadcast_provider.py +++ b/migrations/versions/0333_service_broadcast_provider.py @@ -14,8 +14,15 @@ down_revision = '0332_broadcast_provider_msg' def upgrade(): - pass + op.create_table( + 'service_broadcast_provider_restriction', + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('provider', sa.String(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('service_id') + ) def downgrade(): - pass + op.drop_table('service_broadcast_provider_restriction') diff --git a/migrations/versions/0334_broadcast_message_number.py b/migrations/versions/0334_broadcast_message_number.py index 0440640de..db8360f98 100644 --- a/migrations/versions/0334_broadcast_message_number.py +++ b/migrations/versions/0334_broadcast_message_number.py @@ -15,11 +15,24 @@ down_revision = '0333_service_broadcast_provider' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.execute("create sequence broadcast_provider_message_number_seq") + op.create_table( + 'broadcast_provider_message_number', + sa.Column( + 'broadcast_provider_message_number', + sa.Integer(), + server_default=sa.text("nextval('broadcast_provider_message_number_seq')"), + nullable=False + ), + sa.Column('broadcast_provider_message_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint(['broadcast_provider_message_id'], ['broadcast_provider_message.id'], ), + sa.PrimaryKeyConstraint('broadcast_provider_message_number') + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.drop_table('broadcast_provider_message_number') + op.execute("drop sequence broadcast_provider_message_number_seq") # ### end Alembic commands ### diff --git a/migrations/versions/0335_broadcast_msg_content.py b/migrations/versions/0335_broadcast_msg_content.py index 8b021fc35..a96155702 100644 --- a/migrations/versions/0335_broadcast_msg_content.py +++ b/migrations/versions/0335_broadcast_msg_content.py @@ -14,8 +14,14 @@ down_revision = '0334_broadcast_message_number' def upgrade(): - pass + op.add_column('broadcast_message', sa.Column('content', sa.Text(), nullable=True)) + op.alter_column('broadcast_message', 'template_id', nullable=True) + op.alter_column('broadcast_message', 'template_version', nullable=True) def downgrade(): - pass + # downgrade fails if there are broadcasts without a template. This is deliberate cos I don't feel comfortable + # deleting broadcasts. + op.alter_column('broadcast_message', 'template_id', nullable=False) + op.alter_column('broadcast_message', 'template_version', nullable=False) + op.drop_column('broadcast_message', 'content') diff --git a/migrations/versions/0336_broadcast_msg_content_2.py b/migrations/versions/0336_broadcast_msg_content_2.py index 1652afdec..9c596c98c 100644 --- a/migrations/versions/0336_broadcast_msg_content_2.py +++ b/migrations/versions/0336_broadcast_msg_content_2.py @@ -11,14 +11,28 @@ from notifications_utils.template import BroadcastMessageTemplate from sqlalchemy.dialects import postgresql from sqlalchemy.orm.session import Session - revision = '0336_broadcast_msg_content_2' down_revision = '0335_broadcast_msg_content' def upgrade(): - pass + + conn = op.get_bind() + + results = conn.execute(sa.text(""" + UPDATE + broadcast_message + SET + content = templates_history.content + FROM + templates_history + WHERE + broadcast_message.content is NULL and + broadcast_message.template_id = templates_history.id and + broadcast_message.template_version = templates_history.version + ; + """)) def downgrade(): - pass + op.alter_column('broadcast_message', 'content', nullable=True) diff --git a/migrations/versions/0337_broadcast_msg_api.py b/migrations/versions/0337_broadcast_msg_api.py index 249e0e983..96287e2bc 100644 --- a/migrations/versions/0337_broadcast_msg_api.py +++ b/migrations/versions/0337_broadcast_msg_api.py @@ -14,8 +14,13 @@ down_revision = '0336_broadcast_msg_content_2' def upgrade(): - pass + op.alter_column('broadcast_message', 'created_by_id', nullable=True) + op.add_column('broadcast_message', sa.Column('api_key_id', postgresql.UUID(), nullable=True)) + op.create_foreign_key(None, 'broadcast_message', 'api_keys', ['api_key_id'], ['id']) + op.add_column('broadcast_message', sa.Column('reference', sa.String(length=255), nullable=True)) def downgrade(): - pass + op.alter_column('broadcast_message', 'created_by_id', nullable=False) + op.drop_column('broadcast_message', 'api_key_id') + op.add_column('broadcast_message', 'reference') diff --git a/migrations/versions/0340_stub_training_broadcasts.py b/migrations/versions/0340_stub_training_broadcasts.py index 1d2091159..beb2ac8cc 100644 --- a/migrations/versions/0340_stub_training_broadcasts.py +++ b/migrations/versions/0340_stub_training_broadcasts.py @@ -15,11 +15,11 @@ down_revision = '0339_service_billing_details' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.add_column('broadcast_message', sa.Column('stubbed', sa.Boolean(), nullable=True)) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.drop_column('broadcast_message', 'stubbed') # ### end Alembic commands ### diff --git a/migrations/versions/0342_service_broadcast_settings.py b/migrations/versions/0342_service_broadcast_settings.py index 557ba55b2..ba706f562 100644 --- a/migrations/versions/0342_service_broadcast_settings.py +++ b/migrations/versions/0342_service_broadcast_settings.py @@ -16,11 +16,28 @@ CHANNEL_TYPES = ["test", "severe"] def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - pass - # ### end Alembic commands ### + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('broadcast_channel_types', + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('name') + ) + op.create_table('service_broadcast_settings', + sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('channel', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint(['channel'], ['broadcast_channel_types.name'], ), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], ), + sa.PrimaryKeyConstraint('service_id') + ) + # ### end Alembic commands ### + + for channel in CHANNEL_TYPES: + op.execute(f"INSERT INTO broadcast_channel_types VALUES ('{channel}')") + def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - pass - # ### end Alembic commands ### + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('service_broadcast_settings') + op.drop_table('broadcast_channel_types') + # ### end Alembic commands ### diff --git a/migrations/versions/0344_stubbed_not_nullable.py b/migrations/versions/0344_stubbed_not_nullable.py index 408eb4b5a..eb5e87028 100644 --- a/migrations/versions/0344_stubbed_not_nullable.py +++ b/migrations/versions/0344_stubbed_not_nullable.py @@ -15,11 +15,22 @@ down_revision = '0343_org_billing_details' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.execute("UPDATE broadcast_message SET stubbed = False WHERE stubbed is null") + op.alter_column( + 'broadcast_message', + 'stubbed', + existing_type=sa.BOOLEAN(), + nullable=False + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.alter_column( + 'broadcast_message', + 'stubbed', + existing_type=sa.BOOLEAN(), + nullable=True + ) # ### end Alembic commands ### diff --git a/migrations/versions/0345_move_broadcast_provider.py b/migrations/versions/0345_move_broadcast_provider.py index 4d20419fb..fbcea3078 100644 --- a/migrations/versions/0345_move_broadcast_provider.py +++ b/migrations/versions/0345_move_broadcast_provider.py @@ -14,8 +14,26 @@ down_revision = '0344_stubbed_not_nullable' def upgrade(): - pass + op.add_column('service_broadcast_settings', sa.Column('provider', sa.String(), nullable=True)) + + sql = """ + select service_id, provider + from service_broadcast_provider_restriction + where service_id NOT IN (select service_id from service_broadcast_settings) + """ + insert_sql = """ + insert into service_broadcast_settings(service_id, channel, provider, created_at, updated_at) + values('{}', 'test', '{}', now(), null) + """ + conn = op.get_bind() + results = conn.execute(sql) + restrictions = results.fetchall() + for x in restrictions: + f = insert_sql.format(x.service_id, x.provider) + conn.execute(f) def downgrade(): - pass + # Downgrade does not try and fully undo the upgrade, in particular it does not + # delete the rows added to the service_broadcast_settings table + op.drop_column('service_broadcast_settings', 'provider') diff --git a/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py b/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py index 4b2aaabc4..90577f58b 100644 --- a/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py +++ b/migrations/versions/0348_migrate_broadcast_settings_migrate_broadcast_settings.py @@ -14,7 +14,35 @@ down_revision = '0347_add_dvla_volumes_template' def upgrade(): - pass + # For every service that has the broadcast permission we want it to have + # a row in the broadcast_service_settings table + # + # If it doesnt have a row already, then: + # - if the service is in trial mode, add a row and set the channel as 'severe' + # - if the service is in live mode, add a row and set the channel as 'test' + # + # If it does have a row already no action needed + conn = op.get_bind() + + find_services_sql = """ + SELECT services.id, services.restricted + FROM services + LEFT JOIN service_permissions + ON services.id = service_permissions.service_id + WHERE service_permissions.permission = 'broadcast' + """ + + services = conn.execute(find_services_sql) + for service in services: + setting = conn.execute(f"SELECT service_id, channel, provider FROM service_broadcast_settings WHERE service_id = '{service.id}';").first() + if setting: + print(f"Service {service.id} already has service_broadcast_settings. No action required") + else: + channel = "severe" if service.restricted else "test" + print(f"Service {service.id} does not have service_broadcast_settings. Will insert one with channel {channel}") + conn.execute(f"INSERT INTO service_broadcast_settings (service_id, channel, created_at) VALUES ('{service.id}', '{channel}', now());") + def downgrade(): + # No downgrade as we do not know what the state of the table was before that it should return to pass diff --git a/migrations/versions/0352_broadcast_provider_types.py b/migrations/versions/0352_broadcast_provider_types.py index 48cc52814..6d0d1fad2 100644 --- a/migrations/versions/0352_broadcast_provider_types.py +++ b/migrations/versions/0352_broadcast_provider_types.py @@ -11,10 +11,22 @@ import sqlalchemy as sa revision = '0352_broadcast_provider_types' down_revision = '0351_unique_key_annual_billing' +PROVIDER_TYPES = ('ee', 'three', 'vodafone', 'o2', 'all') + def upgrade(): - pass + op.create_table('broadcast_provider_types', + sa.Column('name', sa.String(length=255), nullable=False), + sa.PrimaryKeyConstraint('name')) + for provider in PROVIDER_TYPES: + op.execute(f"INSERT INTO broadcast_provider_types VALUES ('{provider}')") + op.create_foreign_key('service_broadcast_settings_provider_fkey', + 'service_broadcast_settings', + 'broadcast_provider_types', + ['provider'], + ['name']) def downgrade(): - pass + op.drop_constraint('service_broadcast_settings_provider_fkey', 'service_broadcast_settings', type_='foreignkey') + op.drop_table('broadcast_provider_types') diff --git a/migrations/versions/0353_broadcast_provider_not_null.py b/migrations/versions/0353_broadcast_provider_not_null.py index 6c9f7f2b5..c470de38b 100644 --- a/migrations/versions/0353_broadcast_provider_not_null.py +++ b/migrations/versions/0353_broadcast_provider_not_null.py @@ -13,8 +13,10 @@ down_revision = '0352_broadcast_provider_types' def upgrade(): - pass + op.execute("UPDATE service_broadcast_settings SET provider = 'all' WHERE provider is null") + op.alter_column('service_broadcast_settings', 'provider', existing_type=sa.VARCHAR(), nullable=False) def downgrade(): - pass + op.alter_column('service_broadcast_settings', 'provider', existing_type=sa.VARCHAR(), nullable=True) + op.execute("UPDATE service_broadcast_settings SET provider = null WHERE provider = 'all'") diff --git a/migrations/versions/0354_government_channel.py b/migrations/versions/0354_government_channel.py index fc18b388f..25965c848 100644 --- a/migrations/versions/0354_government_channel.py +++ b/migrations/versions/0354_government_channel.py @@ -12,8 +12,11 @@ down_revision = '0353_broadcast_provider_not_null' def upgrade(): - pass + op.execute("INSERT INTO broadcast_channel_types VALUES ('government')") def downgrade(): - pass + # This can't be downgraded if there are rows in service_broadcast_settings which + # have the channel set to government or if broadcasts have already been sent on the + # government channel - it would break foreign key constraints. + op.execute("DELETE FROM broadcast_channel_types WHERE name = 'government'") diff --git a/migrations/versions/0358_operator_channel.py b/migrations/versions/0358_operator_channel.py index c36481510..eecfbac53 100644 --- a/migrations/versions/0358_operator_channel.py +++ b/migrations/versions/0358_operator_channel.py @@ -12,8 +12,11 @@ down_revision = '0357_validate_constraint' def upgrade(): - pass + op.execute("INSERT INTO broadcast_channel_types VALUES ('operator')") def downgrade(): - pass + # This can't be downgraded if there are rows in service_broadcast_settings which + # have the channel set to operator or if broadcasts have already been sent on the + # operator channel - it would break foreign key constraints. + op.execute("DELETE FROM broadcast_channel_types WHERE name = 'operator'") diff --git a/migrations/versions/0359_more_permissions.py b/migrations/versions/0359_more_permissions.py index 7188d5df3..329090e98 100644 --- a/migrations/versions/0359_more_permissions.py +++ b/migrations/versions/0359_more_permissions.py @@ -11,10 +11,40 @@ import sqlalchemy as sa revision = '0359_more_permissions' down_revision = '0358_operator_channel' +enum_name = 'permission_types' +tmp_name = 'tmp_' + enum_name + +old_options = ( + 'manage_users', + 'manage_templates', + 'manage_settings', + 'send_texts', + 'send_emails', + 'send_letters', + 'manage_api_keys', + 'platform_admin', + 'view_activity', +) +old_type = sa.Enum(*old_options, name=enum_name) + def upgrade(): - pass + # ALTER TYPE must be run outside of a transaction block (see link below for details) + # https://alembic.sqlalchemy.org/en/latest/api/runtime.html#alembic.runtime.migration.MigrationContext.autocommit_block + with op.get_context().autocommit_block(): + op.execute("ALTER TYPE permission_types ADD VALUE 'create_broadcasts'") + op.execute("ALTER TYPE permission_types ADD VALUE 'approve_broadcasts'") + op.execute("ALTER TYPE permission_types ADD VALUE 'cancel_broadcasts'") + op.execute("ALTER TYPE permission_types ADD VALUE 'reject_broadcasts'") def downgrade(): - pass + op.execute( + "DELETE FROM permissions WHERE permission in " + "('create_broadcasts', 'approve_broadcasts', 'cancel_broadcasts', 'reject_broadcasts')" + ) + + op.execute(f'ALTER TYPE {enum_name} RENAME TO {tmp_name}') + old_type.create(op.get_bind()) + op.execute(f'ALTER TABLE permissions ALTER COLUMN permission TYPE {enum_name} USING permission::text::{enum_name}') + op.execute(f'DROP TYPE {tmp_name}') diff --git a/migrations/versions/0362_broadcast_msg_event.py b/migrations/versions/0362_broadcast_msg_event.py index da94ada7e..04146958c 100644 --- a/migrations/versions/0362_broadcast_msg_event.py +++ b/migrations/versions/0362_broadcast_msg_event.py @@ -14,8 +14,8 @@ down_revision = '0361_new_user_bcast_permissions' def upgrade(): - pass + op.add_column('broadcast_message', sa.Column('cap_event', sa.String(length=255), nullable=True)) def downgrade(): - pass + op.drop_column('broadcast_message', 'cap_event') diff --git a/migrations/versions/0363_cancelled_by_api_key.py b/migrations/versions/0363_cancelled_by_api_key.py index 9896efb1c..20856a287 100644 --- a/migrations/versions/0363_cancelled_by_api_key.py +++ b/migrations/versions/0363_cancelled_by_api_key.py @@ -13,11 +13,33 @@ down_revision = '0362_broadcast_msg_event' def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.add_column('broadcast_message', sa.Column('created_by_api_key_id', postgresql.UUID(as_uuid=True), nullable=True)) + op.add_column( + 'broadcast_message', sa.Column('cancelled_by_api_key_id', postgresql.UUID(as_uuid=True), nullable=True) + ) + op.drop_constraint('broadcast_message_api_key_id_fkey', 'broadcast_message', type_='foreignkey') + op.create_foreign_key( + 'broadcast_message_created_by_api_key_id_fkey', + 'broadcast_message', + 'api_keys', + ['created_by_api_key_id'], + ['id'] + ) + op.create_foreign_key( + 'broadcast_message_cancelled_by_api_key_id_fkey', + 'broadcast_message', + 'api_keys', + ['cancelled_by_api_key_id'], + ['id'] + ) # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - pass + op.drop_constraint('broadcast_message_created_by_api_key_id_fkey', 'broadcast_message', type_='foreignkey') + op.drop_constraint('broadcast_message_cancelled_by_api_key_id_fkey', 'broadcast_message', type_='foreignkey') + op.create_foreign_key('broadcast_message_api_key_id_fkey', 'broadcast_message', 'api_keys', ['api_key_id'], ['id']) + op.drop_column('broadcast_message', 'cancelled_by_api_key_id') + op.drop_column('broadcast_message', 'created_by_api_key_id') # ### end Alembic commands ### diff --git a/migrations/versions/0364_drop_old_column.py b/migrations/versions/0364_drop_old_column.py index 039b229a5..40b730b9c 100644 --- a/migrations/versions/0364_drop_old_column.py +++ b/migrations/versions/0364_drop_old_column.py @@ -14,8 +14,20 @@ down_revision = '0363_cancelled_by_api_key' def upgrade(): - pass + # move data over + op.execute("UPDATE broadcast_message SET created_by_api_key_id=api_key_id WHERE created_by_api_key_id IS NULL") + op.create_check_constraint( + "ck_broadcast_message_created_by_not_null", + "broadcast_message", + "created_by_id is not null or created_by_api_key_id is not null" + ) + op.drop_column('broadcast_message', 'api_key_id') def downgrade(): - pass + op.add_column('broadcast_message', sa.Column('api_key_id', postgresql.UUID(), autoincrement=False, nullable=True)) + op.execute("UPDATE broadcast_message SET api_key_id=created_by_api_key_id") # move data over + op.drop_constraint( + "ck_broadcast_message_created_by_not_null", + "broadcast_message" + ) diff --git a/migrations/versions/0379_remove_broadcasts.py b/migrations/versions/0379_remove_broadcasts.py new file mode 100644 index 000000000..e73b79c40 --- /dev/null +++ b/migrations/versions/0379_remove_broadcasts.py @@ -0,0 +1,144 @@ +""" + +Revision ID: 0379_remove_broadcasts +Revises: 0378_add_org_names +Create Date: 2022-10-25 14:41:29.429928 + +""" +from alembic import op +import sqlalchemy as sa +import psycopg2 +from sqlalchemy.dialects import postgresql + +revision = '0379_remove_broadcasts' +down_revision = '0378_add_org_names' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('broadcast_provider_message_number') + op.drop_table('broadcast_provider_message_status_type') + op.drop_table('service_broadcast_settings') + op.drop_table('broadcast_provider_types') + op.drop_table('broadcast_provider_message') + op.drop_table('broadcast_event') + op.drop_table('broadcast_message') + op.drop_table('broadcast_status_type') + op.drop_table('broadcast_channel_types') + op.drop_table('service_broadcast_provider_restriction') + op.drop_column('templates', 'broadcast_data') + op.drop_column('templates_history', 'broadcast_data') + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('templates_history', sa.Column('broadcast_data', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.add_column('templates', sa.Column('broadcast_data', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True)) + op.create_table('service_broadcast_provider_restriction', + sa.Column('service_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('provider', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], name='service_broadcast_provider_restriction_service_id_fkey'), + sa.PrimaryKeyConstraint('service_id', name='service_broadcast_provider_restriction_pkey') + ) + op.create_table('broadcast_channel_types', + sa.Column('name', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('name', name='broadcast_channel_types_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('broadcast_status_type', + sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('name', name='broadcast_status_type_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('service_broadcast_settings', + sa.Column('service_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('channel', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('provider', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['channel'], ['broadcast_channel_types.name'], name='service_broadcast_settings_channel_fkey'), + sa.ForeignKeyConstraint(['provider'], ['broadcast_provider_types.name'], name='service_broadcast_settings_provider_fkey'), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], name='service_broadcast_settings_service_id_fkey'), + sa.PrimaryKeyConstraint('service_id', name='service_broadcast_settings_pkey') + ) + op.create_table('broadcast_event', + sa.Column('id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('service_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('broadcast_message_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('sent_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('message_type', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('transmitted_content', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=True), + sa.Column('transmitted_areas', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), + sa.Column('transmitted_sender', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('transmitted_starts_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('transmitted_finishes_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['broadcast_message_id'], ['broadcast_message.id'], name='broadcast_event_broadcast_message_id_fkey'), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], name='broadcast_event_service_id_fkey'), + sa.PrimaryKeyConstraint('id', name='broadcast_event_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('broadcast_message', + sa.Column('id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('service_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('template_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('template_version', sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column('_personalisation', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('areas', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), + sa.Column('status', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('starts_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('finishes_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('approved_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('cancelled_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('created_by_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('approved_by_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('cancelled_by_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('content', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('reference', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('stubbed', sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column('cap_event', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('created_by_api_key_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('cancelled_by_api_key_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.CheckConstraint('(created_by_id IS NOT NULL) OR (created_by_api_key_id IS NOT NULL)', name='ck_broadcast_message_created_by_not_null'), + sa.ForeignKeyConstraint(['approved_by_id'], ['users.id'], name='broadcast_message_approved_by_id_fkey'), + sa.ForeignKeyConstraint(['cancelled_by_api_key_id'], ['api_keys.id'], name='broadcast_message_cancelled_by_api_key_id_fkey'), + sa.ForeignKeyConstraint(['cancelled_by_id'], ['users.id'], name='broadcast_message_cancelled_by_id_fkey'), + sa.ForeignKeyConstraint(['created_by_api_key_id'], ['api_keys.id'], name='broadcast_message_created_by_api_key_id_fkey'), + sa.ForeignKeyConstraint(['created_by_id'], ['users.id'], name='broadcast_message_created_by_id_fkey'), + sa.ForeignKeyConstraint(['service_id'], ['services.id'], name='broadcast_message_service_id_fkey'), + sa.ForeignKeyConstraint(['status'], ['broadcast_status_type.name'], name='broadcast_message_status_fkey'), + sa.ForeignKeyConstraint(['template_id', 'template_version'], ['templates_history.id', 'templates_history.version'], name='broadcast_message_template_id_template_version_fkey'), + sa.PrimaryKeyConstraint('id', name='broadcast_message_pkey'), + postgresql_ignore_search_path=False + ) + op.create_table('broadcast_provider_message', + sa.Column('id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.Column('broadcast_event_id', postgresql.UUID(), autoincrement=False, nullable=True), + sa.Column('provider', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('status', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('created_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('updated_at', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint(['broadcast_event_id'], ['broadcast_event.id'], name='broadcast_provider_message_broadcast_event_id_fkey'), + sa.PrimaryKeyConstraint('id', name='broadcast_provider_message_pkey'), + sa.UniqueConstraint('broadcast_event_id', 'provider', name='broadcast_provider_message_broadcast_event_id_provider_key'), + postgresql_ignore_search_path=False + ) + op.create_table('broadcast_provider_types', + sa.Column('name', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('name', name='broadcast_provider_types_pkey') + ) + op.create_table('broadcast_provider_message_status_type', + sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('name', name='broadcast_provider_message_status_type_pkey') + ) + op.create_table('broadcast_provider_message_number', + sa.Column('broadcast_provider_message_number', sa.INTEGER(), server_default=sa.text("nextval('broadcast_provider_message_number_seq'::regclass)"), autoincrement=True, nullable=False), + sa.Column('broadcast_provider_message_id', postgresql.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint(['broadcast_provider_message_id'], ['broadcast_provider_message.id'], name='broadcast_provider_message_nu_broadcast_provider_message_i_fkey'), + sa.PrimaryKeyConstraint('broadcast_provider_message_number', name='broadcast_provider_message_number_pkey') + ) + # ### end Alembic commands ### From 637fbdb89139242200d43389ee9d448b97358d32 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 25 Oct 2022 11:53:24 -0400 Subject: [PATCH 53/65] broadcast flake8 cleanup --- app/authentication/auth.py | 1 + app/celery/scheduled_tasks.py | 2 +- app/models.py | 2 -- app/schemas.py | 2 +- app/utils.py | 2 +- tests/app/authentication/test_authentication.py | 3 +-- tests/app/celery/test_scheduled_tasks.py | 1 - tests/app/db.py | 2 -- tests/app/service/test_rest.py | 1 - tests/conftest.py | 2 +- 10 files changed, 6 insertions(+), 12 deletions(-) diff --git a/app/authentication/auth.py b/app/authentication/auth.py index 7201ad56f..7a744c039 100644 --- a/app/authentication/auth.py +++ b/app/authentication/auth.py @@ -63,6 +63,7 @@ class InternalApiKey(): def requires_no_auth(): pass + def requires_admin_auth(): requires_internal_auth(current_app.config.get('ADMIN_CLIENT_ID')) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index be09f3a74..15e5cff3d 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -7,7 +7,7 @@ from notifications_utils.clients.zendesk.zendesk_client import ( from sqlalchemy import between from sqlalchemy.exc import SQLAlchemyError -from app import db, notify_celery, zendesk_client +from app import notify_celery, zendesk_client from app.aws import s3 from app.celery.letters_pdf_tasks import get_pdf_for_templated_letter from app.celery.tasks import ( diff --git a/app/models.py b/app/models.py index d5e0d9e14..61f345b66 100644 --- a/app/models.py +++ b/app/models.py @@ -34,7 +34,6 @@ from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm.collections import attribute_mapped_collection -from sqlalchemy.schema import Sequence from app import db, encryption from app.hashing import check_hash, hashpw @@ -43,7 +42,6 @@ from app.utils import ( DATETIME_FORMAT, DATETIME_FORMAT_NO_TIMEZONE, get_dt_string_or_none, - get_uuid_string_or_none, ) SMS_TYPE = 'sms' diff --git a/app/schemas.py b/app/schemas.py index 1317ab33d..b83382350 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -234,7 +234,7 @@ class ServiceSchema(BaseSchema, UUIDsAsStringsMixin): email_branding = field_for(models.Service, 'email_branding') organisation = field_for(models.Service, 'organisation') go_live_at = field_for(models.Service, 'go_live_at', format=DATETIME_FORMAT_NO_TIMEZONE) - + def get_letter_logo_filename(self, service): return service.letter_branding and service.letter_branding.filename diff --git a/app/utils.py b/app/utils.py index c04e37793..cc3368a67 100644 --- a/app/utils.py +++ b/app/utils.py @@ -95,7 +95,7 @@ def get_public_notify_type_text(notify_type, plural=False): notify_type_text = 'document' elif notify_type == PRECOMPILED_LETTER: notify_type_text = 'precompiled letter' - + return '{}{}'.format(notify_type_text, 's' if plural else '') diff --git a/tests/app/authentication/test_authentication.py b/tests/app/authentication/test_authentication.py index 55bc59fc3..fe9a6cd1d 100644 --- a/tests/app/authentication/test_authentication.py +++ b/tests/app/authentication/test_authentication.py @@ -3,7 +3,7 @@ import uuid import jwt import pytest -from flask import current_app, g, request +from flask import g, request from notifications_python_client.authentication import create_jwt_token from app import db @@ -24,7 +24,6 @@ from app.dao.api_key_dao import ( from app.dao.services_dao import dao_fetch_service_by_id from tests import ( create_admin_authorization_header, - create_internal_authorization_header, create_service_authorization_header, ) from tests.conftest import set_config_values diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 652f79148..3b5c51ce8 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -35,7 +35,6 @@ from app.models import ( ) from tests.app import load_example_csv from tests.app.db import create_job, create_notification, create_template -from tests.conftest import set_config def _create_slow_delivery_notification(template, provider='mmg'): diff --git a/tests/app/db.py b/tests/app/db.py index 383a5019e..53c6f43fe 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -2,8 +2,6 @@ import random import uuid from datetime import date, datetime, timedelta -import pytest - from app import db from app.dao import fact_processing_time_dao from app.dao.email_branding_dao import dao_create_email_branding diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index 2236a4f29..13fe54dbe 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -28,7 +28,6 @@ from app.models import ( KEY_TYPE_TEST, LETTER_TYPE, NOTIFICATION_RETURNED_LETTER, - SERVICE_PERMISSION_TYPES, SMS_TYPE, UPLOAD_LETTERS, AnnualBilling, diff --git a/tests/conftest.py b/tests/conftest.py index 5678bfa3a..dabd3eb33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,7 +5,7 @@ import pytest import sqlalchemy from alembic.command import upgrade from alembic.config import Config -from flask import Flask, current_app +from flask import Flask from app import create_app, db from app.dao.provider_details_dao import get_provider_details_by_identifier From 8ad130893d8c8916be7198a60abdca3208c8b0da Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Tue, 25 Oct 2022 19:57:31 +0000 Subject: [PATCH 54/65] trim rollback from makefile --- Makefile | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Makefile b/Makefile index 1de8c9589..fcd1ce054 100644 --- a/Makefile +++ b/Makefile @@ -100,12 +100,6 @@ cf-login: ## Log in to Cloud Foundry cf-check-api-db-migration-task: ## Get the status for the last notifications-api task @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" -# .PHONY: cf-rollback -# cf-rollback: ## Rollbacks the app to the previous release -# $(if ${CF_APP},,$(error Must specify CF_APP)) -# rm ${CF_MANIFEST_PATH} -# cf cancel-deployment ${CF_APP} - .PHONY: check-if-migrations-to-run check-if-migrations-to-run: @echo $(shell python3 scripts/check_if_new_migration.py) From d27401c7a084e35be474d831559cddc929e8d15d Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 14:05:37 +0000 Subject: [PATCH 55/65] more pipenv transition --- .github/workflows/checks.yml | 2 + .github/workflows/daily_checks.yml | 2 + Makefile | 50 ++--- Pipfile.lock | 55 ++++++ README.md | 13 +- requirements.in | 39 ---- requirements.txt | 297 ----------------------------- requirements_for_test.txt | 14 -- 8 files changed, 80 insertions(+), 392 deletions(-) delete mode 100644 requirements.in delete mode 100644 requirements.txt delete mode 100644 requirements_for_test.txt diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 57e11688e..6fcdf8958 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -71,6 +71,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project + - name: Create requirements.txt + run: pipenv requirements - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 06dd0bc19..1449ef3d5 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -38,6 +38,8 @@ jobs: steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project + - name: Create requirements.txt + run: pipenv requirements - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt diff --git a/Makefile b/Makefile index fcd1ce054..7bc3da75c 100644 --- a/Makefile +++ b/Makefile @@ -7,18 +7,14 @@ APP_VERSION_FILE = app/version.py GIT_BRANCH ?= $(shell git symbolic-ref --short HEAD 2> /dev/null || echo "detached") GIT_COMMIT ?= $(shell git rev-parse HEAD) -CF_SPACE ?= ${DEPLOY_ENV} -CF_HOME ?= ${HOME} -$(eval export CF_HOME) - - ## DEVELOPMENT .PHONY: bootstrap -bootstrap: generate-version-file ## Set up everything to run the app - pip3 install -r requirements_for_test.txt +bootstrap: ## Set up everything to run the app + generate-version-file + pipenv install ---dev createdb notification_api || true - (flask db upgrade) || true + (pipenv run flask db upgrade) || true .PHONY: bootstrap-with-docker bootstrap-with-docker: ## Build the image to run the app in Docker @@ -26,31 +22,23 @@ bootstrap-with-docker: ## Build the image to run the app in Docker .PHONY: run-flask run-flask: ## Run flask - flask run -p 6011 --host=0.0.0.0 + pipenv run flask run -p 6011 --host=0.0.0.0 .PHONY: run-celery run-celery: ## Run celery, TODO remove purge for staging/prod - celery -A run_celery.notify_celery purge -f - celery \ + pipenv run celery -A run_celery.notify_celery purge -f + pipenv run celery \ -A run_celery.notify_celery worker \ --pidfile="/tmp/celery.pid" \ --loglevel=INFO \ --concurrency=4 -.PHONY: run-celery-with-docker -run-celery-with-docker: ## Run celery in Docker container (useful if you can't install pycurl locally) - ./scripts/run_with_docker.sh make run-celery - .PHONY: run-celery-beat run-celery-beat: ## Run celery beat - celery \ + pipenv run celery \ -A run_celery.notify_celery beat \ --loglevel=INFO -.PHONY: run-celery-beat-with-docker -run-celery-beat-with-docker: ## Run celery beat in Docker container (useful if you can't install pycurl locally) - ./scripts/run_with_docker.sh make run-celery-beat - .PHONY: help help: @cat $(MAKEFILE_LIST) | grep -E '^[a-zA-Z_-]+:.*?## .*$$' | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -67,12 +55,14 @@ test: ## Run tests .PHONY: freeze-requirements freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt - pip install --upgrade pip-tools - pip-compile requirements.in + pipenv lock + pipenv requirements .PHONY: audit audit: pip install --upgrade pip-audit + pipenv requirements > requirements.txt + pipenv requirements --dev > requirements_for_test.txt pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237 -pip-audit -r requirements_for_test.txt -l @@ -88,22 +78,6 @@ clean: ## DEPLOYMENT -.PHONY: cf-login -cf-login: ## Log in to Cloud Foundry - $(if ${CF_USERNAME},,$(error Must specify CF_USERNAME)) - $(if ${CF_PASSWORD},,$(error Must specify CF_PASSWORD)) - $(if ${CF_SPACE},,$(error Must specify CF_SPACE)) - @echo "Logging in to Cloud Foundry on ${CF_API}" - @cf login -a "${CF_API}" -u ${CF_USERNAME} -p "${CF_PASSWORD}" -o "${CF_ORG}" -s "${CF_SPACE}" - -.PHONY: cf-check-api-db-migration-task -cf-check-api-db-migration-task: ## Get the status for the last notifications-api task - @cf curl /v3/apps/`cf app --guid notifications-api`/tasks?order_by=-created_at | jq -r ".resources[0].state" - -.PHONY: check-if-migrations-to-run -check-if-migrations-to-run: - @echo $(shell python3 scripts/check_if_new_migration.py) - # .PHONY: cf-deploy-failwhale # cf-deploy-failwhale: # $(if ${CF_SPACE},,$(error Must target space, eg `make preview cf-deploy-failwhale`)) diff --git a/Pipfile.lock b/Pipfile.lock index 8cd0bf67e..565227cdb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,11 @@ { "_meta": { "hash": { +<<<<<<< Updated upstream "sha256": "eca9a39871c8db3e82fb384c39afc89fdc3c145c87653f5090927e578618ce11" +======= + "sha256": "ce99b649bae4b10b084aacc936b72e01957eb9354cbc420d684590133c2da004" +>>>>>>> Stashed changes }, "pipfile-spec": 6, "requires": { @@ -370,11 +374,19 @@ "version": "==0.4.0" }, "flask-sqlalchemy": { +<<<<<<< Updated upstream +======= + "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", +>>>>>>> Stashed changes "hashes": [ "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912", "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", +======= + "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", +>>>>>>> Stashed changes "version": "==2.5.1" }, "flask-sqlalchemy==2-5-1": { @@ -482,6 +494,17 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.1.3.post0" }, +<<<<<<< Updated upstream +======= + "gunicorn": { + "extras": [ + "eventlet" + ], + "git": "https://github.com/benoitc/gunicorn.git", + "ref": "1299ea9e967a61ae2edebe191082fd169b864c64", + "version": "==20.1.0" + }, +>>>>>>> Stashed changes "gunicorn[eventlet]==20-1-0": { "git": "https://github.com/benoitc/gunicorn.git", "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" @@ -732,7 +755,11 @@ }, "notifications-utils": { "git": "https://github.com/GSA/notifications-utils.git", +<<<<<<< Updated upstream "ref": "90c12da575f4e481452d4fcd2a594204b0c28249" +======= + "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd" +>>>>>>> Stashed changes }, "orderedset": { "hashes": [ @@ -1034,7 +1061,11 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.7' and python_version < '4'", +>>>>>>> Stashed changes "version": "==2.28.1" }, "rfc3339-validator": { @@ -1056,7 +1087,11 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==4.7.2" }, "s3transfer": { @@ -1211,7 +1246,11 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==1.26.12" }, "vine": { @@ -1323,11 +1362,19 @@ }, "zipp": { "hashes": [ +<<<<<<< Updated upstream "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb", "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980" ], "markers": "python_version >= '3.7'", "version": "==3.9.0" +======= + "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1", + "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8" + ], + "markers": "python_version >= '3.7'", + "version": "==3.10.0" +>>>>>>> Stashed changes } }, "develop": { @@ -1817,7 +1864,11 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], +<<<<<<< Updated upstream "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '3.7' and python_version < '4'", +>>>>>>> Stashed changes "version": "==2.28.1" }, "requests-mock": { @@ -1880,7 +1931,11 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], +<<<<<<< Updated upstream "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", +======= + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", +>>>>>>> Stashed changes "version": "==1.26.12" }, "werkzeug": { diff --git a/README.md b/README.md index cb0a95a21..1bd8ed0c4 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ This repo contains: - A public-facing REST API for Notify, which teams can integrate with using [API clients built by UK](https://www.notifications.service.gov.uk/documentation) - An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to) -- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc +- Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc. ## Local setup @@ -17,7 +17,7 @@ This repo contains: 1. Install dependencies into a virtual environment ``` - pipenv install --with dev + pipenv install --dev createdb notification_api flask db upgrade ``` @@ -46,7 +46,12 @@ This repo contains: If you're working in VS Code, you can also leverage Docker for a containerized dev environment -1. Create .env file as described in the .env section below. +1. Create the .env file + + ``` + cp sample.env .env + # follow the instructions in .env + ``` 1. Install the Remote-Containers plug-in in VS Code @@ -56,7 +61,7 @@ If you're working in VS Code, you can also leverage Docker for a containerized d 1. Using the command palette (shift+cmd+p) or green button thingy in the bottom left, search and select β€œRemote Containers: Open Folder in Container...” When prompted, choose **devcontainer-api** folder (note: this is a *subfolder* of notification-api). This will startup the container in a new window, replacing the current one. -1. Wait a few minutes while things happen +1. Wait a few minutes while things happen 🍡 1. Open a VS Code terminal and run the Flask application: diff --git a/requirements.in b/requirements.in deleted file mode 100644 index dbcc0f8a6..000000000 --- a/requirements.in +++ /dev/null @@ -1,39 +0,0 @@ -# Run `make freeze-requirements` to update requirements.txt -# with package version changes made in requirements.in - -cffi==1.15.0 -celery[redis]==5.2.7 -Flask-Bcrypt==1.0.1 -flask-marshmallow==0.14.0 -Flask-Migrate==3.1.0 -git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa#egg=Flask-SQLAlchemy==2.5.1 -Flask==2.1.2 -click-datetime==0.2 -# Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues) -git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0 -iso8601==1.0.2 -itsdangerous==2.1.2 -jsonschema[format]==4.5.1 -marshmallow-sqlalchemy==0.28.1 -marshmallow==3.15.0 -psycopg2-binary==2.9.3 -PyJWT==2.4.0 -SQLAlchemy==1.4.40 -cachetools==5.1.0 -beautifulsoup4==4.11.1 -lxml==4.9.1 -defusedxml==0.7.1 -Werkzeug==2.1.1 -python-dotenv==0.20.0 -oscrypto==1.3.0 - -notifications-python-client==6.3.0 - -# PaaS -awscli-cwlogs==1.4.6 - -notifications-utils @ git+https://github.com/GSA/notifications-utils.git#egg=notifications-utils - -# gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains -prometheus-client==0.14.1 -git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72#egg=gds-metrics diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index c5e41644e..000000000 --- a/requirements.txt +++ /dev/null @@ -1,297 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile requirements.in -# -alembic==1.7.7 - # via flask-migrate -amqp==5.1.1 - # via kombu -arrow==1.2.2 - # via isoduration -asn1crypto==1.5.1 - # via oscrypto -async-timeout==4.0.2 - # via redis -attrs==21.4.0 - # via jsonschema -awscli==1.24.8 - # via awscli-cwlogs -awscli-cwlogs==1.4.6 - # via -r requirements.in -bcrypt==3.2.2 - # via flask-bcrypt -beautifulsoup4==4.11.1 - # via -r requirements.in -billiard==3.6.4.0 - # via celery -bleach==4.1.0 - # via notifications-utils -blinker==1.4 - # via gds-metrics -boto3==1.23.8 - # via notifications-utils -botocore==1.26.8 - # via - # awscli - # boto3 - # s3transfer -cachetools==5.1.0 - # via - # -r requirements.in - # notifications-utils -celery[redis]==5.2.7 - # via -r requirements.in -certifi==2022.5.18.1 - # via - # pyproj - # requests -cffi==1.15.0 - # via - # -r requirements.in - # bcrypt -charset-normalizer==2.0.12 - # via requests -click==8.1.3 - # via - # celery - # click-datetime - # click-didyoumean - # click-plugins - # click-repl - # flask -click-datetime==0.2 - # via -r requirements.in -click-didyoumean==0.3.0 - # via celery -click-plugins==1.1.1 - # via celery -click-repl==0.2.0 - # via celery -colorama==0.4.4 - # via awscli -defusedxml==0.7.1 - # via -r requirements.in -deprecated==1.2.13 - # via redis -dnspython==2.2.1 - # via eventlet -docopt==0.6.2 - # via notifications-python-client -docutils==0.16 - # via awscli -eventlet==0.33.1 - # via gunicorn -flask==2.1.2 - # via - # -r requirements.in - # flask-bcrypt - # flask-marshmallow - # flask-migrate - # flask-redis - # flask-sqlalchemy - # gds-metrics - # notifications-utils -flask-bcrypt==1.0.1 - # via -r requirements.in -flask-marshmallow==0.14.0 - # via -r requirements.in -flask-migrate==3.1.0 - # via -r requirements.in -flask-redis==0.4.0 - # via notifications-utils -flask-sqlalchemy @ git+https://github.com/pallets-eco/flask-sqlalchemy.git@aa7a61a5357cf6f5dcc135d98c781192457aa6fa - # via - # -r requirements.in - # flask-migrate -fqdn==1.5.1 - # via jsonschema -gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72 - # via -r requirements.in -geojson==2.5.0 - # via notifications-utils -govuk-bank-holidays==0.11 - # via notifications-utils -greenlet==1.1.2 - # via - # eventlet - # sqlalchemy -gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64 - # via -r requirements.in -idna==3.3 - # via - # jsonschema - # requests -importlib-metadata==4.12.0 - # via flask -iso8601==1.0.2 - # via -r requirements.in -isoduration==20.11.0 - # via jsonschema -itsdangerous==2.1.2 - # via - # -r requirements.in - # flask - # notifications-utils -jinja2==3.1.2 - # via - # flask - # notifications-utils -jmespath==1.0.0 - # via - # boto3 - # botocore -jsonpointer==2.3 - # via jsonschema -jsonschema[format]==4.5.1 - # via -r requirements.in -kombu==5.2.4 - # via celery -lxml==4.9.1 - # via -r requirements.in -mako==1.2.2 - # via alembic -markupsafe==2.1.1 - # via - # jinja2 - # mako -marshmallow==3.15.0 - # via - # -r requirements.in - # flask-marshmallow - # marshmallow-sqlalchemy -marshmallow-sqlalchemy==0.28.1 - # via -r requirements.in -mistune==0.8.4 - # via notifications-utils -notifications-python-client==6.3.0 - # via -r requirements.in -notifications-utils @ git+https://github.com/GSA/notifications-utils.git - # via -r requirements.in -orderedset==2.0.3 - # via notifications-utils -oscrypto==1.3.0 - # via -r requirements.in -packaging==21.3 - # via - # bleach - # marshmallow - # marshmallow-sqlalchemy - # redis -phonenumbers==8.12.48 - # via notifications-utils -prometheus-client==0.14.1 - # via - # -r requirements.in - # gds-metrics -prompt-toolkit==3.0.29 - # via click-repl -psycopg2-binary==2.9.3 - # via -r requirements.in -pyasn1==0.4.8 - # via rsa -pycparser==2.21 - # via cffi -pyjwt==2.4.0 - # via - # -r requirements.in - # notifications-python-client -pyparsing==3.0.9 - # via packaging -pypdf2==2.0.0 - # via notifications-utils -pyproj==3.3.1 - # via notifications-utils -pyrsistent==0.18.1 - # via jsonschema -python-dateutil==2.8.2 - # via - # arrow - # awscli-cwlogs - # botocore -python-dotenv==0.20.0 - # via -r requirements.in -python-json-logger==2.0.2 - # via notifications-utils -pytz==2022.1 - # via - # celery - # notifications-utils -pyyaml==5.4.1 - # via - # awscli - # notifications-utils -redis==4.3.1 - # via - # celery - # flask-redis -requests==2.27.1 - # via - # awscli-cwlogs - # govuk-bank-holidays - # notifications-python-client - # notifications-utils -rfc3339-validator==0.1.4 - # via jsonschema -rfc3987==1.3.8 - # via jsonschema -rsa==4.7.2 - # via awscli -s3transfer==0.5.2 - # via - # awscli - # boto3 -shapely==1.8.2 - # via notifications-utils -six==1.16.0 - # via - # awscli-cwlogs - # bleach - # click-repl - # eventlet - # flask-marshmallow - # python-dateutil - # rfc3339-validator -smartypants==2.0.1 - # via notifications-utils -soupsieve==2.3.2.post1 - # via beautifulsoup4 -sqlalchemy==1.4.40 - # via - # -r requirements.in - # alembic - # flask-sqlalchemy - # marshmallow-sqlalchemy -statsd==3.3.0 - # via notifications-utils -typing-extensions==4.3.0 - # via pypdf2 -uri-template==1.2.0 - # via jsonschema -urllib3==1.26.9 - # via - # botocore - # requests -vine==5.0.0 - # via - # amqp - # celery - # kombu -wcwidth==0.2.5 - # via prompt-toolkit -webcolors==1.12 - # via jsonschema -webencodings==0.5.1 - # via bleach -werkzeug==2.1.1 - # via - # -r requirements.in - # flask -wrapt==1.14.1 - # via deprecated -zipp==3.8.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools diff --git a/requirements_for_test.txt b/requirements_for_test.txt deleted file mode 100644 index 3c6756bc1..000000000 --- a/requirements_for_test.txt +++ /dev/null @@ -1,14 +0,0 @@ ---requirement requirements.txt -flake8==4.0.1 -flake8-bugbear==22.4.25 -isort==5.10.1 -moto==3.1.9 -pytest==7.1.2 -pytest-env==0.6.2 -pytest-mock==3.7.0 -pytest-cov==3.0.0 -pytest-xdist==2.5.0 -freezegun==1.2.1 -requests-mock==1.9.3 -# used for creating manifest file locally -jinja2-cli[yaml]==0.8.2 From 38270c1c5c201a53ed46f77e9edd8a7efaf89b30 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 14:09:21 +0000 Subject: [PATCH 56/65] expand .cfignore and remove deploy-exclude.lst --- .cfignore | 110 ++++++++++++++++++++++++++++++++++++++++++++- deploy-exclude.lst | 13 ------ 2 files changed, 109 insertions(+), 14 deletions(-) mode change 120000 => 100644 .cfignore delete mode 100644 deploy-exclude.lst diff --git a/.cfignore b/.cfignore deleted file mode 120000 index 3e4e48b0b..000000000 --- a/.cfignore +++ /dev/null @@ -1 +0,0 @@ -.gitignore \ No newline at end of file diff --git a/.cfignore b/.cfignore new file mode 100644 index 000000000..fcbe7a227 --- /dev/null +++ b/.cfignore @@ -0,0 +1,109 @@ +# from deploy-exclude.lst + +*__pycache__* +.git/* +app/assets/* +bower_components/* +cache/* +.cache/* +node_modules/* +target/* +venv/* +build/* +.envrc +tests/.cache/* +.cf/* + +# from .gitignore + +queues.csv + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] + +.venv/ +venv/ +venv-freeze/ + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +/cache + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +.pytest_cache +coverage.xml +test_results.xml +*,cover + +# Translations +*.mo +*.pot + +# Django stuff: +*.log + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ +.idea/ +.vscode + +# Mac +*.DS_Store +environment.sh +.envrc +.env +.env* +varsfile + +celerybeat-schedule + +# CloudFoundry +.cf +varsfile* +.secret* + +/scripts/run_my_tests.sh + +# Terraform +.terraform.lock.hcl +**/.terraform/* +secrets.auto.tfvars +terraform.tfstate +terraform.tfstate.backup diff --git a/deploy-exclude.lst b/deploy-exclude.lst deleted file mode 100644 index 060fca01d..000000000 --- a/deploy-exclude.lst +++ /dev/null @@ -1,13 +0,0 @@ -*__pycache__* -.git/* -app/assets/* -bower_components/* -cache/* -.cache/* -node_modules/* -target/* -venv/* -build/* -.envrc -tests/.cache/* -.cf/* From 56329daa9a741b4c887dab29b777fa045f12b4a3 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 15:45:45 +0000 Subject: [PATCH 57/65] more docs --- README.md | 47 +++++++++++++++++++----------- docs/api-usage.md | 10 +++++++ docs/database-management.md | 4 +++ docs/infra-onboarding.md | 7 ----- docs/infra-overview.md | 57 +++++++++++++++++++++++++++++++++++++ docs/infra-setup.md | 20 ------------- 6 files changed, 101 insertions(+), 44 deletions(-) create mode 100644 docs/api-usage.md delete mode 100644 docs/infra-onboarding.md create mode 100644 docs/infra-overview.md delete mode 100644 docs/infra-setup.md diff --git a/README.md b/README.md index 1bd8ed0c4..73877fa92 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,36 @@ This repo contains: - An internal-only REST API built using Flask to manage services, users, templates, etc., which the [admin UI](http://github.com/18F/notifications-admin) talks to) - Asynchronous workers built using Celery to put things on queues and read them off to be processed, sent to providers, updated, etc. +Our other repositories are: + +- [notifications-admin](https://github.com/GSA/notifications-admin) +- [notifications-utils](https://github.com/GSA/notifications-utils) +- [us-notify-compliance](https://github.com/GSA/us-notify-compliance/) +- [notify-python-demo](https://github.com/GSA/notify-python-demo) + +## Documentation, here and elsewhere + +### About Notify + +- [Roadmap](https://notifications-admin.app.cloud.gov/features/roadmap) +- [Using the API](./docs/api-usage.md) + +### Infrastructure + +- [Overview, setup, and onboarding](./docs/infra-overview.md) +- [Database management](./docs/database-management.md) + +### Common dev work + +- [Local setup](#local-setup) +- [Testing](./docs/testing.md) +- [Running one-off tasks](./docs/one-off-tasks.md) + +## UK docs that may still be helpful + +- [Writing public APIs](docs/writing-public-apis.md) +- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies) + ## Local setup ### Direct installation @@ -73,20 +103,3 @@ If you're working in VS Code, you can also leverage Docker for a containerized d NOTE: when you change .env in the future, you'll need to rebuild the devcontainer for the change to take effect. Vscode _should_ detect the change and prompt you with a toast notification during a cached build. If not, you can find a manual rebuild in command pallette or just `docker rm` the notifications-api container. -## Deeper documentation - -### Infrastructure - -- [Checklist for onboarding to all of the things](./docs/infra-onboarding.md) -- [Setting up the initial infrastructure using AWS](./docs/infra-setup.md) -- [Database management](./docs/database-management.md) - -### Common dev work - -- [Testing](./docs/testing.md) -- [Running one-off tasks](./docs/one-off-tasks.md) - -## UK docs that may still be helpful - -- [Writing public APIs](docs/writing-public-apis.md) -- [Updating dependencies](https://github.com/alphagov/notifications-manuals/wiki/Dependencies) diff --git a/docs/api-usage.md b/docs/api-usage.md new file mode 100644 index 000000000..8ec2b4e94 --- /dev/null +++ b/docs/api-usage.md @@ -0,0 +1,10 @@ +# API Usage + +## Connecting to the API + +To make life easier, the [UK API client libraries](https://www.notifications.service.gov.uk/documentation) are compatible with Notify. + +For a usage example, see [our Python demo](https://github.com/GSA/notify-python-demo). + +An API key can be created at https://notifications-admin.app.cloud.gov/services/YOUR_SERVICE_ID/api/keys. However, in order to successfully send messages, you will need to receive a secret header token from the Notify team. + diff --git a/docs/database-management.md b/docs/database-management.md index 589df97fd..9d8685ce7 100644 --- a/docs/database-management.md +++ b/docs/database-management.md @@ -12,6 +12,10 @@ about what is loaded into which tables, and some plans for how we might manage t Flask does not seem to have a great way to squash migrations, but rather wants you to recreate them from the DB structure. This means it's easy to recreate the tables, but hard to recreate the initial data. +## Data Model Diagram + +A diagram of Notify's data model is available [in our compliance repo](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/data.logical.pdf). + ## Migrations Create a migration: diff --git a/docs/infra-onboarding.md b/docs/infra-onboarding.md deleted file mode 100644 index 6d7789dfe..000000000 --- a/docs/infra-onboarding.md +++ /dev/null @@ -1,7 +0,0 @@ -# Infrastructure onboarding - -- [ ] Join [the GSA GitHub org](https://github.com/GSA/GitHub-Administration#join-the-gsa-organization) -- [ ] Get permissions for the repos -- [ ] Get access to the cloud.gov org && space -- [ ] Get access to AWS, if necessary -- [ ] Pull down creds from cloud.gov and create the local .env file \ No newline at end of file diff --git a/docs/infra-overview.md b/docs/infra-overview.md new file mode 100644 index 000000000..db017f8f3 --- /dev/null +++ b/docs/infra-overview.md @@ -0,0 +1,57 @@ +# Infrastructure overview + +A diagram of the system is available [in our compliance repo](https://github.com/GSA/us-notify-compliance/blob/main/diagrams/rendered/apps/application.boundary.png). + +Notify is a Flask application running on [cloud.gov](https://cloud.gov), which also brokers access to a PostgreSQL database and Redis store. + +In addition to the Flask app, Notify uses Celery to manage the task queue. Celery stores tasks in Redis. + +## Terraform + +The cloud.gov environment is configured with Terraform. See [the `terraform` folder](../terraform/) to learn about that. + +## AWS + +In addition to services provisioned through cloud.gov, we have several services provisioned directly in AWS. Our AWS services are currently located in the us-west-2 region using the tts-sandbox account. We plan to move to GovCloud shortly. + +To send messages, we use Amazon Web Services SNS and SES. In addition, we use AWS Pinpoint to provision and manage phone numbers, short codes, and long codes for sending SMS. + +In SES, we are currently using the "sandbox" mode. This requires email addresses to be pre-registered in the AWS console in order to receive emails. The DKIM settings live under the verified domain entry. + +In SNS, we have 3 topics for SMS receipts. These are not currently functional, so senders won't know the status of messages. + +Through Pinpoint, the API needs at least one number so that the application itself can send SMS for authentication codes. + +The API also has access to AWS S3 buckets for storing CSVs of messages and contact lists. It does not access a third S3 bucket that stores agency logos. + +We may be able to provision these services through cloud.gov, as well. In addition to [s3 support](https://cloud.gov/docs/services/s3/), there is [an SES brokerpak](https://github.com/GSA-TTS/datagov-brokerpak-smtp) and work on an SNS brokerpak. + +## Onboarding + +- [ ] Join [the GSA GitHub org](https://github.com/GSA/GitHub-Administration#join-the-gsa-organization) +- [ ] Get permissions for the repos +- [ ] Get access to the cloud.gov org && space +- [ ] Get [access to AWS](https://handbook.tts.gsa.gov/launching-software/infrastructure/#cloud-service-provider-csp-sandbox-accounts), if necessary +- [ ] Pull down creds from cloud.gov and create the local .env file +- [ ] Do stuff! + +## Setting up the infrastructure + +### Steps to prepare SES + +1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. +2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. + +TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures + +### Steps to prepare SNS + +1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" +2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save +3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" +4. Under "Number Settings" choose "Request Phone Number" +5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" +6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" +7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) + +At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! πŸŽ‰ \ No newline at end of file diff --git a/docs/infra-setup.md b/docs/infra-setup.md deleted file mode 100644 index 9bd83f058..000000000 --- a/docs/infra-setup.md +++ /dev/null @@ -1,20 +0,0 @@ -# Setting up the infrastructure - -## Steps to prepare SES - -1. Go to SES console for \$AWS_REGION and create new origin and destination emails. AWS will send a verification via email which you'll need to complete. -2. Find and replace instances in the repo of "testsender", "testreceiver" and "dispostable.com", with your origin and destination email addresses, which you verified in step 1 above. - -TODO: create env vars for these origin and destination email addresses for the root service, and create new migrations to update postgres seed fixtures - -## Steps to prepare SNS - -1. Go to Pinpoints console for \$AWS_PINPOINT_REGION and choose "create new project", then "configure for sms" -2. Tick the box at the top to enable SMS, choose "transactional" as the default type and save -3. In the lefthand sidebar, go the "SMS and Voice" (bottom) and choose "Phone Numbers" -4. Under "Number Settings" choose "Request Phone Number" -5. Choose Toll-free number, tick SMS, untick Voice, choose "transactional", hit next and then "request" -6. Go to SNS console for \$AWS_PINPOINT_REGION, look at lefthand sidebar under "Mobile" and go to "Text Messaging (SMS)" -7. Scroll down to "Sandbox destination phone numbers" and tap "Add phone number" then follow the steps to verify (you'll need to be able to retrieve a code sent to each number) - -At this point, you _should_ be able to complete both the email and phone verification steps of the Notify user sign up process! πŸŽ‰ \ No newline at end of file From 82c5608e0adfd4a6e3cdfcd91629894261b513be Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 15:26:53 -0400 Subject: [PATCH 58/65] github setup change for pipenv --- .github/actions/setup-project/action.yml | 3 +++ Makefile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/setup-project/action.yml b/.github/actions/setup-project/action.yml index 003d8bf6e..a6caa11e9 100644 --- a/.github/actions/setup-project/action.yml +++ b/.github/actions/setup-project/action.yml @@ -13,3 +13,6 @@ runs: uses: actions/setup-python@v3 with: python-version: "3.9" + - name: Install pipenv + shell: bash + run: pip install --upgrade pipenv diff --git a/Makefile b/Makefile index 7bc3da75c..e03c05e46 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ GIT_COMMIT ?= $(shell git rev-parse HEAD) .PHONY: bootstrap bootstrap: ## Set up everything to run the app - generate-version-file + make generate-version-file pipenv install ---dev createdb notification_api || true (pipenv run flask db upgrade) || true From 9b9465c74a88ba546ae25ac666e250f1360a7af9 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 15:34:12 -0400 Subject: [PATCH 59/65] remove extra hyphen --- Makefile | 2 +- Pipfile.lock | 76 ++++------------------------------------------------ 2 files changed, 6 insertions(+), 72 deletions(-) diff --git a/Makefile b/Makefile index e03c05e46..9e69493cc 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ GIT_COMMIT ?= $(shell git rev-parse HEAD) .PHONY: bootstrap bootstrap: ## Set up everything to run the app make generate-version-file - pipenv install ---dev + pipenv install --dev createdb notification_api || true (pipenv run flask db upgrade) || true diff --git a/Pipfile.lock b/Pipfile.lock index 565227cdb..4c841e4d8 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,7 @@ { "_meta": { "hash": { -<<<<<<< Updated upstream - "sha256": "eca9a39871c8db3e82fb384c39afc89fdc3c145c87653f5090927e578618ce11" -======= "sha256": "ce99b649bae4b10b084aacc936b72e01957eb9354cbc420d684590133c2da004" ->>>>>>> Stashed changes }, "pipfile-spec": 6, "requires": { @@ -374,29 +370,10 @@ "version": "==0.4.0" }, "flask-sqlalchemy": { -<<<<<<< Updated upstream -======= "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", ->>>>>>> Stashed changes - "hashes": [ - "sha256:2bda44b43e7cacb15d4e05ff3cc1f8bc97936cc464623424102bfc2c35e95912", - "sha256:f12c3d4cc5cc7fdcc148b9527ea05671718c3ea45d50c7e732cceb33f574b390" - ], -<<<<<<< Updated upstream - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", -======= "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa", ->>>>>>> Stashed changes "version": "==2.5.1" }, - "flask-sqlalchemy==2-5-1": { - "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", - "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa" - }, - "flask-sqlalchemy==2.5.1": { - "git": "https://github.com/pallets-eco/flask-sqlalchemy.git", - "ref": "aa7a61a5357cf6f5dcc135d98c781192457aa6fa" - }, "fqdn": { "hashes": [ "sha256:105ed3677e767fb5ca086a0c1f4bb66ebc3c100be518f0e0d755d9eae164d89f", @@ -494,8 +471,6 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.1.3.post0" }, -<<<<<<< Updated upstream -======= "gunicorn": { "extras": [ "eventlet" @@ -504,15 +479,6 @@ "ref": "1299ea9e967a61ae2edebe191082fd169b864c64", "version": "==20.1.0" }, ->>>>>>> Stashed changes - "gunicorn[eventlet]==20-1-0": { - "git": "https://github.com/benoitc/gunicorn.git", - "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" - }, - "gunicorn[eventlet]==20.1.0": { - "git": "https://github.com/benoitc/gunicorn.git", - "ref": "1299ea9e967a61ae2edebe191082fd169b864c64" - }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -755,11 +721,7 @@ }, "notifications-utils": { "git": "https://github.com/GSA/notifications-utils.git", -<<<<<<< Updated upstream - "ref": "90c12da575f4e481452d4fcd2a594204b0c28249" -======= "ref": "2cdffe3fa2417b61ce3d714dc5a2d67de6632bdd" ->>>>>>> Stashed changes }, "orderedset": { "hashes": [ @@ -988,7 +950,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.2" }, "python-dotenv": { @@ -1061,11 +1023,7 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], -<<<<<<< Updated upstream - "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", -======= "markers": "python_version >= '3.7' and python_version < '4'", ->>>>>>> Stashed changes "version": "==2.28.1" }, "rfc3339-validator": { @@ -1087,11 +1045,7 @@ "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], -<<<<<<< Updated upstream - "markers": "python_version >= '3.5' and python_full_version < '4.0.0'", -======= "markers": "python_version >= '3.5' and python_version < '4'", ->>>>>>> Stashed changes "version": "==4.7.2" }, "s3transfer": { @@ -1160,7 +1114,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "smartypants": { @@ -1246,11 +1200,7 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], -<<<<<<< Updated upstream - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", -======= "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", ->>>>>>> Stashed changes "version": "==1.26.12" }, "vine": { @@ -1362,19 +1312,11 @@ }, "zipp": { "hashes": [ -<<<<<<< Updated upstream - "sha256:3a7af91c3db40ec72dd9d154ae18e008c69efe8ca88dde4f9a731bb82fe2f9eb", - "sha256:972cfa31bc2fedd3fa838a51e9bc7e64b7fb725a8c00e7431554311f180e9980" - ], - "markers": "python_version >= '3.7'", - "version": "==3.9.0" -======= "sha256:4fcb6f278987a6605757302a6e40e896257570d11c51628968ccb2a47e80c6c1", "sha256:7a7262fd930bd3e36c50b9a64897aec3fafff3dfdeec9623ae22b40e93f99bb8" ], "markers": "python_version >= '3.7'", "version": "==3.10.0" ->>>>>>> Stashed changes } }, "develop": { @@ -1814,7 +1756,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==2.8.2" }, "pytz": { @@ -1864,11 +1806,7 @@ "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], -<<<<<<< Updated upstream - "markers": "python_version >= '3.7' and python_full_version < '4.0.0'", -======= "markers": "python_version >= '3.7' and python_version < '4'", ->>>>>>> Stashed changes "version": "==2.28.1" }, "requests-mock": { @@ -1900,7 +1838,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "version": "==1.16.0" }, "toml": { @@ -1908,7 +1846,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "tomli": { @@ -1931,11 +1869,7 @@ "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" ], -<<<<<<< Updated upstream - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_full_version < '4.0.0'", -======= "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", ->>>>>>> Stashed changes "version": "==1.26.12" }, "werkzeug": { From 2889f6220a3084a9c2b4e8676dff0346bccd97b7 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 16:21:45 -0400 Subject: [PATCH 60/65] actually write requirements to file --- .github/workflows/checks.yml | 2 +- .github/workflows/daily_checks.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6fcdf8958..9dad4fbe5 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -72,7 +72,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project - name: Create requirements.txt - run: pipenv requirements + run: pipenv requirements > requirements.txt - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt diff --git a/.github/workflows/daily_checks.yml b/.github/workflows/daily_checks.yml index 1449ef3d5..d63917306 100644 --- a/.github/workflows/daily_checks.yml +++ b/.github/workflows/daily_checks.yml @@ -39,7 +39,7 @@ jobs: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-project - name: Create requirements.txt - run: pipenv requirements + run: pipenv requirements > requirements.txt - uses: trailofbits/gh-action-pip-audit@v1.0.0 with: inputs: requirements.txt From 7b80210884f70c0aa6a62af7984a466436a8f808 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 16:29:51 -0400 Subject: [PATCH 61/65] locate isort in time and space --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9dad4fbe5..7bf04a114 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -60,9 +60,9 @@ jobs: # - name: Run style checks # run: flake8 . - name: Check imports alphabetized - run: isort --check-only ./app ./tests + run: pipenv run isort --check-only ./app ./tests - name: Run tests - run: pytest -n4 --maxfail=10 + run: pipenv run pytest -n4 --maxfail=10 env: SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api From 96431f038867c49e96d1db22cbdde35c0890f0df Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Wed, 26 Oct 2022 16:47:40 -0400 Subject: [PATCH 62/65] pipenv + flake8 --- .github/workflows/checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b07714731..df553a7da 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -57,7 +57,7 @@ jobs: env: SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api - name: Run style checks - run: flake8 . + run: pipenv run flake8 . - name: Check imports alphabetized run: pipenv run isort --check-only ./app ./tests - name: Run tests From 493e7e015a8592a97e67207ef3346e363d32ed30 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 28 Oct 2022 12:58:07 +0000 Subject: [PATCH 63/65] pipenv in devcontainers, probably --- Makefile | 14 +- Pipfile | 2 + Pipfile.lock | 243 +++++++++++++++++- .../scripts/notify-dev-entrypoint.sh | 5 +- .../scripts/notify-worker-entrypoint.sh | 3 +- 5 files changed, 248 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 3318739bf..b14f342fa 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,9 @@ generate-version-file: ## Generates the app version file .PHONY: test test: ## Run tests - flake8 . - isort --check-only ./app ./tests - pytest -n4 --maxfail=10 + pipenv run flake8 . + pipenv run isort --check-only ./app ./tests + pipenv run pytest -n4 --maxfail=10 .PHONY: freeze-requirements freeze-requirements: ## Pin all requirements including sub dependencies into requirements.txt @@ -60,16 +60,14 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req .PHONY: audit audit: - pip install --upgrade pip-audit pipenv requirements > requirements.txt pipenv requirements --dev > requirements_for_test.txt - pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237 - -pip-audit -r requirements_for_test.txt -l + pipenv run pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237 + -pipenv run pip-audit -r requirements_for_test.txt -l .PHONY: static-scan static-scan: - pip install bandit - bandit -r app/ + pipenv run bandit -r app/ .PHONY: clean clean: diff --git a/Pipfile b/Pipfile index c1ae3089f..dea8f5f5c 100644 --- a/Pipfile +++ b/Pipfile @@ -75,6 +75,8 @@ pytest-xdist = "==2.5.0" freezegun = "==1.2.1" requests-mock = "==1.9.3" jinja2-cli = {version = "==0.8.2", extras = ["yaml"]} +pip-audit = "*" +bandit = "*" [requires] python_version = "3.9" diff --git a/Pipfile.lock b/Pipfile.lock index 4c841e4d8..3cdd1fea4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ce99b649bae4b10b084aacc936b72e01957eb9354cbc420d684590133c2da004" + "sha256": "4d2f356b612e2a1c813e6d0f0db1d73948a7124b19ff012d993b74ec5cc4b03e" }, "pipfile-spec": 6, "requires": { @@ -950,7 +950,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "python-dotenv": { @@ -1114,7 +1114,7 @@ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smartypants": { @@ -1328,6 +1328,14 @@ "index": "pypi", "version": "==21.4.0" }, + "bandit": { + "hashes": [ + "sha256:2d63a8c573417bae338962d4b9b06fbc6080f74ecd955a092849e1e65c717bd2", + "sha256:412d3f259dab4077d0e7f0c11f50f650cc7d10db905d98f6520a95a18049658a" + ], + "index": "pypi", + "version": "==1.7.4" + }, "boto3": { "hashes": [ "sha256:15733c2bbedce7a36fcf1749560c72c3ee90785aa6302a98658c7bffdcbe1f2a", @@ -1344,6 +1352,17 @@ "index": "pypi", "version": "==1.26.8" }, + "cachecontrol": { + "extras": [ + "filecache" + ], + "hashes": [ + "sha256:2c75d6a8938cb1933c75c50184549ad42728a27e9f6b92fd677c3151aa72555b", + "sha256:a5b9fcc986b184db101aa280b42ecdcdfc524892596f606858e0b7a8b4d9e144" + ], + "markers": "python_version >= '3.6'", + "version": "==0.12.11" + }, "certifi": { "hashes": [ "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", @@ -1416,6 +1435,13 @@ "index": "pypi", "version": "==2.0.12" }, + "commonmark": { + "hashes": [ + "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", + "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" + ], + "version": "==0.9.1" + }, "coverage": { "extras": [ "toml" @@ -1507,6 +1533,14 @@ "markers": "python_version >= '3.6'", "version": "==38.0.1" }, + "cyclonedx-python-lib": { + "hashes": [ + "sha256:39e9d36347d4dc736474ab4f3a7cd7bc91050c9315df698f83a6d8bbcb290744", + "sha256:3c79f32bb7d6ed34eac3308dbc8f2a77fbd1fd3779991173a147d866eaa7423e" + ], + "markers": "python_version >= '3.6' and python_version < '4.0'", + "version": "==3.1.0" + }, "execnet": { "hashes": [ "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5", @@ -1539,6 +1573,30 @@ "index": "pypi", "version": "==1.2.1" }, + "gitdb": { + "hashes": [ + "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd", + "sha256:bac2fd45c0a1c9cf619e63a90d62bdc63892ef92387424b855792a6cabe789aa" + ], + "markers": "python_version >= '3.6'", + "version": "==4.0.9" + }, + "gitpython": { + "hashes": [ + "sha256:41eea0deec2deea139b459ac03656f0dd28fc4a3387240ec1d3c259a2c47850f", + "sha256:cc36bfc4a3f913e66805a28e84703e419d9c264c1077e537b54f0e1af85dbefd" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.29" + }, + "html5lib": { + "hashes": [ + "sha256:0d78f8fde1c230e99fe37986a60526d7049ed4bf8a9fadbad5f00e22e58e041d", + "sha256:b2e5b40261e20f354d198eae92afc10d750afb487ed5e50f9c4eaf07c184146f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.1" + }, "idna": { "hashes": [ "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", @@ -1588,6 +1646,13 @@ "markers": "python_version >= '3.7'", "version": "==1.0.1" }, + "lockfile": { + "hashes": [ + "sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799", + "sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa" + ], + "version": "==0.12.2" + }, "markupsafe": { "hashes": [ "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", @@ -1649,6 +1714,71 @@ "index": "pypi", "version": "==3.1.9" }, + "msgpack": { + "hashes": [ + "sha256:002b5c72b6cd9b4bafd790f364b8480e859b4712e91f43014fe01e4f957b8467", + "sha256:0a68d3ac0104e2d3510de90a1091720157c319ceeb90d74f7b5295a6bee51bae", + "sha256:0df96d6eaf45ceca04b3f3b4b111b86b33785683d682c655063ef8057d61fd92", + "sha256:0dfe3947db5fb9ce52aaea6ca28112a170db9eae75adf9339a1aec434dc954ef", + "sha256:0e3590f9fb9f7fbc36df366267870e77269c03172d086fa76bb4eba8b2b46624", + "sha256:11184bc7e56fd74c00ead4f9cc9a3091d62ecb96e97653add7a879a14b003227", + "sha256:112b0f93202d7c0fef0b7810d465fde23c746a2d482e1e2de2aafd2ce1492c88", + "sha256:1276e8f34e139aeff1c77a3cefb295598b504ac5314d32c8c3d54d24fadb94c9", + "sha256:1576bd97527a93c44fa856770197dec00d223b0b9f36ef03f65bac60197cedf8", + "sha256:1e91d641d2bfe91ba4c52039adc5bccf27c335356055825c7f88742c8bb900dd", + "sha256:26b8feaca40a90cbe031b03d82b2898bf560027160d3eae1423f4a67654ec5d6", + "sha256:2999623886c5c02deefe156e8f869c3b0aaeba14bfc50aa2486a0415178fce55", + "sha256:2a2df1b55a78eb5f5b7d2a4bb221cd8363913830145fad05374a80bf0877cb1e", + "sha256:2bb8cdf50dd623392fa75525cce44a65a12a00c98e1e37bf0fb08ddce2ff60d2", + "sha256:2cc5ca2712ac0003bcb625c96368fd08a0f86bbc1a5578802512d87bc592fe44", + "sha256:35bc0faa494b0f1d851fd29129b2575b2e26d41d177caacd4206d81502d4c6a6", + "sha256:3c11a48cf5e59026ad7cb0dc29e29a01b5a66a3e333dc11c04f7e991fc5510a9", + "sha256:449e57cc1ff18d3b444eb554e44613cffcccb32805d16726a5494038c3b93dab", + "sha256:462497af5fd4e0edbb1559c352ad84f6c577ffbbb708566a0abaaa84acd9f3ae", + "sha256:4733359808c56d5d7756628736061c432ded018e7a1dff2d35a02439043321aa", + "sha256:48f5d88c99f64c456413d74a975bd605a9b0526293218a3b77220a2c15458ba9", + "sha256:49565b0e3d7896d9ea71d9095df15b7f75a035c49be733051c34762ca95bbf7e", + "sha256:4ab251d229d10498e9a2f3b1e68ef64cb393394ec477e3370c457f9430ce9250", + "sha256:4d5834a2a48965a349da1c5a79760d94a1a0172fbb5ab6b5b33cbf8447e109ce", + "sha256:4dea20515f660aa6b7e964433b1808d098dcfcabbebeaaad240d11f909298075", + "sha256:545e3cf0cf74f3e48b470f68ed19551ae6f9722814ea969305794645da091236", + "sha256:63e29d6e8c9ca22b21846234913c3466b7e4ee6e422f205a2988083de3b08cae", + "sha256:6916c78f33602ecf0509cc40379271ba0f9ab572b066bd4bdafd7434dee4bc6e", + "sha256:6a4192b1ab40f8dca3f2877b70e63799d95c62c068c84dc028b40a6cb03ccd0f", + "sha256:6c9566f2c39ccced0a38d37c26cc3570983b97833c365a6044edef3574a00c08", + "sha256:76ee788122de3a68a02ed6f3a16bbcd97bc7c2e39bd4d94be2f1821e7c4a64e6", + "sha256:7760f85956c415578c17edb39eed99f9181a48375b0d4a94076d84148cf67b2d", + "sha256:77ccd2af37f3db0ea59fb280fa2165bf1b096510ba9fe0cc2bf8fa92a22fdb43", + "sha256:81fc7ba725464651190b196f3cd848e8553d4d510114a954681fd0b9c479d7e1", + "sha256:85f279d88d8e833ec015650fd15ae5eddce0791e1e8a59165318f371158efec6", + "sha256:9667bdfdf523c40d2511f0e98a6c9d3603be6b371ae9a238b7ef2dc4e7a427b0", + "sha256:a75dfb03f8b06f4ab093dafe3ddcc2d633259e6c3f74bb1b01996f5d8aa5868c", + "sha256:ac5bd7901487c4a1dd51a8c58f2632b15d838d07ceedaa5e4c080f7190925bff", + "sha256:aca0f1644d6b5a73eb3e74d4d64d5d8c6c3d577e753a04c9e9c87d07692c58db", + "sha256:b17be2478b622939e39b816e0aa8242611cc8d3583d1cd8ec31b249f04623243", + "sha256:c1683841cd4fa45ac427c18854c3ec3cd9b681694caf5bff04edb9387602d661", + "sha256:c23080fdeec4716aede32b4e0ef7e213c7b1093eede9ee010949f2a418ced6ba", + "sha256:d5b5b962221fa2c5d3a7f8133f9abffc114fe218eb4365e40f17732ade576c8e", + "sha256:d603de2b8d2ea3f3bcb2efe286849aa7a81531abc52d8454da12f46235092bcb", + "sha256:e83f80a7fec1a62cf4e6c9a660e39c7f878f603737a0cdac8c13131d11d97f52", + "sha256:eb514ad14edf07a1dbe63761fd30f89ae79b42625731e1ccf5e1f1092950eaa6", + "sha256:eba96145051ccec0ec86611fe9cf693ce55f2a3ce89c06ed307de0e085730ec1", + "sha256:ed6f7b854a823ea44cf94919ba3f727e230da29feb4a99711433f25800cf747f", + "sha256:f0029245c51fd9473dc1aede1160b0a29f4a912e6b1dd353fa6d317085b219da", + "sha256:f5d869c18f030202eb412f08b28d2afeea553d6613aee89e200d7aca7ef01f5f", + "sha256:fb62ea4b62bfcb0b380d5680f9a4b3f9a2d166d9394e9bbd9666c0ee09a3645c", + "sha256:fcb8a47f43acc113e24e910399376f7277cf8508b27e5b88499f053de6b115a8" + ], + "version": "==1.0.4" + }, + "packageurl-python": { + "hashes": [ + "sha256:5c91334f942cd55d45eb0c67dd339a535ef90e25f05b9ec016ad188ed0ef9048", + "sha256:bf8a1ffe755634776f6563904d792fb0aa13b377fc86115c36fe17f69b6e59db" + ], + "markers": "python_version >= '3.6'", + "version": "==0.10.4" + }, "packaging": { "hashes": [ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", @@ -1657,6 +1787,46 @@ "markers": "python_version >= '3.6'", "version": "==21.3" }, + "pbr": { + "hashes": [ + "sha256:b97bc6695b2aff02144133c2e7399d5885223d42b7912ffaec2ca3898e673bfe", + "sha256:db2317ff07c84c4c63648c9064a79fe9d9f5c7ce85a9099d4b6258b3db83225a" + ], + "markers": "python_version >= '2.6'", + "version": "==5.11.0" + }, + "pip": { + "hashes": [ + "sha256:1daab4b8d3b97d1d763caeb01a4640a2250a0ea899e257b1e44b9eded91e15ab", + "sha256:8182aec21dad6c0a49a2a3d121a87cd524b950e0b6092b181625f07ebdde7530" + ], + "markers": "python_version >= '3.7'", + "version": "==22.3" + }, + "pip-api": { + "hashes": [ + "sha256:2a0314bd31522eb9ffe8a99668b0d07fee34ebc537931e7b6483001dbedcbdc9", + "sha256:a05df2c7aa9b7157374bcf4273544201a0c7bae60a9c65bcf84f3959ef3896f3" + ], + "markers": "python_version >= '3.7'", + "version": "==0.0.30" + }, + "pip-audit": { + "hashes": [ + "sha256:a6205bb586f5964325b1af888914bf547d91f588a5f5b2c4d73f04a39fcc276f", + "sha256:cc7be2253f80dba44e8ae0002c26bf920114436b0097bfd483581c8b607caae2" + ], + "index": "pypi", + "version": "==2.4.4" + }, + "pip-requirements-parser": { + "hashes": [ + "sha256:22fa213a987913385b2484d5698ecfa1d9cf4154978cdf929085548af55355b0", + "sha256:8c2a6f8e091ac2693824a5ef4e3b250226e34f74a20a91a87b9ab0714b47788f" + ], + "markers": "python_version >= '3.6'", + "version": "==31.2.0" + }, "pluggy": { "hashes": [ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", @@ -1696,6 +1866,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.0" }, + "pygments": { + "hashes": [ + "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1", + "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42" + ], + "markers": "python_version >= '3.6'", + "version": "==2.13.0" + }, "pyparsing": { "hashes": [ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", @@ -1756,7 +1934,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "pytz": { @@ -1817,6 +1995,13 @@ "index": "pypi", "version": "==1.9.3" }, + "resolvelib": { + "hashes": [ + "sha256:c6ea56732e9fb6fca1b2acc2ccc68a0b6b8c566d8f3e78e0443310ede61dbd37", + "sha256:d9b7907f055c3b3a2cfc56c914ffd940122915826ff5fb5b1de0c99778f4de98" + ], + "version": "==0.8.1" + }, "responses": { "hashes": [ "sha256:396acb2a13d25297789a5866b4881cf4e46ffd49cc26c43ab1117f40b973102e", @@ -1825,6 +2010,14 @@ "markers": "python_version >= '3.7'", "version": "==0.22.0" }, + "rich": { + "hashes": [ + "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e", + "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0" + ], + "markers": "python_full_version >= '3.6.3' and python_full_version < '4.0.0'", + "version": "==12.6.0" + }, "s3transfer": { "hashes": [ "sha256:7a6f4c4d1fdb9a2b640244008e142cbc2cd3ae34b386584ef044dd0f27101971", @@ -1833,20 +2026,51 @@ "markers": "python_version >= '3.6'", "version": "==0.5.2" }, + "setuptools": { + "hashes": [ + "sha256:512e5536220e38146176efb833d4a62aa726b7bbff82cfbc8ba9eaa3996e0b17", + "sha256:f62ea9da9ed6289bfe868cd6845968a2c854d1427f8548d52cae02a42b4f0356" + ], + "markers": "python_version >= '3.7'", + "version": "==65.5.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, + "smmap": { + "hashes": [ + "sha256:2aba19d6a040e78d8b09de5c57e96207b09ed71d8e55ce0959eeee6c8e190d94", + "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, + "sortedcontainers": { + "hashes": [ + "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", + "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0" + ], + "version": "==2.4.0" + }, + "stevedore": { + "hashes": [ + "sha256:02518a8f0d6d29be8a445b7f2ac63753ff29e8f2a2faa01777568d5500d777a6", + "sha256:3b1cbd592a87315f000d05164941ee5e164899f8fc0ce9a00bb0f321f40ef93e" + ], + "markers": "python_version >= '3.8'", + "version": "==4.1.0" + }, "toml": { "hashes": [ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tomli": { @@ -1872,6 +2096,13 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", "version": "==1.26.12" }, + "webencodings": { + "hashes": [ + "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", + "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" + ], + "version": "==0.5.1" + }, "werkzeug": { "hashes": [ "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6", diff --git a/devcontainer-api/scripts/notify-dev-entrypoint.sh b/devcontainer-api/scripts/notify-dev-entrypoint.sh index bd7b7a30b..469bbfffe 100755 --- a/devcontainer-api/scripts/notify-dev-entrypoint.sh +++ b/devcontainer-api/scripts/notify-dev-entrypoint.sh @@ -30,14 +30,13 @@ cd /workspace git status make generate-version-file -pip3 install -r requirements.txt -pip3 install -r requirements_for_test.txt +pipenv install --dev # Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer pip3 install virtualenv # Upgrade schema of the notification_api database -flask db upgrade +pipenv run flask db upgrade # Run flask server # make run-flask diff --git a/devcontainer-api/scripts/notify-worker-entrypoint.sh b/devcontainer-api/scripts/notify-worker-entrypoint.sh index 98251ef15..2af1696f1 100755 --- a/devcontainer-api/scripts/notify-worker-entrypoint.sh +++ b/devcontainer-api/scripts/notify-worker-entrypoint.sh @@ -30,8 +30,7 @@ cd /workspace git status make generate-version-file -pip3 install -r requirements.txt -pip3 install -r requirements_for_test.txt +pipenv install --dev # Install virtualenv to support running the isolated make freeze-requirements from within the devcontainer pip3 install virtualenv From 9b32bb55eac8216b418191b6048723961237e643 Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 28 Oct 2022 11:11:17 -0400 Subject: [PATCH 64/65] more ci info --- README.md | 1 + docs/deploying.md | 18 ++++++++++++++++++ docs/testing.md | 2 ++ 3 files changed, 21 insertions(+) create mode 100644 docs/deploying.md diff --git a/README.md b/README.md index 73877fa92..16b93b3d3 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ Our other repositories are: - [Local setup](#local-setup) - [Testing](./docs/testing.md) +- [Deploying](./docs/deploying.md) - [Running one-off tasks](./docs/one-off-tasks.md) ## UK docs that may still be helpful diff --git a/docs/deploying.md b/docs/deploying.md new file mode 100644 index 000000000..5fa129b01 --- /dev/null +++ b/docs/deploying.md @@ -0,0 +1,18 @@ +# Deploying + +We deploy automatically to cloud.gov for production and staging environments. + +Deployment runs via the [deployment action](../.github/workflows/deploy.yml) on GitHub, which pulls credentials from GitHub's secrets store. + +The [action that we use](https://github.com/18F/cg-deploy-action) deploys using [a rolling strategy](https://docs.cloudfoundry.org/devguide/deploy-apps/rolling-deploy.html), so all deployments should have zero downtime. + +The API has 2 deployment environments: + +- Production, which deploys from `main` +- Staging, which does not, in fact, exist + +Configurations for these are located in [the `deploy-config` folder](../deploy-config/). + +In the event that a deployment includes a Terraform change, that change will run before any code is deployed to the environment. Each environment has its own Terraform GitHub Action to handle that change. + +Failures in any of these GitHub workflows will be surfaced in the Pull Request related to the code change, and in the case of `checks.yml` actively prevent the PR from being merged. Failure in the Terraform workflow will not actively prevent the PR from being merged, but reviewers should not approve a PR with a failing terraform plan. \ No newline at end of file diff --git a/docs/testing.md b/docs/testing.md index 2294c52cf..90c5bdb2d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,6 +21,8 @@ On GitHub, in addition to these tests, we run: We're using GitHub Actions. See [/.github](../.github/) for the configuration. +In addition to commit-triggered scans, the `daily_checks.yml` workflow runs the relevant dependency audits, static scan, and/or dynamic scans at 10am UTC each day. Developers will be notified of failures in daily scans by GitHub notifications. + ## To run a local OWASP scan 1. Run `make run-flask` from within the dev container. From 64f8641013a84028a3177debf2790256e4bd0bfe Mon Sep 17 00:00:00 2001 From: stvnrlly Date: Fri, 28 Oct 2022 12:48:22 -0400 Subject: [PATCH 65/65] improvements from feedback --- README.md | 18 ++++++------------ docs/one-off-tasks.md | 2 +- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 16b93b3d3..79a682726 100644 --- a/README.md +++ b/README.md @@ -43,15 +43,13 @@ Our other repositories are: ### Direct installation -1. Set up Postgres && Redis +1. Set up Postgres && Redis on your machine + +1. Install [pipenv](https://pipenv.pypa.io/en/latest/) 1. Install dependencies into a virtual environment - ``` - pipenv install --dev - createdb notification_api - flask db upgrade - ``` + `make bootstrap` 1. Create the .env file @@ -62,15 +60,11 @@ Our other repositories are: 1. Run Flask - ``` - pipenv run make run-flask - ``` + `make run-flask` 1. Run Celery - ``` - pipenv run make run-celery - ``` + `make run-celery` ### VS Code && Docker installation diff --git a/docs/one-off-tasks.md b/docs/one-off-tasks.md index edbfcefea..a337eaf01 100644 --- a/docs/one-off-tasks.md +++ b/docs/one-off-tasks.md @@ -8,7 +8,7 @@ both with `pytest` and with trial runs. To run a command on cloud.gov, use this format: ``` -cf run-task CLOUD-GOV-SPACE --commmand "YOUR COMMAND HERE" --name YOUR-COMMAND +cf run-task CLOUD-GOV-APP --commmand "YOUR COMMAND HERE" --name YOUR-COMMAND ``` [Here's more documentation](https://docs.cloudfoundry.org/devguide/using-tasks.html) about Cloud Foundry tasks.