From c04d1df6b3f9a010c7c7361c72ecc9ecac3d0f1d Mon Sep 17 00:00:00 2001 From: jimmoffet Date: Mon, 3 Oct 2022 17:16:59 -0700 Subject: [PATCH] fixing tests --- app/__init__.py | 4 +- app/celery/nightly_tasks.py | 4 +- app/celery/process_ses_receipts_tasks.py | 186 +++++++++++++----- .../process_sms_client_response_tasks.py | 4 +- .../notifications_ses_callback.py | 143 +++----------- .../notifications_sms_callback.py | 1 + app/notifications/receive_notifications.py | 142 ++++++------- .../sns_cert_validator.py} | 0 .../sns_handlers.py} | 4 +- devcontainer-api/.devcontainer.json | 3 +- migrations/versions/0375_fix_service_name.py | 8 +- .../versions/0377_add_inbound_sms_number.py | 52 ++--- .../celery/test_process_ses_receipts_tasks.py | 16 +- .../test_notifications_ses_callback.py | 6 +- 14 files changed, 279 insertions(+), 294 deletions(-) rename app/{celery/validate_sns_cert.py => notifications/sns_cert_validator.py} (100%) rename app/{celery/validate_sns_message.py => notifications/sns_handlers.py} (93%) diff --git a/app/__init__.py b/app/__init__.py index 035ce112a..4d3e47721 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -139,7 +139,6 @@ def register_blueprint(application): ) from app.billing.rest import billing_blueprint from app.broadcast_message.rest import broadcast_message_blueprint - from app.celery.process_ses_receipts_tasks import ses_callback_blueprint from app.complaint.complaint_rest import complaint_blueprint from app.email_branding.rest import email_branding_blueprint from app.events.rest import events as events_blueprint @@ -154,6 +153,9 @@ def register_blueprint(application): from app.notifications.notifications_letter_callback import ( letter_callback_blueprint, ) + from app.notifications.notifications_ses_callback import ( + ses_callback_blueprint, + ) from app.notifications.notifications_sms_callback import ( sms_callback_blueprint, ) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index bcca9818e..4dd8f1a78 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -11,6 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError from app import notify_celery, statsd_client, zendesk_client from app.aws import s3 +from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task from app.config import QueueNames from app.cronitor import cronitor from app.dao.fact_processing_time_dao import insert_update_processing_time @@ -37,9 +38,6 @@ from app.models import ( FactProcessingTime, Notification, ) -from app.notifications.notifications_ses_callback import ( - check_and_queue_callback_task, -) from app.utils import get_london_midnight_in_utc diff --git a/app/celery/process_ses_receipts_tasks.py b/app/celery/process_ses_receipts_tasks.py index 6c101893c..95aa92a86 100644 --- a/app/celery/process_ses_receipts_tasks.py +++ b/app/celery/process_ses_receipts_tasks.py @@ -6,64 +6,26 @@ from json import decoder import iso8601 import requests from celery.exceptions import Retry -from flask import Blueprint, current_app, json, jsonify, request +from flask import current_app, json from sqlalchemy.orm.exc import NoResultFound from app import notify_celery, statsd_client -from app.celery.validate_sns_message import sns_notification_handler +from app.celery.service_callback_tasks import ( + create_complaint_callback_data, + create_delivery_status_callback_data, + send_complaint_to_service, + send_delivery_status_to_service, +) from app.config import QueueNames from app.dao import notifications_dao -from app.errors import InvalidRequest, register_errors -from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING -from app.notifications.notifications_ses_callback import ( - _check_and_queue_complaint_callback_task, - check_and_queue_callback_task, - determine_notification_bounce_type, - get_aws_responses, - handle_complaint, +from app.dao.complaint_dao import save_complaint +from app.dao.notifications_dao import dao_get_notification_history_by_reference +from app.dao.service_callback_api_dao import ( + get_service_complaint_callback_api_for_service, + get_service_delivery_status_callback_api_for_service, ) - -ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) -DEFAULT_MAX_AGE = timedelta(days=10000) - -register_errors(ses_callback_blueprint) -class SNSMessageType(enum.Enum): - SubscriptionConfirmation = 'SubscriptionConfirmation' - Notification = 'Notification' - UnsubscribeConfirmation = 'UnsubscribeConfirmation' - - -class InvalidMessageTypeException(Exception): - pass - - -def verify_message_type(message_type: str): - try: - SNSMessageType(message_type) - except ValueError: - raise InvalidMessageTypeException(f'{message_type} is not a valid message type.') - - -# 400 counts as a permanent failure so SNS will not retry. -# 500 counts as a failed delivery attempt so SNS will retry. -# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies -# This should not be here, it used to be in notifications/notifications_ses_callback. It then -# got refactored into a task, which is fine, but it created a circular dependency. Will need -# to investigate why GDS extracted this into a lambda -@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) -def email_ses_callback_handler(): - try: - data = sns_notification_handler(request.data, request.headers) - except Exception as e: - raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) - - message = data.get("Message") - if "mail" in message: - process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) - - return jsonify( - result="success", message="SES-SNS callback succeeded" - ), 200 +from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint +from app.notifications.callbacks import create_complaint_callback_data @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) @@ -145,3 +107,123 @@ def process_ses_results(self, response): current_app.logger.exception("Error processing SES results: {}".format(type(e))) self.retry(queue=QueueNames.RETRY) +def determine_notification_bounce_type(ses_message): + notification_type = ses_message["notificationType"] + if notification_type in ["Delivery", "Complaint"]: + return notification_type + + if notification_type != "Bounce": + raise KeyError(f"Unhandled notification type {notification_type}") + + remove_emails_from_bounce(ses_message) + current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + if ses_message["bounce"]["bounceType"] == "Permanent": + return "Permanent" + return "Temporary" + + +def _determine_provider_response(ses_message): + if ses_message["notificationType"] != "Bounce": + return None + + bounce_type = ses_message["bounce"]["bounceType"] + bounce_subtype = ses_message["bounce"]["bounceSubType"] + + # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html + if bounce_type == "Permanent" and bounce_subtype == "Suppressed": + return "The email address is on our email provider suppression list" + elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList": + return "The email address is on the GC Notify suppression list" + elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected": + return "The email was rejected because of its attachments" + + return None + + +def get_aws_responses(ses_message): + status = determine_notification_bounce_type(ses_message) + + base = { + "Permanent": { + "message": "Hard bounced", + "success": False, + "notification_status": "permanent-failure", + }, + "Temporary": { + "message": "Soft bounced", + "success": False, + "notification_status": "temporary-failure", + }, + "Delivery": { + "message": "Delivered", + "success": True, + "notification_status": "delivered", + }, + "Complaint": { + "message": "Complaint", + "success": True, + "notification_status": "delivered", + }, + }[status] + + base["provider_response"] = _determine_provider_response(ses_message) + + return base + + +def handle_complaint(ses_message): + recipient_email = remove_emails_from_complaint(ses_message)[0] + current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) + try: + reference = ses_message["mail"]["messageId"] + except KeyError as e: + current_app.logger.exception("Complaint from SES failed to get reference from message", e) + return + notification = dao_get_notification_history_by_reference(reference) + ses_complaint = ses_message.get("complaint", None) + + complaint = Complaint( + notification_id=notification.id, + service_id=notification.service_id, + ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None, + complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None, + complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None, + ) + save_complaint(complaint) + return complaint, notification, recipient_email + + +def remove_mail_headers(dict_to_edit): + if dict_to_edit["mail"].get("headers"): + dict_to_edit["mail"].pop("headers") + if dict_to_edit["mail"].get("commonHeaders"): + dict_to_edit["mail"].pop("commonHeaders") + + +def remove_emails_from_bounce(bounce_dict): + remove_mail_headers(bounce_dict) + bounce_dict["mail"].pop("destination", None) + bounce_dict["bounce"].pop("bouncedRecipients", None) + + +def remove_emails_from_complaint(complaint_dict): + remove_mail_headers(complaint_dict) + complaint_dict["complaint"].pop("complainedRecipients") + return complaint_dict["mail"].pop("destination") + + +def check_and_queue_callback_task(notification): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + notification_data = create_delivery_status_callback_data(notification, service_callback_api) + send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) + + +def _check_and_queue_complaint_callback_task(complaint, notification, recipient): + # queue callback task only if the service_callback_api exists + service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) + if service_callback_api: + complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) + send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) + \ No newline at end of file diff --git a/app/celery/process_sms_client_response_tasks.py b/app/celery/process_sms_client_response_tasks.py index e7726f305..4da60d477 100644 --- a/app/celery/process_sms_client_response_tasks.py +++ b/app/celery/process_sms_client_response_tasks.py @@ -6,13 +6,11 @@ from flask import current_app from notifications_utils.template import SMSMessageTemplate from app import notify_celery, statsd_client +from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task from app.clients import ClientException from app.dao import notifications_dao from app.dao.templates_dao import dao_get_template_by_id from app.models import NOTIFICATION_PENDING -from app.notifications.notifications_ses_callback import ( - check_and_queue_callback_task, -) sms_response_mapper = { # 'MMG': get_mmg_responses, diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py index 472891418..d2437b92c 100644 --- a/app/notifications/notifications_ses_callback.py +++ b/app/notifications/notifications_ses_callback.py @@ -1,5 +1,9 @@ -from flask import current_app, json +import enum +from datetime import timedelta +from flask import Blueprint, current_app, json, jsonify, request + +from app.celery.process_ses_receipts_tasks import process_ses_results from app.celery.service_callback_tasks import ( create_complaint_callback_data, create_delivery_status_callback_data, @@ -13,127 +17,28 @@ from app.dao.service_callback_api_dao import ( get_service_complaint_callback_api_for_service, get_service_delivery_status_callback_api_for_service, ) +from app.errors import InvalidRequest from app.models import Complaint from app.notifications.callbacks import create_complaint_callback_data +from app.notifications.sns_handlers import sns_notification_handler +ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) +DEFAULT_MAX_AGE = timedelta(days=10000) -def determine_notification_bounce_type(ses_message): - notification_type = ses_message["notificationType"] - if notification_type in ["Delivery", "Complaint"]: - return notification_type - - if notification_type != "Bounce": - raise KeyError(f"Unhandled notification type {notification_type}") - - remove_emails_from_bounce(ses_message) - current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) - if ses_message["bounce"]["bounceType"] == "Permanent": - return "Permanent" - return "Temporary" - - -def _determine_provider_response(ses_message): - if ses_message["notificationType"] != "Bounce": - return None - - bounce_type = ses_message["bounce"]["bounceType"] - bounce_subtype = ses_message["bounce"]["bounceSubType"] - - # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html - if bounce_type == "Permanent" and bounce_subtype == "Suppressed": - return "The email address is on our email provider suppression list" - elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList": - return "The email address is on the GC Notify suppression list" - elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected": - return "The email was rejected because of its attachments" - - return None - - -def get_aws_responses(ses_message): - status = determine_notification_bounce_type(ses_message) - - base = { - "Permanent": { - "message": "Hard bounced", - "success": False, - "notification_status": "permanent-failure", - }, - "Temporary": { - "message": "Soft bounced", - "success": False, - "notification_status": "temporary-failure", - }, - "Delivery": { - "message": "Delivered", - "success": True, - "notification_status": "delivered", - }, - "Complaint": { - "message": "Complaint", - "success": True, - "notification_status": "delivered", - }, - }[status] - - base["provider_response"] = _determine_provider_response(ses_message) - - return base - - -def handle_complaint(ses_message): - recipient_email = remove_emails_from_complaint(ses_message)[0] - current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")"))) +# 400 counts as a permanent failure so SNS will not retry. +# 500 counts as a failed delivery attempt so SNS will retry. +# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies +@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST']) +def email_ses_callback_handler(): try: - reference = ses_message["mail"]["messageId"] - except KeyError as e: - current_app.logger.exception("Complaint from SES failed to get reference from message", e) - return - notification = dao_get_notification_history_by_reference(reference) - ses_complaint = ses_message.get("complaint", None) + data = sns_notification_handler(request.data, request.headers) + except Exception as e: + raise InvalidRequest("SES-SNS callback failed: invalid message type", 400) + + message = data.get("Message") + if "mail" in message: + process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY) - complaint = Complaint( - notification_id=notification.id, - service_id=notification.service_id, - ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None, - complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None, - complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None, - ) - save_complaint(complaint) - return complaint, notification, recipient_email - - -def remove_mail_headers(dict_to_edit): - if dict_to_edit["mail"].get("headers"): - dict_to_edit["mail"].pop("headers") - if dict_to_edit["mail"].get("commonHeaders"): - dict_to_edit["mail"].pop("commonHeaders") - - -def remove_emails_from_bounce(bounce_dict): - remove_mail_headers(bounce_dict) - bounce_dict["mail"].pop("destination", None) - bounce_dict["bounce"].pop("bouncedRecipients", None) - - -def remove_emails_from_complaint(complaint_dict): - remove_mail_headers(complaint_dict) - complaint_dict["complaint"].pop("complainedRecipients") - return complaint_dict["mail"].pop("destination") - - -def check_and_queue_callback_task(notification): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - notification_data = create_delivery_status_callback_data(notification, service_callback_api) - send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS) - - -def _check_and_queue_complaint_callback_task(complaint, notification, recipient): - # queue callback task only if the service_callback_api exists - service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id) - if service_callback_api: - complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient) - send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS) - \ No newline at end of file + return jsonify( + result="success", message="SES-SNS callback succeeded" + ), 200 diff --git a/app/notifications/notifications_sms_callback.py b/app/notifications/notifications_sms_callback.py index cb221d08c..50c345f49 100644 --- a/app/notifications/notifications_sms_callback.py +++ b/app/notifications/notifications_sms_callback.py @@ -9,6 +9,7 @@ from app.errors import InvalidRequest, register_errors sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms") register_errors(sms_callback_blueprint) +# TODO SNS SMS delivery receipts delivered here # @sms_callback_blueprint.route('/mmg', methods=['POST']) # def process_mmg_response(): diff --git a/app/notifications/receive_notifications.py b/app/notifications/receive_notifications.py index d7f94e755..4a7ecab50 100644 --- a/app/notifications/receive_notifications.py +++ b/app/notifications/receive_notifications.py @@ -2,17 +2,17 @@ from datetime import datetime from urllib.parse import unquote import iso8601 -from flask import Blueprint, abort, current_app, jsonify, request, json +from flask import Blueprint, abort, current_app, json, jsonify, request from gds_metrics.metrics import Counter from notifications_utils.recipients import try_validate_and_format_phone_number from app.celery import tasks -from app.celery.validate_sns_message import sns_notification_handler from app.config import QueueNames from app.dao.inbound_sms_dao import dao_create_inbound_sms from app.dao.services_dao import dao_fetch_service_by_inbound_number -from app.errors import register_errors, InvalidRequest +from app.errors import InvalidRequest, register_errors from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms +from app.notifications.sns_handlers import sns_notification_handler receive_notifications_blueprint = Blueprint('receive_notifications', __name__) register_errors(receive_notifications_blueprint) @@ -83,90 +83,90 @@ def receive_sns_sms(): ), 200 -@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) -def receive_mmg_sms(): - """ - { - 'MSISDN': '447123456789' - 'Number': '40604', - 'Message': 'some+uri+encoded+message%3A', - 'ID': 'SOME-MMG-SPECIFIC-ID', - 'DateRecieved': '2017-05-21+11%3A56%3A11' - } - """ - post_data = request.get_json() +# @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) +# def receive_mmg_sms(): +# """ +# { +# 'MSISDN': '447123456789' +# 'Number': '40604', +# 'Message': 'some+uri+encoded+message%3A', +# 'ID': 'SOME-MMG-SPECIFIC-ID', +# 'DateRecieved': '2017-05-21+11%3A56%3A11' +# } +# """ +# post_data = request.get_json() - auth = request.authorization +# auth = request.authorization - if not auth: - current_app.logger.warning("Inbound sms (MMG) no auth header") - abort(401) - elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ - or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: - current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) - abort(403) +# if not auth: +# current_app.logger.warning("Inbound sms (MMG) no auth header") +# abort(401) +# elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ +# or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: +# current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) +# abort(403) - inbound_number = strip_leading_forty_four(post_data['Number']) +# inbound_number = strip_leading_forty_four(post_data['Number']) - service = fetch_potential_service(inbound_number, 'mmg') - if not service: - # since this is an issue with our service <-> number mapping, or no inbound_sms service permission - # we should still tell MMG that we received it successfully - return 'RECEIVED', 200 +# service = fetch_potential_service(inbound_number, 'mmg') +# if not service: +# # since this is an issue with our service <-> number mapping, or no inbound_sms service permission +# # we should still tell MMG that we received it successfully +# return 'RECEIVED', 200 - INBOUND_SMS_COUNTER.labels("mmg").inc() +# INBOUND_SMS_COUNTER.labels("mmg").inc() - inbound = create_inbound_sms_object(service, - content=format_mmg_message(post_data["Message"]), - from_number=post_data['MSISDN'], - provider_ref=post_data["ID"], - date_received=post_data.get('DateRecieved'), - provider_name="mmg") +# inbound = create_inbound_sms_object(service, +# content=format_mmg_message(post_data["Message"]), +# from_number=post_data['MSISDN'], +# provider_ref=post_data["ID"], +# date_received=post_data.get('DateRecieved'), +# provider_name="mmg") - tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) +# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) - current_app.logger.debug( - '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) - return jsonify({ - "status": "ok" - }), 200 +# current_app.logger.debug( +# '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) +# return jsonify({ +# "status": "ok" +# }), 200 -@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) -def receive_firetext_sms(): - post_data = request.form +# @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) +# def receive_firetext_sms(): +# post_data = request.form - auth = request.authorization - if not auth: - current_app.logger.warning("Inbound sms (Firetext) no auth header") - abort(401) - elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: - current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) - abort(403) +# auth = request.authorization +# if not auth: +# current_app.logger.warning("Inbound sms (Firetext) no auth header") +# abort(401) +# elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: +# current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) +# abort(403) - inbound_number = strip_leading_forty_four(post_data['destination']) +# inbound_number = strip_leading_forty_four(post_data['destination']) - service = fetch_potential_service(inbound_number, 'firetext') - if not service: - return jsonify({ - "status": "ok" - }), 200 +# service = fetch_potential_service(inbound_number, 'firetext') +# if not service: +# return jsonify({ +# "status": "ok" +# }), 200 - inbound = create_inbound_sms_object(service=service, - content=post_data["message"], - from_number=post_data['source'], - provider_ref=None, - date_received=post_data['time'], - provider_name="firetext") +# inbound = create_inbound_sms_object(service=service, +# content=post_data["message"], +# from_number=post_data['source'], +# provider_ref=None, +# date_received=post_data['time'], +# provider_name="firetext") - INBOUND_SMS_COUNTER.labels("firetext").inc() +# INBOUND_SMS_COUNTER.labels("firetext").inc() - tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) - current_app.logger.debug( - '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) - return jsonify({ - "status": "ok" - }), 200 +# tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) +# current_app.logger.debug( +# '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) +# return jsonify({ +# "status": "ok" +# }), 200 def format_mmg_message(message): diff --git a/app/celery/validate_sns_cert.py b/app/notifications/sns_cert_validator.py similarity index 100% rename from app/celery/validate_sns_cert.py rename to app/notifications/sns_cert_validator.py diff --git a/app/celery/validate_sns_message.py b/app/notifications/sns_handlers.py similarity index 93% rename from app/celery/validate_sns_message.py rename to app/notifications/sns_handlers.py index 37295dc7d..a0a6d3b98 100644 --- a/app/celery/validate_sns_message.py +++ b/app/notifications/sns_handlers.py @@ -5,9 +5,8 @@ from json import decoder import requests from flask import current_app, json -from app.celery.validate_sns_cert import validate_sns_cert from app.errors import InvalidRequest - +from app.notifications.sns_cert_validator import validate_sns_cert DEFAULT_MAX_AGE = timedelta(days=10000) @@ -50,6 +49,7 @@ def sns_notification_handler(data, headers): raise InvalidRequest("SES-SNS callback failed: validation failed", 400) if message.get('Type') == 'SubscriptionConfirmation': + # NOTE once a request is sent to SubscribeURL, AWS considers Notify a confirmed subscriber to this topic url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') response = requests.get(url) try: diff --git a/devcontainer-api/.devcontainer.json b/devcontainer-api/.devcontainer.json index 1d9b2d3d9..bea5f3978 100644 --- a/devcontainer-api/.devcontainer.json +++ b/devcontainer-api/.devcontainer.json @@ -16,7 +16,8 @@ "python.defaultInterpreterPath": "/usr/bin/python3", "python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint", "python.analysis.extraPaths": [ - "/home/vscode/.local/lib/python3.9/site-packages" + "/home/vscode/.local/lib/python3.9/site-packages", + "/home/vscode/.local/bin" ] }, "features": { diff --git a/migrations/versions/0375_fix_service_name.py b/migrations/versions/0375_fix_service_name.py index 6ec34574f..72e93e8ca 100644 --- a/migrations/versions/0375_fix_service_name.py +++ b/migrations/versions/0375_fix_service_name.py @@ -6,17 +6,13 @@ Create Date: 2022-08-29 11:04:15.888017 """ -# revision identifiers, used by Alembic. -from datetime import datetime - revision = '0375_fix_service_name' down_revision = '0374_fix_reg_template_history' from alembic import op -import sqlalchemy as sa +from flask import current_app -service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' -user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6' +service_id = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): op.get_bind() diff --git a/migrations/versions/0377_add_inbound_sms_number.py b/migrations/versions/0377_add_inbound_sms_number.py index b3641d843..6b4a74044 100644 --- a/migrations/versions/0377_add_inbound_sms_number.py +++ b/migrations/versions/0377_add_inbound_sms_number.py @@ -19,34 +19,36 @@ INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER'] DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] def upgrade(): - op.get_bind() + # op.get_bind() - # add the inbound number for the default service to inbound_numbers - table_name = 'inbound_numbers' - provider = 'sns' - active = 'true' - op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") + # # add the inbound number for the default service to inbound_numbers + # table_name = 'inbound_numbers' + # provider = 'sns' + # active = 'true' + # op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") - # add the inbound number for the default service to service_sms_senders - table_name = 'service_sms_senders' - id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' - is_default = 'true' - sms_sender = INBOUND_NUMBER - inbound_number_id = INBOUND_NUMBER_ID - archived = 'false' - op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") + # # add the inbound number for the default service to service_sms_senders + # table_name = 'service_sms_senders' + # id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' + # is_default = 'true' + # sms_sender = INBOUND_NUMBER + # inbound_number_id = INBOUND_NUMBER_ID + # archived = 'false' + # op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") - # add the inbound number for the default service to inbound_numbers - table_name = 'service_permissions' - permission = 'inbound_sms' - active = 'true' - op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + # # add the inbound number for the default service to inbound_numbers + # table_name = 'service_permissions' + # permission = 'inbound_sms' + # active = 'true' + # op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") + pass def downgrade(): - delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" - delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" - delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" - op.execute(delete_sms_sender) - op.execute(delete_inbound_number) - op.execute(delete_service_inbound_permission) + # delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" + # delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" + # delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" + # op.execute(delete_sms_sender) + # op.execute(delete_inbound_number) + # op.execute(delete_service_inbound_permission) + pass diff --git a/tests/app/celery/test_process_ses_receipts_tasks.py b/tests/app/celery/test_process_ses_receipts_tasks.py index c6a9bbf4b..ae6728398 100644 --- a/tests/app/celery/test_process_ses_receipts_tasks.py +++ b/tests/app/celery/test_process_ses_receipts_tasks.py @@ -4,7 +4,11 @@ from datetime import datetime from freezegun import freeze_time from app import encryption, statsd_client -from app.celery.process_ses_receipts_tasks import process_ses_results +from app.celery.process_ses_receipts_tasks import ( + process_ses_results, + remove_emails_from_bounce, + remove_emails_from_complaint, +) from app.celery.research_mode_tasks import ( ses_hard_bounce_callback, ses_notification_callback, @@ -15,10 +19,6 @@ from app.celery.service_callback_tasks import ( ) from app.dao.notifications_dao import get_notification_by_id from app.models import Complaint, Notification -from app.notifications.notifications_ses_callback import ( - remove_emails_from_bounce, - remove_emails_from_complaint, -) from tests.app.conftest import create_sample_notification from tests.app.db import ( create_notification, @@ -105,7 +105,7 @@ def test_process_ses_results(sample_email_template): assert process_ses_results(response=ses_notification_callback(reference='ref1')) -def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker): +def test_process_ses_results_retry_called(sample_email_template, mocker): create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending') mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED")) mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') @@ -178,7 +178,7 @@ def test_ses_callback_should_not_update_notification_status_if_already_delivered assert mock_upd.call_count == 0 -def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker): +def test_ses_callback_should_retry_if_notification_is_new(mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') with freeze_time('2017-11-17T12:14:03.646Z'): @@ -192,7 +192,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, assert process_ses_results(ses_notification_callback(reference='ref')) is None assert mock_retry.call_count == 0 mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)') -def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker): +def test_ses_callback_should_not_retry_if_notification_is_old(mocker): mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') with freeze_time('2017-11-21T12:14:03.646Z'): diff --git a/tests/app/notifications/test_notifications_ses_callback.py b/tests/app/notifications/test_notifications_ses_callback.py index 8e6409eeb..dcec4aa32 100644 --- a/tests/app/notifications/test_notifications_ses_callback.py +++ b/tests/app/notifications/test_notifications_ses_callback.py @@ -2,12 +2,12 @@ import pytest from flask import json from sqlalchemy.exc import SQLAlchemyError -from app.dao.notifications_dao import get_notification_by_id -from app.models import Complaint -from app.notifications.notifications_ses_callback import ( +from app.celery.process_ses_receipts_tasks import ( check_and_queue_callback_task, handle_complaint, ) +from app.dao.notifications_dao import get_notification_by_id +from app.models import Complaint from tests.app.db import ( create_notification, create_notification_history,