fixing tests

app/__init__.py

@@ -139,7 +139,6 @@ def register_blueprint(application):
     )
     from app.billing.rest import billing_blueprint
     from app.broadcast_message.rest import broadcast_message_blueprint
-    from app.celery.process_ses_receipts_tasks import ses_callback_blueprint
     from app.complaint.complaint_rest import complaint_blueprint
     from app.email_branding.rest import email_branding_blueprint
     from app.events.rest import events as events_blueprint
@@ -154,6 +153,9 @@ def register_blueprint(application):
     from app.notifications.notifications_letter_callback import (
         letter_callback_blueprint,
     )
+    from app.notifications.notifications_ses_callback import (
+        ses_callback_blueprint,
+    )
     from app.notifications.notifications_sms_callback import (
         sms_callback_blueprint,
     )

app/celery/nightly_tasks.py

@@ -11,6 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app import notify_celery, statsd_client, zendesk_client
 from app.aws import s3
+from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_processing_time_dao import insert_update_processing_time
@@ -37,9 +38,6 @@ from app.models import (
     FactProcessingTime,
     Notification,
 )
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 from app.utils import get_london_midnight_in_utc
 
 

app/celery/process_ses_receipts_tasks.py

@@ -6,64 +6,26 @@ from json import decoder
 import iso8601
 import requests
 from celery.exceptions import Retry
-from flask import Blueprint, current_app, json, jsonify, request
+from flask import current_app, json
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
-from app.celery.validate_sns_message import sns_notification_handler
+from app.celery.service_callback_tasks import (
+    create_complaint_callback_data,
+    create_delivery_status_callback_data,
+    send_complaint_to_service,
+    send_delivery_status_to_service,
+)
 from app.config import QueueNames
 from app.dao import notifications_dao
-from app.errors import InvalidRequest, register_errors
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
-from app.notifications.notifications_ses_callback import (
-    _check_and_queue_complaint_callback_task,
-    check_and_queue_callback_task,
-    determine_notification_bounce_type,
-    get_aws_responses,
-    handle_complaint,
+from app.dao.complaint_dao import save_complaint
+from app.dao.notifications_dao import dao_get_notification_history_by_reference
+from app.dao.service_callback_api_dao import (
+    get_service_complaint_callback_api_for_service,
+    get_service_delivery_status_callback_api_for_service,
 )
-
-ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
-DEFAULT_MAX_AGE = timedelta(days=10000)
-
-register_errors(ses_callback_blueprint)
-class SNSMessageType(enum.Enum):
-    SubscriptionConfirmation = 'SubscriptionConfirmation'
-    Notification = 'Notification'
-    UnsubscribeConfirmation = 'UnsubscribeConfirmation'
-
-
-class InvalidMessageTypeException(Exception):
-    pass
-
-
-def verify_message_type(message_type: str):
-    try:
-        SNSMessageType(message_type)
-    except ValueError:
-        raise InvalidMessageTypeException(f'{message_type} is not a valid message type.')
-
-
-# 400 counts as a permanent failure so SNS will not retry.
-# 500 counts as a failed delivery attempt so SNS will retry.
-# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
-# This should not be here, it used to be in notifications/notifications_ses_callback. It then
-# got refactored into a task, which is fine, but it created a circular dependency. Will need
-# to investigate why GDS extracted this into a lambda
-@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
-def email_ses_callback_handler():
-    try:
-        data = sns_notification_handler(request.data, request.headers)
-    except Exception as e:
-        raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
-    
-    message = data.get("Message")
-    if "mail" in message:
-        process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY)
-
-    return jsonify(
-        result="success", message="SES-SNS callback succeeded"
-    ), 200
+from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
+from app.notifications.callbacks import create_complaint_callback_data
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
@@ -145,3 +107,123 @@ def process_ses_results(self, response):
         current_app.logger.exception("Error processing SES results: {}".format(type(e)))
         self.retry(queue=QueueNames.RETRY)
 
+def determine_notification_bounce_type(ses_message):
+    notification_type = ses_message["notificationType"]
+    if notification_type in ["Delivery", "Complaint"]:
+        return notification_type
+
+    if notification_type != "Bounce":
+        raise KeyError(f"Unhandled notification type {notification_type}")
+
+    remove_emails_from_bounce(ses_message)
+    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
+    if ses_message["bounce"]["bounceType"] == "Permanent":
+        return "Permanent"
+    return "Temporary"
+
+
+def _determine_provider_response(ses_message):
+    if ses_message["notificationType"] != "Bounce":
+        return None
+
+    bounce_type = ses_message["bounce"]["bounceType"]
+    bounce_subtype = ses_message["bounce"]["bounceSubType"]
+
+    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
+    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
+        return "The email address is on our email provider suppression list"
+    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
+        return "The email address is on the GC Notify suppression list"
+    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
+        return "The email was rejected because of its attachments"
+
+    return None
+
+
+def get_aws_responses(ses_message):
+    status = determine_notification_bounce_type(ses_message)
+
+    base = {
+        "Permanent": {
+            "message": "Hard bounced",
+            "success": False,
+            "notification_status": "permanent-failure",
+        },
+        "Temporary": {
+            "message": "Soft bounced",
+            "success": False,
+            "notification_status": "temporary-failure",
+        },
+        "Delivery": {
+            "message": "Delivered",
+            "success": True,
+            "notification_status": "delivered",
+        },
+        "Complaint": {
+            "message": "Complaint",
+            "success": True,
+            "notification_status": "delivered",
+        },
+    }[status]
+
+    base["provider_response"] = _determine_provider_response(ses_message)
+
+    return base
+
+
+def handle_complaint(ses_message):
+    recipient_email = remove_emails_from_complaint(ses_message)[0]
+    current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
+    try:
+        reference = ses_message["mail"]["messageId"]
+    except KeyError as e:
+        current_app.logger.exception("Complaint from SES failed to get reference from message", e)
+        return
+    notification = dao_get_notification_history_by_reference(reference)
+    ses_complaint = ses_message.get("complaint", None)
+
+    complaint = Complaint(
+        notification_id=notification.id,
+        service_id=notification.service_id,
+        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
+        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
+        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
+    )
+    save_complaint(complaint)
+    return complaint, notification, recipient_email
+
+
+def remove_mail_headers(dict_to_edit):
+    if dict_to_edit["mail"].get("headers"):
+        dict_to_edit["mail"].pop("headers")
+    if dict_to_edit["mail"].get("commonHeaders"):
+        dict_to_edit["mail"].pop("commonHeaders")
+
+
+def remove_emails_from_bounce(bounce_dict):
+    remove_mail_headers(bounce_dict)
+    bounce_dict["mail"].pop("destination", None)
+    bounce_dict["bounce"].pop("bouncedRecipients", None)
+
+
+def remove_emails_from_complaint(complaint_dict):
+    remove_mail_headers(complaint_dict)
+    complaint_dict["complaint"].pop("complainedRecipients")
+    return complaint_dict["mail"].pop("destination")
+
+
+def check_and_queue_callback_task(notification):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
+        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
+
+
+def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
+        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
+        

app/celery/process_sms_client_response_tasks.py

@@ -6,13 +6,11 @@ from flask import current_app
 from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
+from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.clients import ClientException
 from app.dao import notifications_dao
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 
 sms_response_mapper = {
     # 'MMG': get_mmg_responses,

app/notifications/notifications_ses_callback.py

@@ -1,5 +1,9 @@
-from flask import current_app, json
+import enum
+from datetime import timedelta
 
+from flask import Blueprint, current_app, json, jsonify, request
+
+from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.celery.service_callback_tasks import (
     create_complaint_callback_data,
     create_delivery_status_callback_data,
@@ -13,127 +17,28 @@ from app.dao.service_callback_api_dao import (
     get_service_complaint_callback_api_for_service,
     get_service_delivery_status_callback_api_for_service,
 )
+from app.errors import InvalidRequest
 from app.models import Complaint
 from app.notifications.callbacks import create_complaint_callback_data
+from app.notifications.sns_handlers import sns_notification_handler
 
+ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
+DEFAULT_MAX_AGE = timedelta(days=10000)
 
-def determine_notification_bounce_type(ses_message):
-    notification_type = ses_message["notificationType"]
-    if notification_type in ["Delivery", "Complaint"]:
-        return notification_type
-
-    if notification_type != "Bounce":
-        raise KeyError(f"Unhandled notification type {notification_type}")
-
-    remove_emails_from_bounce(ses_message)
-    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
-    if ses_message["bounce"]["bounceType"] == "Permanent":
-        return "Permanent"
-    return "Temporary"
-
-
-def _determine_provider_response(ses_message):
-    if ses_message["notificationType"] != "Bounce":
-        return None
-
-    bounce_type = ses_message["bounce"]["bounceType"]
-    bounce_subtype = ses_message["bounce"]["bounceSubType"]
-
-    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
-    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
-        return "The email address is on our email provider suppression list"
-    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
-        return "The email address is on the GC Notify suppression list"
-    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
-        return "The email was rejected because of its attachments"
-
-    return None
-
-
-def get_aws_responses(ses_message):
-    status = determine_notification_bounce_type(ses_message)
-
-    base = {
-        "Permanent": {
-            "message": "Hard bounced",
-            "success": False,
-            "notification_status": "permanent-failure",
-        },
-        "Temporary": {
-            "message": "Soft bounced",
-            "success": False,
-            "notification_status": "temporary-failure",
-        },
-        "Delivery": {
-            "message": "Delivered",
-            "success": True,
-            "notification_status": "delivered",
-        },
-        "Complaint": {
-            "message": "Complaint",
-            "success": True,
-            "notification_status": "delivered",
-        },
-    }[status]
-
-    base["provider_response"] = _determine_provider_response(ses_message)
-
-    return base
-
-
-def handle_complaint(ses_message):
-    recipient_email = remove_emails_from_complaint(ses_message)[0]
-    current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
+# 400 counts as a permanent failure so SNS will not retry.
+# 500 counts as a failed delivery attempt so SNS will retry.
+# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
+@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
+def email_ses_callback_handler():
     try:
-        reference = ses_message["mail"]["messageId"]
-    except KeyError as e:
-        current_app.logger.exception("Complaint from SES failed to get reference from message", e)
-        return
-    notification = dao_get_notification_history_by_reference(reference)
-    ses_complaint = ses_message.get("complaint", None)
+        data = sns_notification_handler(request.data, request.headers)
+    except Exception as e:
+        raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
     
-    complaint = Complaint(
-        notification_id=notification.id,
-        service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
-        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
-        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
-    )
-    save_complaint(complaint)
-    return complaint, notification, recipient_email
-
-
-def remove_mail_headers(dict_to_edit):
-    if dict_to_edit["mail"].get("headers"):
-        dict_to_edit["mail"].pop("headers")
-    if dict_to_edit["mail"].get("commonHeaders"):
-        dict_to_edit["mail"].pop("commonHeaders")
-
-
-def remove_emails_from_bounce(bounce_dict):
-    remove_mail_headers(bounce_dict)
-    bounce_dict["mail"].pop("destination", None)
-    bounce_dict["bounce"].pop("bouncedRecipients", None)
-
-
-def remove_emails_from_complaint(complaint_dict):
-    remove_mail_headers(complaint_dict)
-    complaint_dict["complaint"].pop("complainedRecipients")
-    return complaint_dict["mail"].pop("destination")
-
-
-def check_and_queue_callback_task(notification):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
-
-
-def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
-        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
+    message = data.get("Message")
+    if "mail" in message:
+        process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY)
 
+    return jsonify(
+        result="success", message="SES-SNS callback succeeded"
+    ), 200

app/notifications/notifications_sms_callback.py

@@ -9,6 +9,7 @@ from app.errors import InvalidRequest, register_errors
 sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
 register_errors(sms_callback_blueprint)
 
+# TODO SNS SMS delivery receipts delivered here
 
 # @sms_callback_blueprint.route('/mmg', methods=['POST'])
 # def process_mmg_response():

app/notifications/receive_notifications.py

@@ -2,17 +2,17 @@ from datetime import datetime
 from urllib.parse import unquote
 
 import iso8601
-from flask import Blueprint, abort, current_app, jsonify, request, json
+from flask import Blueprint, abort, current_app, json, jsonify, request
 from gds_metrics.metrics import Counter
 from notifications_utils.recipients import try_validate_and_format_phone_number
 
 from app.celery import tasks
-from app.celery.validate_sns_message import sns_notification_handler
 from app.config import QueueNames
 from app.dao.inbound_sms_dao import dao_create_inbound_sms
 from app.dao.services_dao import dao_fetch_service_by_inbound_number
-from app.errors import register_errors, InvalidRequest
+from app.errors import InvalidRequest, register_errors
 from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms
+from app.notifications.sns_handlers import sns_notification_handler
 
 receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
 register_errors(receive_notifications_blueprint)
@@ -83,90 +83,90 @@ def receive_sns_sms():
     ), 200
 
 
-@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
-def receive_mmg_sms():
-    """
-    {
-        'MSISDN': '447123456789'
-        'Number': '40604',
-        'Message': 'some+uri+encoded+message%3A',
-        'ID': 'SOME-MMG-SPECIFIC-ID',
-        'DateRecieved': '2017-05-21+11%3A56%3A11'
-    }
-    """
-    post_data = request.get_json()
+# @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
+# def receive_mmg_sms():
+#     """
+#     {
+#         'MSISDN': '447123456789'
+#         'Number': '40604',
+#         'Message': 'some+uri+encoded+message%3A',
+#         'ID': 'SOME-MMG-SPECIFIC-ID',
+#         'DateRecieved': '2017-05-21+11%3A56%3A11'
+#     }
+#     """
+#     post_data = request.get_json()
 
-    auth = request.authorization
+#     auth = request.authorization
 
-    if not auth:
-        current_app.logger.warning("Inbound sms (MMG) no auth header")
-        abort(401)
-    elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \
-            or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']:
-        current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username))
-        abort(403)
+#     if not auth:
+#         current_app.logger.warning("Inbound sms (MMG) no auth header")
+#         abort(401)
+#     elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \
+#             or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']:
+#         current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username))
+#         abort(403)
 
-    inbound_number = strip_leading_forty_four(post_data['Number'])
+#     inbound_number = strip_leading_forty_four(post_data['Number'])
 
-    service = fetch_potential_service(inbound_number, 'mmg')
-    if not service:
-        # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
-        # we should still tell MMG that we received it successfully
-        return 'RECEIVED', 200
+#     service = fetch_potential_service(inbound_number, 'mmg')
+#     if not service:
+#         # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
+#         # we should still tell MMG that we received it successfully
+#         return 'RECEIVED', 200
 
-    INBOUND_SMS_COUNTER.labels("mmg").inc()
+#     INBOUND_SMS_COUNTER.labels("mmg").inc()
 
-    inbound = create_inbound_sms_object(service,
-                                        content=format_mmg_message(post_data["Message"]),
-                                        from_number=post_data['MSISDN'],
-                                        provider_ref=post_data["ID"],
-                                        date_received=post_data.get('DateRecieved'),
-                                        provider_name="mmg")
+#     inbound = create_inbound_sms_object(service,
+#                                         content=format_mmg_message(post_data["Message"]),
+#                                         from_number=post_data['MSISDN'],
+#                                         provider_ref=post_data["ID"],
+#                                         date_received=post_data.get('DateRecieved'),
+#                                         provider_name="mmg")
 
-    tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
+#     tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
 
-    current_app.logger.debug(
-        '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference))
-    return jsonify({
-        "status": "ok"
-    }), 200
+#     current_app.logger.debug(
+#         '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference))
+#     return jsonify({
+#         "status": "ok"
+#     }), 200
 
 
-@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
-def receive_firetext_sms():
-    post_data = request.form
+# @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
+# def receive_firetext_sms():
+#     post_data = request.form
 
-    auth = request.authorization
-    if not auth:
-        current_app.logger.warning("Inbound sms (Firetext) no auth header")
-        abort(401)
-    elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']:
-        current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username))
-        abort(403)
+#     auth = request.authorization
+#     if not auth:
+#         current_app.logger.warning("Inbound sms (Firetext) no auth header")
+#         abort(401)
+#     elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']:
+#         current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username))
+#         abort(403)
 
-    inbound_number = strip_leading_forty_four(post_data['destination'])
+#     inbound_number = strip_leading_forty_four(post_data['destination'])
 
-    service = fetch_potential_service(inbound_number, 'firetext')
-    if not service:
-        return jsonify({
-            "status": "ok"
-        }), 200
+#     service = fetch_potential_service(inbound_number, 'firetext')
+#     if not service:
+#         return jsonify({
+#             "status": "ok"
+#         }), 200
 
-    inbound = create_inbound_sms_object(service=service,
-                                        content=post_data["message"],
-                                        from_number=post_data['source'],
-                                        provider_ref=None,
-                                        date_received=post_data['time'],
-                                        provider_name="firetext")
+#     inbound = create_inbound_sms_object(service=service,
+#                                         content=post_data["message"],
+#                                         from_number=post_data['source'],
+#                                         provider_ref=None,
+#                                         date_received=post_data['time'],
+#                                         provider_name="firetext")
 
-    INBOUND_SMS_COUNTER.labels("firetext").inc()
+#     INBOUND_SMS_COUNTER.labels("firetext").inc()
 
-    tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
-    current_app.logger.debug(
-        '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference))
-    return jsonify({
-        "status": "ok"
-    }), 200
+#     tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
+#     current_app.logger.debug(
+#         '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference))
+#     return jsonify({
+#         "status": "ok"
+#     }), 200
 
 
 def format_mmg_message(message):

app/notifications/sns_handlers.py

@@ -5,9 +5,8 @@ from json import decoder
 import requests
 from flask import current_app, json
 
-from app.celery.validate_sns_cert import validate_sns_cert
 from app.errors import InvalidRequest
-
+from app.notifications.sns_cert_validator import validate_sns_cert
 
 DEFAULT_MAX_AGE = timedelta(days=10000)
 
@@ -50,6 +49,7 @@ def sns_notification_handler(data, headers):
         raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
 
     if message.get('Type') == 'SubscriptionConfirmation':
+        # NOTE once a request is sent to SubscribeURL, AWS considers Notify a confirmed subscriber to this topic
         url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
         response = requests.get(url)
         try:

devcontainer-api/.devcontainer.json

@@ -16,7 +16,8 @@
 		"python.defaultInterpreterPath": "/usr/bin/python3",
 		"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint",
 		"python.analysis.extraPaths": [
-			"/home/vscode/.local/lib/python3.9/site-packages"
+			"/home/vscode/.local/lib/python3.9/site-packages",
+			"/home/vscode/.local/bin"
 		]
 	},
 	"features": {

migrations/versions/0375_fix_service_name.py

@@ -6,17 +6,13 @@ Create Date: 2022-08-29 11:04:15.888017
 
 """
 
-# revision identifiers, used by Alembic.
-from datetime import datetime
-
 revision = '0375_fix_service_name'
 down_revision = '0374_fix_reg_template_history'
 
 from alembic import op
-import sqlalchemy as sa
+from flask import current_app
 
-service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
-user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6'
+service_id = current_app.config['NOTIFY_SERVICE_ID']
 
 def upgrade():
     op.get_bind()

migrations/versions/0377_add_inbound_sms_number.py

@@ -19,34 +19,36 @@ INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER']
 DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID']
 
 def upgrade():
-    op.get_bind()
+    # op.get_bind()
     
-    # add the inbound number for the default service to inbound_numbers
-    table_name = 'inbound_numbers'
-    provider = 'sns'
-    active = 'true'
-    op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')")
+    # # add the inbound number for the default service to inbound_numbers
+    # table_name = 'inbound_numbers'
+    # provider = 'sns'
+    # active = 'true'
+    # op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')")
     
-    # add the inbound number for the default service to service_sms_senders
-    table_name = 'service_sms_senders'
-    id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4'
-    is_default = 'true'
-    sms_sender = INBOUND_NUMBER
-    inbound_number_id = INBOUND_NUMBER_ID
-    archived = 'false'
-    op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')")
+    # # add the inbound number for the default service to service_sms_senders
+    # table_name = 'service_sms_senders'
+    # id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4'
+    # is_default = 'true'
+    # sms_sender = INBOUND_NUMBER
+    # inbound_number_id = INBOUND_NUMBER_ID
+    # archived = 'false'
+    # op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')")
     
-    # add the inbound number for the default service to inbound_numbers
-    table_name = 'service_permissions'
-    permission = 'inbound_sms'
-    active = 'true'
-    op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')")
+    # # add the inbound number for the default service to inbound_numbers
+    # table_name = 'service_permissions'
+    # permission = 'inbound_sms'
+    # active = 'true'
+    # op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')")
+    pass
 
 
 def downgrade():
-    delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'"
-    delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'"
-    delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'"
-    op.execute(delete_sms_sender)
-    op.execute(delete_inbound_number)
-    op.execute(delete_service_inbound_permission)
+    # delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'"
+    # delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'"
+    # delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'"
+    # op.execute(delete_sms_sender)
+    # op.execute(delete_inbound_number)
+    # op.execute(delete_service_inbound_permission)
+    pass

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -4,7 +4,11 @@ from datetime import datetime
 from freezegun import freeze_time
 
 from app import encryption, statsd_client
-from app.celery.process_ses_receipts_tasks import process_ses_results
+from app.celery.process_ses_receipts_tasks import (
+    process_ses_results,
+    remove_emails_from_bounce,
+    remove_emails_from_complaint,
+)
 from app.celery.research_mode_tasks import (
     ses_hard_bounce_callback,
     ses_notification_callback,
@@ -15,10 +19,6 @@ from app.celery.service_callback_tasks import (
 )
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import Complaint, Notification
-from app.notifications.notifications_ses_callback import (
-    remove_emails_from_bounce,
-    remove_emails_from_complaint,
-)
 from tests.app.conftest import create_sample_notification
 from tests.app.db import (
     create_notification,
@@ -105,7 +105,7 @@ def test_process_ses_results(sample_email_template):
     assert process_ses_results(response=ses_notification_callback(reference='ref1'))
 
 
-def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker):
+def test_process_ses_results_retry_called(sample_email_template, mocker):
     create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
     mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED"))
     mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
@@ -178,7 +178,7 @@ def test_ses_callback_should_not_update_notification_status_if_already_delivered
     assert mock_upd.call_count == 0
 
 
-def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker):
+def test_ses_callback_should_retry_if_notification_is_new(mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
     with freeze_time('2017-11-17T12:14:03.646Z'):
@@ -192,7 +192,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db,
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_retry.call_count == 0
         mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)')
-def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker):
+def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
     with freeze_time('2017-11-21T12:14:03.646Z'):

tests/app/notifications/test_notifications_ses_callback.py

@@ -2,12 +2,12 @@ import pytest
 from flask import json
 from sqlalchemy.exc import SQLAlchemyError
 
-from app.dao.notifications_dao import get_notification_by_id
-from app.models import Complaint
-from app.notifications.notifications_ses_callback import (
+from app.celery.process_ses_receipts_tasks import (
     check_and_queue_callback_task,
     handle_complaint,
 )
+from app.dao.notifications_dao import get_notification_by_id
+from app.models import Complaint
 from tests.app.db import (
     create_notification,
     create_notification_history,