darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-21 07:51:13 -05:00
Code Issues Packages Projects Releases Wiki Activity

fixing tests

Browse Source
This commit is contained in:
jimmoffet
2022-10-03 17:16:59 -07:00
parent c7ccc3b0dd
commit c04d1df6b3
14 changed files with 279 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
app/__init__.py
View File

@@ -139,7 +139,6 @@ def register_blueprint(application):
) )
from app.billing.rest import billing_blueprint from app.billing.rest import billing_blueprint
from app.broadcast_message.rest import broadcast_message_blueprint from app.broadcast_message.rest import broadcast_message_blueprint
from app.celery.process_ses_receipts_tasks import ses_callback_blueprint
from app.complaint.complaint_rest import complaint_blueprint from app.complaint.complaint_rest import complaint_blueprint
from app.email_branding.rest import email_branding_blueprint from app.email_branding.rest import email_branding_blueprint
from app.events.rest import events as events_blueprint from app.events.rest import events as events_blueprint
@@ -154,6 +153,9 @@ def register_blueprint(application):
from app.notifications.notifications_letter_callback import ( from app.notifications.notifications_letter_callback import (
letter_callback_blueprint, letter_callback_blueprint,
) )
from app.notifications.notifications_ses_callback import (
ses_callback_blueprint,
)
from app.notifications.notifications_sms_callback import ( from app.notifications.notifications_sms_callback import (
sms_callback_blueprint, sms_callback_blueprint,
) )

4
app/celery/nightly_tasks.py
View File

@@ -11,6 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError
from app import notify_celery, statsd_client, zendesk_client from app import notify_celery, statsd_client, zendesk_client
from app.aws import s3 from app.aws import s3
from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
from app.config import QueueNames from app.config import QueueNames
from app.cronitor import cronitor from app.cronitor import cronitor
from app.dao.fact_processing_time_dao import insert_update_processing_time from app.dao.fact_processing_time_dao import insert_update_processing_time
@@ -37,9 +38,6 @@ from app.models import (
FactProcessingTime, FactProcessingTime,
Notification, Notification,
) )
from app.notifications.notifications_ses_callback import (
check_and_queue_callback_task,
)
from app.utils import get_london_midnight_in_utc from app.utils import get_london_midnight_in_utc

186
app/celery/process_ses_receipts_tasks.py
View File

@@ -6,64 +6,26 @@ from json import decoder
import iso8601 import iso8601
import requests import requests
from celery.exceptions import Retry from celery.exceptions import Retry
from flask import Blueprint, current_app, json, jsonify, request from flask import current_app, json
from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.orm.exc import NoResultFound
from app import notify_celery, statsd_client from app import notify_celery, statsd_client
from app.celery.validate_sns_message import sns_notification_handler from app.celery.service_callback_tasks import (
create_complaint_callback_data,
create_delivery_status_callback_data,
send_complaint_to_service,
send_delivery_status_to_service,
)
from app.config import QueueNames from app.config import QueueNames
from app.dao import notifications_dao from app.dao import notifications_dao
from app.errors import InvalidRequest, register_errors from app.dao.complaint_dao import save_complaint
from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING from app.dao.notifications_dao import dao_get_notification_history_by_reference
from app.notifications.notifications_ses_callback import ( from app.dao.service_callback_api_dao import (
_check_and_queue_complaint_callback_task, get_service_complaint_callback_api_for_service,
check_and_queue_callback_task, get_service_delivery_status_callback_api_for_service,
determine_notification_bounce_type,
get_aws_responses,
handle_complaint,
) )
from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__) from app.notifications.callbacks import create_complaint_callback_data
DEFAULT_MAX_AGE = timedelta(days=10000)
register_errors(ses_callback_blueprint)
class SNSMessageType(enum.Enum):
SubscriptionConfirmation = 'SubscriptionConfirmation'
Notification = 'Notification'
UnsubscribeConfirmation = 'UnsubscribeConfirmation'
class InvalidMessageTypeException(Exception):
pass
def verify_message_type(message_type: str):
try:
SNSMessageType(message_type)
except ValueError:
raise InvalidMessageTypeException(f'{message_type} is not a valid message type.')
# 400 counts as a permanent failure so SNS will not retry.
# 500 counts as a failed delivery attempt so SNS will retry.
# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
# This should not be here, it used to be in notifications/notifications_ses_callback. It then
# got refactored into a task, which is fine, but it created a circular dependency. Will need
# to investigate why GDS extracted this into a lambda
@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
def email_ses_callback_handler():
try:
data = sns_notification_handler(request.data, request.headers)
except Exception as e:
raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
message = data.get("Message")
if "mail" in message:
process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY)
return jsonify(
result="success", message="SES-SNS callback succeeded"
), 200
@notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300) @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
@@ -145,3 +107,123 @@ def process_ses_results(self, response):
current_app.logger.exception("Error processing SES results: {}".format(type(e))) current_app.logger.exception("Error processing SES results: {}".format(type(e)))
self.retry(queue=QueueNames.RETRY) self.retry(queue=QueueNames.RETRY)
def determine_notification_bounce_type(ses_message):
notification_type = ses_message["notificationType"]
if notification_type in ["Delivery", "Complaint"]:
return notification_type
if notification_type != "Bounce":
raise KeyError(f"Unhandled notification type {notification_type}")
remove_emails_from_bounce(ses_message)
current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
if ses_message["bounce"]["bounceType"] == "Permanent":
return "Permanent"
return "Temporary"
def _determine_provider_response(ses_message):
if ses_message["notificationType"] != "Bounce":
return None
bounce_type = ses_message["bounce"]["bounceType"]
bounce_subtype = ses_message["bounce"]["bounceSubType"]
# See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
return "The email address is on our email provider suppression list"
elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
return "The email address is on the GC Notify suppression list"
elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
return "The email was rejected because of its attachments"
return None
def get_aws_responses(ses_message):
status = determine_notification_bounce_type(ses_message)
base = {
"Permanent": {
"message": "Hard bounced",
"success": False,
"notification_status": "permanent-failure",
},
"Temporary": {
"message": "Soft bounced",
"success": False,
"notification_status": "temporary-failure",
},
"Delivery": {
"message": "Delivered",
"success": True,
"notification_status": "delivered",
},
"Complaint": {
"message": "Complaint",
"success": True,
"notification_status": "delivered",
},
}[status]
base["provider_response"] = _determine_provider_response(ses_message)
return base
def handle_complaint(ses_message):
recipient_email = remove_emails_from_complaint(ses_message)[0]
current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
try:
reference = ses_message["mail"]["messageId"]
except KeyError as e:
current_app.logger.exception("Complaint from SES failed to get reference from message", e)
return
notification = dao_get_notification_history_by_reference(reference)
ses_complaint = ses_message.get("complaint", None)
complaint = Complaint(
notification_id=notification.id,
service_id=notification.service_id,
ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
)
save_complaint(complaint)
return complaint, notification, recipient_email
def remove_mail_headers(dict_to_edit):
if dict_to_edit["mail"].get("headers"):
dict_to_edit["mail"].pop("headers")
if dict_to_edit["mail"].get("commonHeaders"):
dict_to_edit["mail"].pop("commonHeaders")
def remove_emails_from_bounce(bounce_dict):
remove_mail_headers(bounce_dict)
bounce_dict["mail"].pop("destination", None)
bounce_dict["bounce"].pop("bouncedRecipients", None)
def remove_emails_from_complaint(complaint_dict):
remove_mail_headers(complaint_dict)
complaint_dict["complaint"].pop("complainedRecipients")
return complaint_dict["mail"].pop("destination")
def check_and_queue_callback_task(notification):
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
if service_callback_api:
notification_data = create_delivery_status_callback_data(notification, service_callback_api)
send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
if service_callback_api:
complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)

4
app/celery/process_sms_client_response_tasks.py
View File

@@ -6,13 +6,11 @@ from flask import current_app
from notifications_utils.template import SMSMessageTemplate from notifications_utils.template import SMSMessageTemplate
from app import notify_celery, statsd_client from app import notify_celery, statsd_client
from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
from app.clients import ClientException from app.clients import ClientException
from app.dao import notifications_dao from app.dao import notifications_dao
from app.dao.templates_dao import dao_get_template_by_id from app.dao.templates_dao import dao_get_template_by_id
from app.models import NOTIFICATION_PENDING from app.models import NOTIFICATION_PENDING
from app.notifications.notifications_ses_callback import (
check_and_queue_callback_task,
)
sms_response_mapper = { sms_response_mapper = {
# 'MMG': get_mmg_responses, # 'MMG': get_mmg_responses,

141
app/notifications/notifications_ses_callback.py
View File

@@ -1,5 +1,9 @@
from flask import current_app, json import enum
from datetime import timedelta
from flask import Blueprint, current_app, json, jsonify, request
from app.celery.process_ses_receipts_tasks import process_ses_results
from app.celery.service_callback_tasks import ( from app.celery.service_callback_tasks import (
create_complaint_callback_data, create_complaint_callback_data,
create_delivery_status_callback_data, create_delivery_status_callback_data,
@@ -13,127 +17,28 @@ from app.dao.service_callback_api_dao import (
get_service_complaint_callback_api_for_service, get_service_complaint_callback_api_for_service,
get_service_delivery_status_callback_api_for_service, get_service_delivery_status_callback_api_for_service,
) )
from app.errors import InvalidRequest
from app.models import Complaint from app.models import Complaint
from app.notifications.callbacks import create_complaint_callback_data from app.notifications.callbacks import create_complaint_callback_data
from app.notifications.sns_handlers import sns_notification_handler
ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
DEFAULT_MAX_AGE = timedelta(days=10000)
def determine_notification_bounce_type(ses_message): # 400 counts as a permanent failure so SNS will not retry.
notification_type = ses_message["notificationType"] # 500 counts as a failed delivery attempt so SNS will retry.
if notification_type in ["Delivery", "Complaint"]: # See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
return notification_type @ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
def email_ses_callback_handler():
if notification_type != "Bounce":
raise KeyError(f"Unhandled notification type {notification_type}")
remove_emails_from_bounce(ses_message)
current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
if ses_message["bounce"]["bounceType"] == "Permanent":
return "Permanent"
return "Temporary"
def _determine_provider_response(ses_message):
if ses_message["notificationType"] != "Bounce":
return None
bounce_type = ses_message["bounce"]["bounceType"]
bounce_subtype = ses_message["bounce"]["bounceSubType"]
# See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
return "The email address is on our email provider suppression list"
elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
return "The email address is on the GC Notify suppression list"
elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
return "The email was rejected because of its attachments"
return None
def get_aws_responses(ses_message):
status = determine_notification_bounce_type(ses_message)
base = {
"Permanent": {
"message": "Hard bounced",
"success": False,
"notification_status": "permanent-failure",
},
"Temporary": {
"message": "Soft bounced",
"success": False,
"notification_status": "temporary-failure",
},
"Delivery": {
"message": "Delivered",
"success": True,
"notification_status": "delivered",
},
"Complaint": {
"message": "Complaint",
"success": True,
"notification_status": "delivered",
},
}[status]
base["provider_response"] = _determine_provider_response(ses_message)
return base
def handle_complaint(ses_message):
recipient_email = remove_emails_from_complaint(ses_message)[0]
current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
try: try:
reference = ses_message["mail"]["messageId"] data = sns_notification_handler(request.data, request.headers)
except KeyError as e: except Exception as e:
current_app.logger.exception("Complaint from SES failed to get reference from message", e) raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
return
notification = dao_get_notification_history_by_reference(reference)
ses_complaint = ses_message.get("complaint", None)
complaint = Complaint( message = data.get("Message")
notification_id=notification.id, if "mail" in message:
service_id=notification.service_id, process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY)
ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
)
save_complaint(complaint)
return complaint, notification, recipient_email
def remove_mail_headers(dict_to_edit):
if dict_to_edit["mail"].get("headers"):
dict_to_edit["mail"].pop("headers")
if dict_to_edit["mail"].get("commonHeaders"):
dict_to_edit["mail"].pop("commonHeaders")
def remove_emails_from_bounce(bounce_dict):
remove_mail_headers(bounce_dict)
bounce_dict["mail"].pop("destination", None)
bounce_dict["bounce"].pop("bouncedRecipients", None)
def remove_emails_from_complaint(complaint_dict):
remove_mail_headers(complaint_dict)
complaint_dict["complaint"].pop("complainedRecipients")
return complaint_dict["mail"].pop("destination")
def check_and_queue_callback_task(notification):
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
if service_callback_api:
notification_data = create_delivery_status_callback_data(notification, service_callback_api)
send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
# queue callback task only if the service_callback_api exists
service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
if service_callback_api:
complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
return jsonify(
result="success", message="SES-SNS callback succeeded"
), 200

1
app/notifications/notifications_sms_callback.py
View File

@@ -9,6 +9,7 @@ from app.errors import InvalidRequest, register_errors
sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms") sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
register_errors(sms_callback_blueprint) register_errors(sms_callback_blueprint)
# TODO SNS SMS delivery receipts delivered here
# @sms_callback_blueprint.route('/mmg', methods=['POST']) # @sms_callback_blueprint.route('/mmg', methods=['POST'])
# def process_mmg_response(): # def process_mmg_response():

142
app/notifications/receive_notifications.py
View File

@@ -2,17 +2,17 @@ from datetime import datetime
from urllib.parse import unquote from urllib.parse import unquote
import iso8601 import iso8601
from flask import Blueprint, abort, current_app, jsonify, request, json from flask import Blueprint, abort, current_app, json, jsonify, request
from gds_metrics.metrics import Counter from gds_metrics.metrics import Counter
from notifications_utils.recipients import try_validate_and_format_phone_number from notifications_utils.recipients import try_validate_and_format_phone_number
from app.celery import tasks from app.celery import tasks
from app.celery.validate_sns_message import sns_notification_handler
from app.config import QueueNames from app.config import QueueNames
from app.dao.inbound_sms_dao import dao_create_inbound_sms from app.dao.inbound_sms_dao import dao_create_inbound_sms
from app.dao.services_dao import dao_fetch_service_by_inbound_number from app.dao.services_dao import dao_fetch_service_by_inbound_number
from app.errors import register_errors, InvalidRequest from app.errors import InvalidRequest, register_errors
from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms
from app.notifications.sns_handlers import sns_notification_handler
receive_notifications_blueprint = Blueprint('receive_notifications', __name__) receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
register_errors(receive_notifications_blueprint) register_errors(receive_notifications_blueprint)
@@ -83,90 +83,90 @@ def receive_sns_sms():
), 200 ), 200
@receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST']) # @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
def receive_mmg_sms(): # def receive_mmg_sms():
""" # """
{ # {
'MSISDN': '447123456789' # 'MSISDN': '447123456789'
'Number': '40604', # 'Number': '40604',
'Message': 'some+uri+encoded+message%3A', # 'Message': 'some+uri+encoded+message%3A',
'ID': 'SOME-MMG-SPECIFIC-ID', # 'ID': 'SOME-MMG-SPECIFIC-ID',
'DateRecieved': '2017-05-21+11%3A56%3A11' # 'DateRecieved': '2017-05-21+11%3A56%3A11'
} # }
""" # """
post_data = request.get_json() # post_data = request.get_json()
auth = request.authorization # auth = request.authorization
if not auth: # if not auth:
current_app.logger.warning("Inbound sms (MMG) no auth header") # current_app.logger.warning("Inbound sms (MMG) no auth header")
abort(401) # abort(401)
elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \ # elif auth.username not in current_app.config['MMG_INBOUND_SMS_USERNAME'] \
or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']: # or auth.password not in current_app.config['MMG_INBOUND_SMS_AUTH']:
current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username)) # current_app.logger.warning("Inbound sms (MMG) incorrect username ({}) or password".format(auth.username))
abort(403) # abort(403)
inbound_number = strip_leading_forty_four(post_data['Number']) # inbound_number = strip_leading_forty_four(post_data['Number'])
service = fetch_potential_service(inbound_number, 'mmg') # service = fetch_potential_service(inbound_number, 'mmg')
if not service: # if not service:
# since this is an issue with our service <-> number mapping, or no inbound_sms service permission # # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
# we should still tell MMG that we received it successfully # # we should still tell MMG that we received it successfully
return 'RECEIVED', 200 # return 'RECEIVED', 200
INBOUND_SMS_COUNTER.labels("mmg").inc() # INBOUND_SMS_COUNTER.labels("mmg").inc()
inbound = create_inbound_sms_object(service, # inbound = create_inbound_sms_object(service,
content=format_mmg_message(post_data["Message"]), # content=format_mmg_message(post_data["Message"]),
from_number=post_data['MSISDN'], # from_number=post_data['MSISDN'],
provider_ref=post_data["ID"], # provider_ref=post_data["ID"],
date_received=post_data.get('DateRecieved'), # date_received=post_data.get('DateRecieved'),
provider_name="mmg") # provider_name="mmg")
tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) # tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug( # current_app.logger.debug(
'{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference)) # '{} received inbound SMS with reference {} from MMG'.format(service.id, inbound.provider_reference))
return jsonify({ # return jsonify({
"status": "ok" # "status": "ok"
}), 200 # }), 200
@receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST']) # @receive_notifications_blueprint.route('/notifications/sms/receive/firetext', methods=['POST'])
def receive_firetext_sms(): # def receive_firetext_sms():
post_data = request.form # post_data = request.form
auth = request.authorization # auth = request.authorization
if not auth: # if not auth:
current_app.logger.warning("Inbound sms (Firetext) no auth header") # current_app.logger.warning("Inbound sms (Firetext) no auth header")
abort(401) # abort(401)
elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']: # elif auth.username != 'notify' or auth.password not in current_app.config['FIRETEXT_INBOUND_SMS_AUTH']:
current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username)) # current_app.logger.warning("Inbound sms (Firetext) incorrect username ({}) or password".format(auth.username))
abort(403) # abort(403)
inbound_number = strip_leading_forty_four(post_data['destination']) # inbound_number = strip_leading_forty_four(post_data['destination'])
service = fetch_potential_service(inbound_number, 'firetext') # service = fetch_potential_service(inbound_number, 'firetext')
if not service: # if not service:
return jsonify({ # return jsonify({
"status": "ok" # "status": "ok"
}), 200 # }), 200
inbound = create_inbound_sms_object(service=service, # inbound = create_inbound_sms_object(service=service,
content=post_data["message"], # content=post_data["message"],
from_number=post_data['source'], # from_number=post_data['source'],
provider_ref=None, # provider_ref=None,
date_received=post_data['time'], # date_received=post_data['time'],
provider_name="firetext") # provider_name="firetext")
INBOUND_SMS_COUNTER.labels("firetext").inc() # INBOUND_SMS_COUNTER.labels("firetext").inc()
tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY) # tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
current_app.logger.debug( # current_app.logger.debug(
'{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference)) # '{} received inbound SMS with reference {} from Firetext'.format(service.id, inbound.provider_reference))
return jsonify({ # return jsonify({
"status": "ok" # "status": "ok"
}), 200 # }), 200
def format_mmg_message(message): def format_mmg_message(message):

0
app/celery/validate_sns_cert.py → app/notifications/sns_cert_validator.py
View File

4
app/celery/validate_sns_message.py → app/notifications/sns_handlers.py
View File

@@ -5,9 +5,8 @@ from json import decoder
import requests import requests
from flask import current_app, json from flask import current_app, json
from app.celery.validate_sns_cert import validate_sns_cert
from app.errors import InvalidRequest from app.errors import InvalidRequest
from app.notifications.sns_cert_validator import validate_sns_cert
DEFAULT_MAX_AGE = timedelta(days=10000) DEFAULT_MAX_AGE = timedelta(days=10000)
@@ -50,6 +49,7 @@ def sns_notification_handler(data, headers):
raise InvalidRequest("SES-SNS callback failed: validation failed", 400) raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
if message.get('Type') == 'SubscriptionConfirmation': if message.get('Type') == 'SubscriptionConfirmation':
# NOTE once a request is sent to SubscribeURL, AWS considers Notify a confirmed subscriber to this topic
url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL') url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
response = requests.get(url) response = requests.get(url)
try: try:

3
devcontainer-api/.devcontainer.json
View File

@@ -16,7 +16,8 @@
"python.defaultInterpreterPath": "/usr/bin/python3", "python.defaultInterpreterPath": "/usr/bin/python3",
"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint", "python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint",
"python.analysis.extraPaths": [ "python.analysis.extraPaths": [
"/home/vscode/.local/lib/python3.9/site-packages" "/home/vscode/.local/lib/python3.9/site-packages",
"/home/vscode/.local/bin"
] ]
}, },
"features": { "features": {

8
migrations/versions/0375_fix_service_name.py
View File

@@ -6,17 +6,13 @@ Create Date: 2022-08-29 11:04:15.888017
""" """
# revision identifiers, used by Alembic.
from datetime import datetime
revision = '0375_fix_service_name' revision = '0375_fix_service_name'
down_revision = '0374_fix_reg_template_history' down_revision = '0374_fix_reg_template_history'
from alembic import op from alembic import op
import sqlalchemy as sa from flask import current_app
service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553' service_id = current_app.config['NOTIFY_SERVICE_ID']
user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6'
def upgrade(): def upgrade():
op.get_bind() op.get_bind()

52
migrations/versions/0377_add_inbound_sms_number.py
View File

@@ -19,34 +19,36 @@ INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER']
DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID'] DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID']
def upgrade(): def upgrade():
op.get_bind() # op.get_bind()
# add the inbound number for the default service to inbound_numbers # # add the inbound number for the default service to inbound_numbers
table_name = 'inbound_numbers' # table_name = 'inbound_numbers'
provider = 'sns' # provider = 'sns'
active = 'true' # active = 'true'
op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')") # op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')")
# add the inbound number for the default service to service_sms_senders # # add the inbound number for the default service to service_sms_senders
table_name = 'service_sms_senders' # table_name = 'service_sms_senders'
id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4' # id = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4'
is_default = 'true' # is_default = 'true'
sms_sender = INBOUND_NUMBER # sms_sender = INBOUND_NUMBER
inbound_number_id = INBOUND_NUMBER_ID # inbound_number_id = INBOUND_NUMBER_ID
archived = 'false' # archived = 'false'
op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')") # op.execute(f"insert into {table_name} (id, sms_sender, service_id, is_default, inbound_number_id, created_at, archived) VALUES('{id}', '{INBOUND_NUMBER}', '{DEFAULT_SERVICE_ID}', '{is_default}', '{INBOUND_NUMBER_ID}', 'now()','{archived}')")
# add the inbound number for the default service to inbound_numbers # # add the inbound number for the default service to inbound_numbers
table_name = 'service_permissions' # table_name = 'service_permissions'
permission = 'inbound_sms' # permission = 'inbound_sms'
active = 'true' # active = 'true'
op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')") # op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')")
pass
def downgrade(): def downgrade():
delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'" # delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'"
delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'" # delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'"
delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'" # delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'"
op.execute(delete_sms_sender) # op.execute(delete_sms_sender)
op.execute(delete_inbound_number) # op.execute(delete_inbound_number)
op.execute(delete_service_inbound_permission) # op.execute(delete_service_inbound_permission)
pass

16
tests/app/celery/test_process_ses_receipts_tasks.py
View File

@@ -4,7 +4,11 @@ from datetime import datetime
from freezegun import freeze_time from freezegun import freeze_time
from app import encryption, statsd_client from app import encryption, statsd_client
from app.celery.process_ses_receipts_tasks import process_ses_results from app.celery.process_ses_receipts_tasks import (
process_ses_results,
remove_emails_from_bounce,
remove_emails_from_complaint,
)
from app.celery.research_mode_tasks import ( from app.celery.research_mode_tasks import (
ses_hard_bounce_callback, ses_hard_bounce_callback,
ses_notification_callback, ses_notification_callback,
@@ -15,10 +19,6 @@ from app.celery.service_callback_tasks import (
) )
from app.dao.notifications_dao import get_notification_by_id from app.dao.notifications_dao import get_notification_by_id
from app.models import Complaint, Notification from app.models import Complaint, Notification
from app.notifications.notifications_ses_callback import (
remove_emails_from_bounce,
remove_emails_from_complaint,
)
from tests.app.conftest import create_sample_notification from tests.app.conftest import create_sample_notification
from tests.app.db import ( from tests.app.db import (
create_notification, create_notification,
@@ -105,7 +105,7 @@ def test_process_ses_results(sample_email_template):
assert process_ses_results(response=ses_notification_callback(reference='ref1')) assert process_ses_results(response=ses_notification_callback(reference='ref1'))
def test_process_ses_results_retry_called(sample_email_template, _notify_db, mocker): def test_process_ses_results_retry_called(sample_email_template, mocker):
create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending') create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED")) mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED"))
mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
@@ -178,7 +178,7 @@ def test_ses_callback_should_not_update_notification_status_if_already_delivered
assert mock_upd.call_count == 0 assert mock_upd.call_count == 0
def test_ses_callback_should_retry_if_notification_is_new(client, _notify_db, mocker): def test_ses_callback_should_retry_if_notification_is_new(mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
with freeze_time('2017-11-17T12:14:03.646Z'): with freeze_time('2017-11-17T12:14:03.646Z'):
@@ -192,7 +192,7 @@ def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db,
assert process_ses_results(ses_notification_callback(reference='ref')) is None assert process_ses_results(ses_notification_callback(reference='ref')) is None
assert mock_retry.call_count == 0 assert mock_retry.call_count == 0
mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)') mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)')
def test_ses_callback_should_not_retry_if_notification_is_old(client, _notify_db, mocker): def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry') mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error') mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
with freeze_time('2017-11-21T12:14:03.646Z'): with freeze_time('2017-11-21T12:14:03.646Z'):

6
tests/app/notifications/test_notifications_ses_callback.py
View File

@@ -2,12 +2,12 @@ import pytest
from flask import json from flask import json
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from app.dao.notifications_dao import get_notification_by_id from app.celery.process_ses_receipts_tasks import (
from app.models import Complaint
from app.notifications.notifications_ses_callback import (
check_and_queue_callback_task, check_and_queue_callback_task,
handle_complaint, handle_complaint,
) )
from app.dao.notifications_dao import get_notification_by_id
from app.models import Complaint
from tests.app.db import ( from tests.app.db import (
create_notification, create_notification,
create_notification_history, create_notification_history,