Merge pull request #2774 from alphagov/fix-report-and-delete-tasks

Fix report and delete tasks

app/celery/process_ses_receipts_tasks.py

@@ -39,7 +39,7 @@ def process_ses_results(self, response):
         reference = ses_message['mail']['messageId']
 
         try:
-            notification = notifications_dao.dao_get_notification_by_reference(reference)
+            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
         except NoResultFound:
             message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
@@ -50,22 +50,17 @@ def process_ses_results(self, response):
                 )
             return
 
-        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
-            notifications_dao._duplicate_update_warning(notification, notification_status)
-            return
-
-        notifications_dao._update_notification_status(notification=notification, status=notification_status)
-
-        if not aws_response_dict['success']:
-            current_app.logger.info(
-                "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
-                    notification.id, reference, aws_response_dict['message']
-                )
+        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
+            notifications_dao._duplicate_update_warning(
+                notification=notification,
+                status=notification_status
             )
+            return
         else:
-            current_app.logger.info('SES callback return status of {} for notification: {}'.format(
-                notification_status, notification.id
-            ))
+            notifications_dao.dao_update_notifications_by_reference(
+                references=[reference],
+                update_dict={'status': notification_status}
+            )
 
         statsd_client.incr('callback.ses.{}'.format(notification_status))
 

app/celery/tasks.py

@@ -41,7 +41,7 @@ from app.dao.notifications_dao import (
     dao_update_notifications_by_reference,
     dao_get_last_notification_added_for_job_id,
     update_notification_status_by_reference,
-    dao_get_notification_history_by_reference,
+    dao_get_notification_or_history_by_reference,
 )
 from app.dao.provider_details_dao import get_provider_details_by_notification_type
 from app.dao.returned_letters_dao import insert_or_update_returned_letters
@@ -303,7 +303,6 @@ def save_api_email(self,
     service = dao_fetch_service_by_id(notification['service_id'])
 
     try:
-        current_app.logger.info(f"Persisting notification {notification['id']}")
 
         persist_notification(
             notification_id=notification["id"],
@@ -327,7 +326,7 @@ def save_api_email(self,
             [notification['id']],
             queue=q
         )
-        current_app.logger.info(f"Email {notification['id']} has been persisted.")
+        current_app.logger.info(f"Email {notification['id']} has been persisted and sent to delivery queue.")
     except IntegrityError:
         current_app.logger.info(f"Email {notification['id']} already exists.")
 
@@ -557,7 +556,7 @@ def update_letter_notification(filename, temporary_failures, update):
 
 
 def check_billable_units(notification_update):
-    notification = dao_get_notification_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
 
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(

app/dao/notifications_dao.py

@@ -352,7 +352,6 @@ def insert_notification_history_delete_notifications(
           AND notification_type = :notification_type
           AND created_at < :timestamp_to_delete_backwards_from
           AND key_type = 'normal'
-          AND notification_status in ('delivered', 'permanent-failure', 'temporary-failure')
         limit :qry_limit
         """
     # Insert into NotificationHistory if the row already exists do nothing.
@@ -632,7 +631,7 @@ def dao_get_notification_by_reference(reference):
 
 
 @statsd(namespace="dao")
-def dao_get_notification_history_by_reference(reference):
+def dao_get_notification_or_history_by_reference(reference):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object

app/notifications/notifications_ses_callback.py

@@ -1,7 +1,7 @@
 from flask import current_app
 
 from app.dao.complaint_dao import save_complaint
-from app.dao.notifications_dao import dao_get_notification_history_by_reference
+from app.dao.notifications_dao import dao_get_notification_or_history_by_reference
 from app.dao.service_callback_api_dao import (
     get_service_delivery_status_callback_api_for_service, get_service_complaint_callback_api_for_service
 )
@@ -33,7 +33,7 @@ def handle_complaint(ses_message):
     except KeyError as e:
         current_app.logger.exception("Complaint from SES failed to get reference from message", e)
         return
-    notification = dao_get_notification_history_by_reference(reference)
+    notification = dao_get_notification_or_history_by_reference(reference)
     ses_complaint = ses_message.get('complaint', None)
 
     complaint = Complaint(