more work

app/celery/scheduled_tasks.py

@@ -23,7 +23,10 @@ from app.dao.jobs_dao import (
     find_jobs_with_missing_rows,
     find_missing_row_for_job,
 )
-from app.dao.notifications_dao import notifications_not_yet_sent
+from app.dao.notifications_dao import (
+    dao_update_delivery_receipts,
+    notifications_not_yet_sent,
+)
 from app.dao.services_dao import (
     dao_find_services_sending_to_tv_numbers,
     dao_find_services_with_high_failure_rates,
@@ -234,9 +237,10 @@ def check_for_services_with_high_failure_rates_or_sending_to_tv_numbers():
             zendesk_client.send_ticket_to_zendesk(ticket)
 
 
-@notify_celery.task(name="process_delivery_receipts_first_wave")
-def process_delivery_receipts_first_wave():
+@notify_celery.task(name="process-delivery-receipts")
+def process_delivery_receipts():
     cloudwatch = AwsCloudwatchClient()
-    start_time = utc_now() - timedelta(hours=1)
+    start_time = utc_now() - timedelta(minutes=10)
     end_time = utc_now()
     receipts = cloudwatch.check_delivery_receipts(start_time, end_time)
+    dao_update_delivery_receipts(receipts)

app/clients/cloudwatch/aws_cloudwatch.py

@@ -1,7 +1,5 @@
-import json
 import os
 import re
-from datetime import timedelta
 from time import sleep
 
 from boto3 import client
@@ -10,8 +8,7 @@ from flask import current_app
 from app.clients import AWS_CLIENT_CONFIG, Client
 from app.cloudfoundry_config import cloud_config
 from app.dao.notifications_dao import dao_update_delivery_receipts
-from app.exceptions import NotificationTechnicalFailureException
-from app.utils import hilite, utc_now
+from app.utils import utc_now
 
 
 class AwsCloudwatchClient(Client):
@@ -111,7 +108,7 @@ class AwsCloudwatchClient(Client):
         return None
 
 
-def do_log_insights(self):
+def check_delivery_receipts(self, start, end):
     region = cloud_config.sns_region
     account_number = self._extract_account_number(cloud_config.ses_domain_arn)
 
@@ -121,11 +118,10 @@ def do_log_insights(self):
     )
 
     query = """
-    fields @timestamp, status, delivery.providerResponse, delivery.destination, notification.messageId, delivery.phoneCarrier
+    fields @timestamp, status, delivery.providerResponse, delivery.destination,
+      notification.messageId, delivery.phoneCarrier
     | sort @timestamp asc
     """
-    start = utc_now() - timedelta(hours=1)
-    end = utc_now()
 
     response = client._client.start_query(
         logGroupName=log_group_name,

app/config.py

@@ -196,6 +196,11 @@ class Config(object):
                 "schedule": timedelta(minutes=63),
                 "options": {"queue": QueueNames.PERIODIC},
             },
+            "process-delivery-receipts": {
+                "task": "process-delivery-receipts",
+                "schedule": timedelta(minutes=8),
+                "options": {"queue": QueueNames.PERIODIC},
+            },
             "expire-or-delete-invitations": {
                 "task": "expire-or-delete-invitations",
                 "schedule": timedelta(minutes=66),

app/dao/notifications_dao.py

@@ -727,6 +727,7 @@ def dao_update_delivery_receipts(receipts):
             status=case(id_to_status),
             notification_status=case(id_to_status),
             sent_at=case(id_to_timestamp),
+            provider_response=case(id_to_provider_response),
         )
     )
     db.session.execute(stmt)