Added process for dvla acknowledgement file

Daily schedule task to check ack file against zip file lists
if we haven't receive ack for a zip file, raise a 500 exception

app/aws/s3.py

@@ -107,3 +107,19 @@ def upload_letters_pdf(reference, crown, filedata):
 
     current_app.logger.info("Uploading letters PDF {} to {}".format(
         upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME']))
+
+
+def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix=''):
+    s3_client = client('s3', current_app.config['AWS_REGION'])
+    paginator = s3_client.get_paginator('list_objects_v2')
+
+    page_iterator = paginator.paginate(
+        Bucket=bucket_name,
+        Prefix=subfolder
+    )
+
+    for page in page_iterator:
+        for obj in page['Contents']:
+            key = obj['Key']
+            if key.endswith(suffix):
+                yield key

app/celery/scheduled_tasks.py

@@ -59,7 +59,7 @@ from app.celery.tasks import (
 )
 from app.config import QueueNames, TaskNames
 from app.utils import convert_utc_to_bst
-from app.v2.errors import JobIncompleteError
+from app.v2.errors import JobIncompleteError, NoAckFileReceived
 from app.dao.service_callback_api_dao import get_service_callback_api_for_service
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 
@@ -451,18 +451,30 @@ def daily_stats_template_usage_by_month():
 
 @notify_celery.task(name='raise-alert-if-no-letter-ack-file')
 @statsd(namespace="tasks")
-def raise_alert_if_no_letter_ack_file():
-    """
-    Get all files sent "today"
-    list_of_zip_files => get file names s3.get_s3_bucket_objects() look in the folder with todays date
+def letter_raise_alert_if_no_ack_file_for_zip():
+    # get a list of today's zip files
+    zip_file_list = []
+    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+                                              subfolder=datetime.utcnow().strftime('%Y-%m-%d'), suffix='.ZIP'):
+        zip_file_list.append(key)
 
-    list_of_ack_files => Get all files sent today with name containing ack.txt from a diff bucket
+    # get acknowledgement file
+    ack_file_list = []
+    for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
+                                              subfolder='root/dispatch', suffix='.ACK.txt'):
+        ack_file_list.append(key)
 
-    For filename in list_of_zip_files:
-        for ack_file in list_of_ack_files if name= file.strip("NOTIFY.", ".ZIP")
-        IF no ack_file
-            raise NoAckFileReceived(status=500, message="No ack file received for {filename}")
+    todaystr = datetime.utcnow().strftime('%Y%m%d')
 
-    """
+    for key in ack_file_list:
+        if todaystr in key:
+            content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
 
-    pass
+            for zip_file in content.split('\n'):    # each line
+                s = zip_file.split('|')
+                for zf in zip_file_list:
+                    if s[0] in zf:
+                        zip_file_list.remove(zf)
+
+    if zip_file_list:
+        raise NoAckFileReceived(message=zip_file_list)

app/config.py

@@ -251,7 +251,7 @@ class Config(object):
         },
         'raise-alert-if-no-letter-ack-file': {
             'task': 'raise-alert-if-no-letter-ack-file',
-            'schedule': crontab(hour=19, minute=00),
+            'schedule': crontab(hour=23, minute=00),
             'options': {'queue': QueueNames.PERIODIC}
         },
         'run-letter-api-notifications': {

app/notifications/notifications_letter_callback.py

@@ -53,7 +53,6 @@ def process_letter_response():
         message = json.loads(req_json['Message'])
         filename = message['Records'][0]['s3']['object']['key']
         current_app.logger.info('Received file from DVLA: {}'.format(filename))
-        # IF file name contains .rs.txt THEN continue ELSE log message with file name. and return
         current_app.logger.info('DVLA callback: Calling task to update letter notifications')
         update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY)
 

app/v2/errors.py

@@ -26,6 +26,23 @@ class JobIncompleteError(Exception):
         }
 
 
+class NoAckFileReceived(Exception):
+    def __init__(self, message):
+        self.message = message
+        self.status_code = 500
+
+    def to_dict_v2(self):
+        return {
+            'status_code': self.status_code,
+            "errors": [
+                {
+                    "error": 'NoAckFileReceived',
+                    "message": str(self.message)
+                }
+            ]
+        }
+
+
 class TooManyRequestsError(InvalidRequest):
     status_code = 429
     message_template = 'Exceeded send limits ({}) for today'