Merge pull request #1588 from alphagov/letter-S3zipchange-deskpro

Process letter acknowledge file with new S3 zip file meta file structure

app/aws/s3.py

@@ -120,7 +120,7 @@ def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modif
 
     for page in page_iterator:
         for obj in page.get('Contents', []):
-            key = obj['Key'].lower()
+            key = obj['Key']
-            if key.endswith(suffix.lower()):
+            if key.lower().endswith(suffix.lower()):
                 if not last_modified or obj['LastModified'] >= last_modified:
                     yield key

app/celery/scheduled_tasks.py

@@ -494,40 +494,42 @@ def daily_stats_template_usage_by_month():
 @statsd(namespace="tasks")
 def letter_raise_alert_if_no_ack_file_for_zip():
     # get a list of zip files since yesterday
-    zip_file_list = []
+    zip_file_set = set()
 
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
-                                              subfolder=datetime.utcnow().strftime('%Y-%m-%d'),
+                                              subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
-                                              suffix='.zip'):
+                                              suffix='.TXT'):
-        zip_file_list.append(key)
+        zip_file_set.add(key.upper().rstrip('.TXT'))
 
     # get acknowledgement file
-    ack_file_list = []
+    ack_file_set = set()
     # yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1)
     yesterday = datetime.utcnow() - timedelta(days=1)
     for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
                                               subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday):
-        ack_file_list.append(key)
+        ack_file_set.add(key)
 
     today_str = datetime.utcnow().strftime('%Y%m%d')
-    zip_not_today = []
 
-    for key in ack_file_list:
+    ack_content_set = set()
+    for key in ack_file_set:
         if today_str in key:
             content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
             for zip_file in content.split('\n'):    # each line
                 s = zip_file.split('|')
-                for zf in zip_file_list:
+                ack_content_set.add(s[0].upper())
-                    if s[0].lower() in zf.lower():
-                        zip_file_list.remove(zf)
-                    else:
-                        zip_not_today.append(s[0])
 
-    if zip_file_list:
+    if len(zip_file_set - ack_content_set) > 0:
-
+        deskpro_client.create_ticket(
-        raise NoAckFileReceived(message=zip_file_list)
+            subject="Letter acknowledge error",
-
+            message="Letter acknowledgement file do not contains all zip files sent: {}".format(datetime.utcnow()
-    if zip_not_today:
+                                                                                                .strftime('%Y-%m-%d')),
-        current_app.logger.info(
+            ticket_type='alert'
-            "letter ack contains zip that is not for today {} ".format(zip_not_today)
+        )
+
+        raise NoAckFileReceived(message=str(zip_file_set - ack_content_set))
+
+    if len(ack_content_set - zip_file_set) > 0:
+        current_app.logger.info(
+            "letter ack contains zip that is not for today: {}".format(ack_content_set - zip_file_set)
         )

tests/app/celery/test_scheduled_tasks.py

@@ -1109,16 +1109,16 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou
 
 def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
 
-    if subfolder == '2018-01-11':
+    if subfolder == '2018-01-11/zips_sent':
-        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP']
+        return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT']
     if subfolder == 'root/dispatch':
         return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
 
 
 def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
-    if subfolder == '2018-01-11':
+    if subfolder == '2018-01-11/zips_sent':
-        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP',
+        return ['NOTIFY.20180111175007.ZIP.TXT', 'NOTIFY.20180111175008.ZIP.TXT', 'NOTIFY.20180111175009.ZIP.TXT',
-                'NOTIFY.20180111175010.ZIP']
+                'NOTIFY.20180111175010.ZIP.TXT']
     if subfolder == 'root/dispatch':
         return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
 
@@ -1133,10 +1133,10 @@ def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
     letter_raise_alert_if_no_ack_file_for_zip()
 
     yesterday = datetime.utcnow() - timedelta(days=1)
-    subfoldername = datetime.utcnow().strftime('%Y-%m-%d')
+    subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
     assert mock_file_list.call_count == 2
     assert mock_file_list.call_args_list == [
-        call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.zip'),
+        call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
         call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
              suffix='.ACK.txt', last_modified=yesterday),
     ]
@@ -1149,9 +1149,29 @@ def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_d
     mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
                                  return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
                                               'NOTIFY.20180111175008.ZIP|20180111175734')
+    mock_deskpro = mocker.patch("app.celery.scheduled_tasks.deskpro_client.create_ticket")
+
     with pytest.raises(expected_exception=NoAckFileReceived) as e:
         letter_raise_alert_if_no_ack_file_for_zip()
 
-    assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']
+    assert e.value.message == str(set(['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP']))
     assert mock_file_list.call_count == 2
     assert mock_get_file.call_count == 1
+    mock_deskpro.assert_called_once_with(
+        subject="Letter acknowledge error",
+        message="Letter acknowledgement file do not contains all zip files sent: 2018-01-11",
+        ticket_type='alert'
+    )
+
+
+@freeze_time('2018-01-11T23:00:00')
+def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db):
+    mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
+    mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
+                                 return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
+                                              'NOTIFY.20180111175008.ZIP|20180111175734')
+
+    letter_raise_alert_if_no_ack_file_for_zip()
+
+    assert mock_file_list.call_count == 2
+    assert mock_get_file.call_count == 0