Added process for dvla acknowledgement file

Daily schedule task to check ack file against zip file lists
if we haven't receive ack for a zip file, raise a 500 exception

tests/app/celery/test_scheduled_tasks.py

@@ -34,7 +34,8 @@ from app.celery.scheduled_tasks import (
     switch_current_sms_provider_on_slow_delivery,
     timeout_job_statistics,
     timeout_notifications,
-    daily_stats_template_usage_by_month
+    daily_stats_template_usage_by_month,
+    letter_raise_alert_if_no_ack_file_for_zip
 )
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.config import QueueNames, TaskNames
@@ -60,7 +61,7 @@ from app.models import (
     SMS_TYPE
 )
 from app.utils import get_london_midnight_in_utc
-from app.v2.errors import JobIncompleteError
+from app.v2.errors import JobIncompleteError, NoAckFileReceived
 from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
 
 from tests.app.conftest import (
@@ -1026,3 +1027,48 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou
     ).all()
 
     assert len(result) == 1
+
+
+def mock_s3_get_list_match(bucket_name, subfolder='', suffix=''):
+
+    if subfolder == '2018-01-11':
+        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP']
+    print(suffix)
+    if subfolder == 'root/dispatch':
+        return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
+
+
+def mock_s3_get_list_diff(bucket_name, subfolder='', suffix=''):
+
+    if subfolder == '2018-01-11':
+        return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP']
+    print(suffix)
+    if subfolder == 'root/dispatch':
+        return ['root/dispatch/NOTIFY.20180111175733.ACK.txt']
+
+
+@freeze_time('2018-01-11T23:00:00')
+def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db):
+    mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match)
+    mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
+                                 return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
+                                              'NOTIFY.20180111175008.ZIP|20180111175734')
+
+    letter_raise_alert_if_no_ack_file_for_zip()
+
+    assert mock_file_list.call_count == 2
+    assert mock_get_file.call_count == 1
+
+
+@freeze_time('2018-01-11T23:00:00')
+def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db):
+    mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff)
+    mock_get_file = mocker.patch("app.aws.s3.get_s3_file",
+                                 return_value='NOTIFY.20180111175007.ZIP|20180111175733\n'
+                                              'NOTIFY.20180111175008.ZIP|20180111175734')
+    with pytest.raises(expected_exception=NoAckFileReceived) as e:
+        letter_raise_alert_if_no_ack_file_for_zip()
+
+    assert e.value.message == ['NOTIFY.20180111175009.ZIP']
+    assert mock_file_list.call_count == 2
+    assert mock_get_file.call_count == 1

tests/app/s3/test_s3.py

@@ -0,0 +1,66 @@
+from app.aws.s3 import get_list_of_files_by_suffix, get_s3_file
+
+
+zip_bucket_name = 'development-letters-pdf'
+zip_sub_folder = '2018-01-11'
+zip_file_name = '2018-01-11/NOTIFY.20180111175007.ZIP'
+ack_bucket_name = 'development-letters-pdf'
+ack_subfolder = 'root/dispatch'
+ack_file_name = 'root/dispatch/NOTIFY.20180111175733.ACK.txt'
+
+# Tests for boto3 and s3, can only perform locally against the Tools aws account and have permissions to access S3.
+# The tests are based on the above folders and files already uploaded to S3 Tools aws account (If these are removed or
+# renamed, the tests won't pass. 
+
+
+def test_get_zip_files():
+    zip_file_list = []
+    for key in get_list_of_files_by_suffix(bucket_name=zip_bucket_name, subfolder=zip_sub_folder, suffix='.ZIP'):
+        print('File: ' + key)
+        zip_file_list.append(key)
+    assert zip_file_name in zip_file_list
+
+
+def test_get_ack_files():
+    ack_file_list = []
+    for key in get_list_of_files_by_suffix(bucket_name=ack_bucket_name, subfolder=ack_subfolder, suffix='.ACK.txt'):
+        print('File: ' + key)
+        ack_file_list.append(key)
+    assert ack_file_name in ack_file_list
+
+
+def test_get_file_content():
+    ack_file_list = []
+    for key in get_list_of_files_by_suffix(bucket_name=ack_bucket_name, subfolder=ack_subfolder, suffix='.ACK.txt'):
+        ack_file_list.append(key)
+    assert ack_file_name in key
+
+    todaystr = '20180111'
+    for key in ack_file_list:
+        if todaystr in key:
+            content = get_s3_file(ack_bucket_name, key)
+            print(content)
+
+
+def test_letter_ack_file_strip_correctly():
+    # Test ack files are stripped correctly. In the acknowledgement file, there should be 2 zip files,
+    # 'NOTIFY.20180111175007.ZIP','NOTIFY.20180111175008.ZIP'.
+    zip_file_list = ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP']
+    # get acknowledgement file
+    ack_file_list = []
+    for key in get_list_of_files_by_suffix(bucket_name=ack_bucket_name, subfolder=ack_subfolder, suffix='.ACK.txt'):
+        ack_file_list.append(key)
+
+    for key in ack_file_list:
+        if '20180111' in key:
+            content = get_s3_file(ack_bucket_name, key)
+            print(content)
+            for zip_file in content.split():  # iterate each line
+                s = zip_file.split('|')
+                print(s[0])
+                for zf in zip_file_list:
+                    if s[0] in zf:
+                        zip_file_list.remove(zf)
+
+    print('zip_file_list: ' + str(zip_file_list))
+    assert zip_file_list == ['NOTIFY.20180111175009.ZIP']