diff --git a/app/aws/s3.py b/app/aws/s3.py index 5ffb24068..1b8d436d3 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -107,3 +107,20 @@ def upload_letters_pdf(reference, crown, filedata): current_app.logger.info("Uploading letters PDF {} to {}".format( upload_file_name, current_app.config['LETTERS_PDF_BUCKET_NAME'])) + + +def get_list_of_files_by_suffix(bucket_name, subfolder='', suffix='', last_modified=None): + s3_client = client('s3', current_app.config['AWS_REGION']) + paginator = s3_client.get_paginator('list_objects_v2') + + page_iterator = paginator.paginate( + Bucket=bucket_name, + Prefix=subfolder + ) + + for page in page_iterator: + for obj in page['Contents']: + key = obj['Key'].lower() + if key.endswith(suffix.lower()): + if not last_modified or obj['LastModified'] >= last_modified: + yield key diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 5bac860a6..0562e4baa 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -59,7 +59,7 @@ from app.celery.tasks import ( ) from app.config import QueueNames, TaskNames from app.utils import convert_utc_to_bst -from app.v2.errors import JobIncompleteError +from app.v2.errors import JobIncompleteError, NoAckFileReceived from app.dao.service_callback_api_dao import get_service_callback_api_for_service from app.celery.service_callback_tasks import send_delivery_status_to_service @@ -447,3 +447,46 @@ def daily_stats_template_usage_by_month(): result.year, result.count ) + + +@notify_celery.task(name='raise-alert-if-no-letter-ack-file') +@statsd(namespace="tasks") +def letter_raise_alert_if_no_ack_file_for_zip(): + # get a list of zip files since yesterday + zip_file_list = [] + + for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], + subfolder=datetime.utcnow().strftime('%Y-%m-%d'), + suffix='.zip'): + zip_file_list.append(key) + + # get acknowledgement file + ack_file_list = [] + # yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) + yesterday = datetime.utcnow() - timedelta(days=1) + for key in s3.get_list_of_files_by_suffix(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], + subfolder='root/dispatch', suffix='.ACK.txt', lastModified=yesterday): + ack_file_list.append(key) + + today_str = datetime.utcnow().strftime('%Y%m%d') + zip_not_today = [] + + for key in ack_file_list: + if today_str in key: + content = s3.get_s3_file(current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) + for zip_file in content.split('\n'): # each line + s = zip_file.split('|') + for zf in zip_file_list: + if s[0].lower() in zf.lower(): + zip_file_list.remove(zf) + else: + zip_not_today.append(s[0]) + + if zip_file_list: + + raise NoAckFileReceived(message=zip_file_list) + + if zip_not_today: + current_app.logger.info( + "letter ack contains zip that is not for today {} ".format(zip_not_today) + ) diff --git a/app/config.py b/app/config.py index 42ba4177e..63beae32b 100644 --- a/app/config.py +++ b/app/config.py @@ -249,6 +249,11 @@ class Config(object): 'schedule': crontab(hour=17, minute=50), 'options': {'queue': QueueNames.PERIODIC} }, + 'raise-alert-if-no-letter-ack-file': { + 'task': 'raise-alert-if-no-letter-ack-file', + 'schedule': crontab(hour=23, minute=00), + 'options': {'queue': QueueNames.PERIODIC} + }, 'run-letter-api-notifications': { 'task': 'run-letter-api-notifications', 'schedule': crontab(hour=17, minute=40), diff --git a/app/notifications/notifications_letter_callback.py b/app/notifications/notifications_letter_callback.py index ac2de9e5c..05cf890a9 100644 --- a/app/notifications/notifications_letter_callback.py +++ b/app/notifications/notifications_letter_callback.py @@ -53,8 +53,9 @@ def process_letter_response(): message = json.loads(req_json['Message']) filename = message['Records'][0]['s3']['object']['key'] current_app.logger.info('Received file from DVLA: {}'.format(filename)) - current_app.logger.info('DVLA callback: Calling task to update letter notifications') - update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY) + + if 'rs.txt' in filename.lower(): + update_letter_notifications_statuses.apply_async([filename], queue=QueueNames.NOTIFY) return jsonify( result="success", message="DVLA callback succeeded" diff --git a/app/v2/errors.py b/app/v2/errors.py index a70d61d53..bba50b162 100644 --- a/app/v2/errors.py +++ b/app/v2/errors.py @@ -26,6 +26,23 @@ class JobIncompleteError(Exception): } +class NoAckFileReceived(Exception): + def __init__(self, message): + self.message = message + self.status_code = 500 + + def to_dict_v2(self): + return { + 'status_code': self.status_code, + "errors": [ + { + "error": 'NoAckFileReceived', + "message": str(self.message) + } + ] + } + + class TooManyRequestsError(InvalidRequest): status_code = 429 message_template = 'Exceeded send limits ({}) for today' diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index 23f890b9f..d4d3f196c 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -1,7 +1,7 @@ from unittest.mock import call from datetime import datetime, timedelta import pytest - +import pytz from flask import current_app from freezegun import freeze_time @@ -11,7 +11,8 @@ from app.aws.s3 import ( get_s3_file, filter_s3_bucket_objects_within_date_range, remove_transformed_dvla_file, - upload_letters_pdf + upload_letters_pdf, + get_list_of_files_by_suffix, ) from tests.app.conftest import datetime_in_past @@ -173,3 +174,37 @@ def test_upload_letters_pdf_puts_in_tomorrows_bucket_after_half_five(notify_api, # in tomorrow's folder, but still has this evening's timestamp file_location='2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF' ) + + +@freeze_time("2018-01-11 00:00:00") +@pytest.mark.parametrize('suffix_str, days_before, returned_no', [ + ('.ACK.txt', None, 1), + ('.ack.txt', None, 1), + ('.ACK.TXT', None, 1), + ('', None, 2), + ('', 1, 1), +]) +def test_get_list_of_files_by_suffix(notify_api, mocker, suffix_str, days_before, returned_no): + paginator_mock = mocker.patch('app.aws.s3.client') + multiple_pages_s3_object = [ + { + "Contents": [ + single_s3_object_stub('bar/foo.ACK.txt', datetime_in_past(1, 0)), + ] + }, + { + "Contents": [ + single_s3_object_stub('bar/foo1.rs.txt', datetime_in_past(2, 0)), + ] + } + ] + paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object + if (days_before): + key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix=suffix_str, + last_modified=datetime.now(tz=pytz.utc) - timedelta(days=days_before)) + else: + key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix=suffix_str) + + assert sum(1 for x in key) == returned_no + for k in key: + assert k == 'bar/foo.ACK.txt' diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index 83f6a6696..e600f1748 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -34,7 +34,8 @@ from app.celery.scheduled_tasks import ( switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, timeout_notifications, - daily_stats_template_usage_by_month + daily_stats_template_usage_by_month, + letter_raise_alert_if_no_ack_file_for_zip ) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames @@ -60,7 +61,7 @@ from app.models import ( SMS_TYPE ) from app.utils import get_london_midnight_in_utc -from app.v2.errors import JobIncompleteError +from app.v2.errors import JobIncompleteError, NoAckFileReceived from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( @@ -1026,3 +1027,46 @@ def test_dao_fetch_monthly_historical_stats_by_template_null_template_id_not_cou ).all() assert len(result) == 1 + + +def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', lastModified=None): + + if subfolder == '2018-01-11': + return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP'] + if subfolder == 'root/dispatch': + return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + + +def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', lastModified=None): + if subfolder == '2018-01-11': + return ['NOTIFY.20180111175007.ZIP', 'NOTIFY.20180111175008.ZIP', 'NOTIFY.20180111175009.ZIP', + 'NOTIFY.20180111175010.ZIP'] + if subfolder == 'root/dispatch': + return ['root/dispatch/NOTIFY.20180111175733.ACK.txt'] + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_not_raise_alert_if_ack_files_match_zip_list(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + + letter_raise_alert_if_no_ack_file_for_zip() + + assert mock_file_list.call_count == 2 + assert mock_get_file.call_count == 1 + + +@freeze_time('2018-01-11T23:00:00') +def test_letter_not_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db): + mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff) + mock_get_file = mocker.patch("app.aws.s3.get_s3_file", + return_value='NOTIFY.20180111175007.ZIP|20180111175733\n' + 'NOTIFY.20180111175008.ZIP|20180111175734') + with pytest.raises(expected_exception=NoAckFileReceived) as e: + letter_raise_alert_if_no_ack_file_for_zip() + + assert e.value.message == ['NOTIFY.20180111175009.ZIP', 'NOTIFY.20180111175010.ZIP'] + assert mock_file_list.call_count == 2 + assert mock_get_file.call_count == 1 diff --git a/tests/app/notifications/rest/test_callbacks.py b/tests/app/notifications/rest/test_callbacks.py index 0e224871b..e81a91042 100644 --- a/tests/app/notifications/rest/test_callbacks.py +++ b/tests/app/notifications/rest/test_callbacks.py @@ -72,20 +72,22 @@ def test_dvla_callback_autoconfirm_does_not_call_update_letter_notifications_tas def test_dvla_callback_calls_update_letter_notifications_task(client, mocker): update_task = \ mocker.patch('app.notifications.notifications_letter_callback.update_letter_notifications_statuses.apply_async') - data = _sample_sns_s3_callback() + data = _sample_sns_s3_dvla_response_callback() response = dvla_post(client, data) assert response.status_code == 200 assert update_task.called - update_task.assert_called_with(['bar.txt'], queue='notify-internal-tasks') + update_task.assert_called_with(['bar.rs.txt'], queue='notify-internal-tasks') -def test_dvla_callback_does_not_raise_error_parsing_json_for_plaintext_header(client, mocker): - mocker.patch('app.notifications.notifications_letter_callback.update_letter_notifications_statuses.apply_async') - data = _sample_sns_s3_callback() +def test_dvla_ack_calls_does_not_call_letter_notifications_task(client, mocker): + update_task = \ + mocker.patch('app.notifications.notifications_letter_callback.update_letter_notifications_statuses.apply_async') + data = _sample_sns_s3_dvla_ack() response = dvla_post(client, data) assert response.status_code == 200 + update_task.assert_not_called() def test_firetext_callback_should_not_need_auth(client, mocker): @@ -460,7 +462,7 @@ def test_firetext_callback_should_record_statsd(client, notify_db, notify_db_ses app.statsd_client.incr.assert_any_call("callback.firetext.delivered") -def _sample_sns_s3_callback(): +def _sample_sns_s3_dvla_ack(): return json.dumps({ "SigningCertURL": "foo.pem", "UnsubscribeURL": "bar", @@ -471,7 +473,22 @@ def _sample_sns_s3_callback(): "MessageId": "6adbfe0a-d610-509a-9c47-af894e90d32d", "Subject": "Amazon S3 Notification", "TopicArn": "sample-topic-arn", - "Message": '{"Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"eu-west-1","eventTime":"2017-05-16T11:38:41.073Z","eventName":"ObjectCreated:Put","userIdentity":{"principalId":"some-p-id"},"requestParameters":{"sourceIPAddress":"8.8.8.8"},"responseElements":{"x-amz-request-id":"some-r-id","x-amz-id-2":"some-x-am-id"},"s3":{"s3SchemaVersion":"1.0","configurationId":"some-c-id","bucket":{"name":"some-bucket","ownerIdentity":{"principalId":"some-p-id"},"arn":"some-bucket-arn"},"object":{"key":"bar.txt","size":200,"eTag":"some-e-tag","versionId":"some-v-id","sequencer":"some-seq"}}}]}' # noqa + "Message": '{"Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"eu-west-1","eventTime":"2017-05-16T11:38:41.073Z","eventName":"ObjectCreated:Put","userIdentity":{"principalId":"some-p-id"},"requestParameters":{"sourceIPAddress":"8.8.8.8"},"responseElements":{"x-amz-request-id":"some-r-id","x-amz-id-2":"some-x-am-id"},"s3":{"s3SchemaVersion":"1.0","configurationId":"some-c-id","bucket":{"name":"some-bucket","ownerIdentity":{"principalId":"some-p-id"},"arn":"some-bucket-arn"},"object":{"key":"bar.ack.txt","size":200,"eTag":"some-e-tag","versionId":"some-v-id","sequencer":"some-seq"}}}]}' # noqa + }) + + +def _sample_sns_s3_dvla_response_callback(): + return json.dumps({ + "SigningCertURL": "foo.pem", + "UnsubscribeURL": "bar", + "Signature": "some-signature", + "Type": "Notification", + "Timestamp": "2016-05-03T08:35:12.884Z", + "SignatureVersion": "1", + "MessageId": "6adbfe0a-d610-509a-9c47-af894e90d32d", + "Subject": "Amazon S3 Notification", + "TopicArn": "sample-topic-arn", + "Message": '{"Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"eu-west-1","eventTime":"2017-05-16T11:38:41.073Z","eventName":"ObjectCreated:Put","userIdentity":{"principalId":"some-p-id"},"requestParameters":{"sourceIPAddress":"8.8.8.8"},"responseElements":{"x-amz-request-id":"some-r-id","x-amz-id-2":"some-x-am-id"},"s3":{"s3SchemaVersion":"1.0","configurationId":"some-c-id","bucket":{"name":"some-bucket","ownerIdentity":{"principalId":"some-p-id"},"arn":"some-bucket-arn"},"object":{"key":"bar.rs.txt","size":200,"eTag":"some-e-tag","versionId":"some-v-id","sequencer":"some-seq"}}}]}' # noqa })