diff --git a/app/aws/s3.py b/app/aws/s3.py index 2aa7aac39..ed206b3a4 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -4,7 +4,12 @@ from flask import current_app FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv' -def get_s3_job_object(bucket_name, file_location): +def get_s3_file(bucket_name, file_location): + s3_file = get_s3_object(bucket_name, file_location) + return s3_file.get()['Body'].read().decode('utf-8') + + +def get_s3_object(bucket_name, file_location): s3 = resource('s3') return s3.Object(bucket_name, file_location) @@ -12,12 +17,12 @@ def get_s3_job_object(bucket_name, file_location): def get_job_from_s3(service_id, job_id): bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME'] file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id) - obj = get_s3_job_object(bucket_name, file_location) + obj = get_s3_object(bucket_name, file_location) return obj.get()['Body'].read().decode('utf-8') def remove_job_from_s3(service_id, job_id): bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME'] file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id) - obj = get_s3_job_object(bucket_name, file_location) + obj = get_s3_object(bucket_name, file_location) return obj.delete() diff --git a/app/celery/tasks.py b/app/celery/tasks.py index 464e3e1be..48923d228 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,6 +1,7 @@ import random from datetime import (datetime) +from collections import namedtuple from flask import current_app from notifications_utils.recipients import ( @@ -354,3 +355,27 @@ def get_template_class(template_type): # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can # use the same base template return WithSubjectTemplate + + +@notify_celery.task(bind=True, name='update-letter-notifications-statuses') +@statsd(namespace="tasks") +def update_letter_notifications_statuses(self, filename): + bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) + response_file = s3.get_s3_file(bucket_location, filename) + + try: + NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold']) + notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()] + + except TypeError: + current_app.logger.exception('DVLA response file: {} has an invalid format'.format(filename)) + raise + + else: + if notification_updates: + for update in notification_updates: + current_app.logger.info('DVLA update: {}'.format(str(update))) + # TODO: Update notifications with desired status + return notification_updates + else: + current_app.logger.exception('DVLA response file contained no updates') diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py new file mode 100644 index 000000000..38c9966dd --- /dev/null +++ b/tests/app/aws/test_s3.py @@ -0,0 +1,11 @@ +from app.aws.s3 import get_s3_file + + +def test_get_s3_file_makes_correct_call(sample_service, sample_job, mocker): + get_s3_mock = mocker.patch('app.aws.s3.get_s3_object') + get_s3_file('foo-bucket', 'bar-file.txt') + + get_s3_mock.assert_called_with( + 'foo-bucket', + 'bar-file.txt' + ) diff --git a/tests/app/celery/test_tasks.py b/tests/app/celery/test_tasks.py index 8b9631b4f..882ac1fcb 100644 --- a/tests/app/celery/test_tasks.py +++ b/tests/app/celery/test_tasks.py @@ -1071,3 +1071,37 @@ def test_update_dvla_job_to_error(sample_letter_template, sample_letter_job): assert not n.sent_by assert 'error' == Job.query.filter_by(id=sample_letter_job.id).one().job_status + + +def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker): + invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2' + mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file) + + with pytest.raises(TypeError): + update_letter_notifications_statuses(filename='foo.txt') + + +def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker): + s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object') + + with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'): + update_letter_notifications_statuses(filename='foo.txt') + s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt') + + +def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker): + valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted' + mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file) + updates = update_letter_notifications_statuses(filename='foo.txt') + + assert len(updates) == 2 + + assert updates[0].reference == 'ref-foo' + assert updates[0].status == 'Sent' + assert updates[0].page_count == '1' + assert updates[0].cost_threshold == 'Unsorted' + + assert updates[1].reference == 'ref-bar' + assert updates[1].status == 'Sent' + assert updates[1].page_count == '2' + assert updates[1].cost_threshold == 'Sorted'