Add get_s3_file method for use in DVLA processing

app/aws/s3.py

@@ -4,7 +4,12 @@ from flask import current_app
 FILE_LOCATION_STRUCTURE = 'service-{}-notify/{}.csv'
 
 
-def get_s3_job_object(bucket_name, file_location):
+def get_s3_file(bucket_name, file_location):
+    s3_file = get_s3_object(bucket_name, file_location)
+    return s3_file.get()['Body'].read().decode('utf-8')
+
+
+def get_s3_object(bucket_name, file_location):
     s3 = resource('s3')
     return s3.Object(bucket_name, file_location)
 
@@ -12,12 +17,12 @@ def get_s3_job_object(bucket_name, file_location):
 def get_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.get()['Body'].read().decode('utf-8')
 
 
 def remove_job_from_s3(service_id, job_id):
     bucket_name = current_app.config['CSV_UPLOAD_BUCKET_NAME']
     file_location = FILE_LOCATION_STRUCTURE.format(service_id, job_id)
-    obj = get_s3_job_object(bucket_name, file_location)
+    obj = get_s3_object(bucket_name, file_location)
     return obj.delete()

app/celery/tasks.py

@@ -1,6 +1,7 @@
 import random
 
 from datetime import (datetime)
+from collections import namedtuple
 
 from flask import current_app
 from notifications_utils.recipients import (
@@ -354,3 +355,27 @@ def get_template_class(template_type):
         # since we don't need rendering capabilities (we only need to extract placeholders) both email and letter can
         # use the same base template
         return WithSubjectTemplate
+
+
+@notify_celery.task(bind=True, name='update-letter-notifications-statuses')
+@statsd(namespace="tasks")
+def update_letter_notifications_statuses(self, filename):
+    bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
+    response_file = s3.get_s3_file(bucket_location, filename)
+
+    try:
+        NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
+        notification_updates = [NotificationUpdate(*line.split('|')) for line in response_file.splitlines()]
+
+    except TypeError:
+        current_app.logger.exception('DVLA response file: {} has an invalid format'.format(filename))
+        raise
+
+    else:
+        if notification_updates:
+            for update in notification_updates:
+                current_app.logger.info('DVLA update: {}'.format(str(update)))
+                # TODO: Update notifications with desired status
+            return notification_updates
+        else:
+            current_app.logger.exception('DVLA response file contained no updates')

tests/app/aws/test_s3.py

@@ -0,0 +1,11 @@
+from app.aws.s3 import get_s3_file
+
+
+def test_get_s3_file_makes_correct_call(sample_service, sample_job, mocker):
+    get_s3_mock = mocker.patch('app.aws.s3.get_s3_object')
+    get_s3_file('foo-bucket', 'bar-file.txt')
+
+    get_s3_mock.assert_called_with(
+        'foo-bucket',
+        'bar-file.txt'
+    )

tests/app/celery/test_tasks.py

@@ -1071,3 +1071,37 @@ def test_update_dvla_job_to_error(sample_letter_template, sample_letter_job):
         assert not n.sent_by
 
     assert 'error' == Job.query.filter_by(id=sample_letter_job.id).one().job_status
+
+
+def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker):
+    invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2'
+    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)
+
+    with pytest.raises(TypeError):
+        update_letter_notifications_statuses(filename='foo.txt')
+
+
+def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
+    s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')
+
+    with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
+        update_letter_notifications_statuses(filename='foo.txt')
+        s3_mock.assert_called_with('{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']), 'foo.txt')
+
+
+def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker):
+    valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
+    mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
+    updates = update_letter_notifications_statuses(filename='foo.txt')
+
+    assert len(updates) == 2
+
+    assert updates[0].reference == 'ref-foo'
+    assert updates[0].status == 'Sent'
+    assert updates[0].page_count == '1'
+    assert updates[0].cost_threshold == 'Unsorted'
+
+    assert updates[1].reference == 'ref-bar'
+    assert updates[1].status == 'Sent'
+    assert updates[1].page_count == '2'
+    assert updates[1].cost_threshold == 'Sorted'