Send daily email with letter and sheet volumes to DVLA

app/celery/letters_pdf_tasks.py

@@ -14,6 +14,7 @@ from app import encryption, notify_celery
 from app.aws import s3
 from app.config import QueueNames, TaskNames
 from app.dao.notifications_dao import (
+    dao_get_letters_and_sheets_volume_by_postage,
     dao_get_letters_to_be_printed,
     dao_get_notification_by_reference,
     dao_update_notification,
@@ -21,6 +22,7 @@ from app.dao.notifications_dao import (
     get_notification_by_id,
     update_notification_status_by_id,
 )
+from app.dao.templates_dao import dao_get_template_by_id
 from app.letters.utils import get_letter_pdf_filename
 from app.errors import VirusScanError
 from app.exceptions import NotificationTechnicalFailureException
@@ -36,6 +38,8 @@ from app.letters.utils import (
 )
 from app.models import (
     INTERNATIONAL_LETTERS,
+    INTERNATIONAL_POSTAGE_TYPES,
+    KEY_TYPE_NORMAL,
     KEY_TYPE_TEST,
     NOTIFICATION_CREATED,
     NOTIFICATION_DELIVERED,
@@ -45,7 +49,9 @@ from app.models import (
     NOTIFICATION_VIRUS_SCAN_FAILED,
     POSTAGE_TYPES,
     RESOLVE_POSTAGE_FOR_FILE_NAME,
-    INTERNATIONAL_POSTAGE_TYPES)
+    Service,
+)
+
 from app.cronitor import cronitor
 
 
@@ -131,12 +137,15 @@ def collate_letter_pdfs_to_be_sent():
     print_run_deadline = print_run_date.replace(
         hour=17, minute=30, second=0, microsecond=0
     )
+    _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline)
+
     for postage in POSTAGE_TYPES:
         current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}")
         letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
 
         for i, letters in enumerate(group_letters(letters_to_print)):
             filenames = [letter['Key'] for letter in letters]
+
             service_id = letters[0]['ServiceId']
 
             hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
@@ -170,6 +179,58 @@ def collate_letter_pdfs_to_be_sent():
     current_app.logger.info("finished collate-letter-pdfs-to-be-sent")
 
 
+def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline):
+    letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline)
+    send_letters_volume_email_to_dvla(letters_volumes)
+
+
+def send_letters_volume_email_to_dvla(letters_volumes):
+    personalisation = {
+        'total_volume': 0,
+        'first_class_volume': 0,
+        'second_class_volume': 0,
+        'international_volume': 0,
+        'total_sheets': 0,
+        'first_class_sheets': 0,
+        "second_class_sheets": 0,
+        'international_sheets': 0
+    }
+    for item in letters_volumes:
+        personalisation['total_volume'] += item.letters_count
+        personalisation['total_sheets'] += item.sheets_count
+        if f"{item.postage}_class_volume" in personalisation:
+            personalisation[f"{item.postage}_class_volume"] = item.letters_count
+            personalisation[f"{item.postage}_class_sheets"] = item.sheets_count
+        else:
+            personalisation["international_volume"] += item.letters_count
+            personalisation["international_sheets"] += item.sheets_count
+
+    template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'])
+    recipient = current_app.config['DVLA_EMAIL_ADDRESS']
+    reply_to = template.service.get_default_reply_to_email_address()
+    service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
+
+    # avoid circular imports:
+    from app.notifications.process_notifications import (
+        persist_notification,
+        send_notification_to_queue
+    )
+
+    saved_notification = persist_notification(
+        template_id=template.id,
+        template_version=template.version,
+        recipient=recipient,
+        service=service,
+        personalisation=personalisation,
+        notification_type=template.template_type,
+        api_key_id=None,
+        key_type=KEY_TYPE_NORMAL,
+        reply_to_text=reply_to
+    )
+
+    send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+
+
 def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
     letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
     for letter in letters_awaiting_sending:

app/config.py

@@ -179,6 +179,8 @@ class Config(object):
     MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
     NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
+    LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
+    DVLA_EMAIL_ADDRESS = os.getenv('DVLA_EMAIL_ADDRESS')
 
     BROKER_URL = 'sqs://'
     BROKER_TRANSPORT_OPTIONS = {
@@ -478,6 +480,7 @@ class Test(Development):
     FIRETEXT_URL = 'https://example.com/firetext'
 
     CBC_PROXY_ENABLED = True
+    DVLA_EMAIL_ADDRESS = 'some_email@example.com'
 
 
 class Preview(Config):

app/dao/notifications_dao.py

@@ -784,6 +784,8 @@ def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline):
         Notification.key_type == KEY_TYPE_NORMAL,
     ).group_by(
         Notification.postage
+    ).order_by(
+        Notification.postage
     ).all()
     return notifications
 

tests/app/celery/test_letters_pdf_tasks.py

@@ -1,6 +1,7 @@
 from unittest.mock import call
 
 import boto3
+from collections import namedtuple
 from datetime import datetime, timedelta
 from moto import mock_s3
 from flask import current_app
@@ -23,10 +24,13 @@ from app.celery.letters_pdf_tasks import (
     process_virus_scan_error,
     replay_letters_in_error,
     sanitise_letter,
+    send_letters_volume_email_to_dvla,
     update_billable_units_for_letter,
     _move_invalid_letter_and_update_status,
 )
 from app.config import QueueNames, TaskNames
+from app.dao.notifications_dao import get_notifications
+
 from app.letters.utils import ScanErrorType
 from app.models import (
     INTERNATIONAL_LETTERS,
@@ -240,6 +244,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
 
     assert results == [
         {
+
             'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
             'Size': 2,
             'ServiceId': str(sample_letter_template.service_id)
@@ -310,7 +315,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
     "2020-02-18 02:00:00",  # the next day after midnight, before 5:30pm we expect the same results
 ])
 def test_collate_letter_pdfs_to_be_sent(
-    notify_api, notify_db_session, mocker, time_to_run_task
+    notify_api, notify_db_session, mocker, time_to_run_task, letter_volumes_email_template
 ):
     with freeze_time("2020-02-17 18:00:00"):
         service_1 = create_service(service_name="service 1", service_id='f2fe37b0-1301-11eb-aba9-4c3275916899')
@@ -381,11 +386,18 @@ def test_collate_letter_pdfs_to_be_sent(
     ])
 
     mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task')
+    mock_send_email_to_dvla = mocker.patch(
+        'app.celery.letters_pdf_tasks.send_letters_volume_email_to_dvla'
+    )
 
     with set_config_values(notify_api, {'MAX_LETTER_PDF_COUNT_PER_ZIP': 2}):
         with freeze_time(time_to_run_task):
             collate_letter_pdfs_to_be_sent()
 
+    mock_send_email_to_dvla.assert_called_once_with([
+        (1, 1, 'europe'), (1, 1, 'first'), (1, 1, 'rest-of-world'), (4, 4, 'second')
+    ])
+
     assert len(mock_celery.call_args_list) == 6
     assert mock_celery.call_args_list[0] == call(
         name='zip-and-send-letter-pdfs',
@@ -454,6 +466,36 @@ def test_collate_letter_pdfs_to_be_sent(
     )
 
 
+def test_send_letters_volume_email_to_dvla(notify_api, notify_db_session, mocker, letter_volumes_email_template):
+    MockVolume = namedtuple('LettersVolume', ['postage', 'letters_count', 'sheets_count'])
+    letters_volumes = [
+        MockVolume('first', 5, 7),
+        MockVolume('second', 4, 12),
+        MockVolume('europe', 1, 3),
+        MockVolume('rest-of-world', 1, 2),
+    ]
+    send_mock = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
+
+    send_letters_volume_email_to_dvla(letters_volumes)
+
+    email_to_dvla = get_notifications().all()[0]
+
+    send_mock.assert_called_once_with([str(email_to_dvla.id)], queue=QueueNames.NOTIFY)
+
+    assert str(email_to_dvla.template_id) == current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID']
+    assert email_to_dvla.to == current_app.config['DVLA_EMAIL_ADDRESS']
+    assert email_to_dvla.personalisation == {
+        'total_volume': 11,
+        'first_class_volume': 5,
+        'second_class_volume': 4,
+        'international_volume': 2,
+        'total_sheets': 24,
+        'first_class_sheets': 7,
+        "second_class_sheets": 12,
+        'international_sheets': 5
+    }
+
+
 def test_group_letters_splits_on_file_size(notify_api):
     letters = [
         # ends under max but next one is too big

tests/app/conftest.py

@@ -819,6 +819,34 @@ def create_custom_template(service, user, template_config_name, template_type, c
     return template
 
 
+@pytest.fixture(scope='function')
+def letter_volumes_email_template(notify_db,
+                                  notify_db_session):
+    service, user = notify_service(notify_db, notify_db_session)
+
+    email_template_content = '\n'.join([
+        "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today''s batch. These include: ",  # noqa
+        "",
+        "((first_class_volume)) first class letters (((first_class_sheets)) sheets).",
+        "((second_class_volume)) second class letters (((second_class_sheets)) sheets).",
+        "((international_volume)) international letters (((international_sheets)) sheets).",
+        "",
+        "Thanks",
+        "",
+        "GOV.​UK Notify team",
+        "https://www.gov.uk/notify"
+    ])
+
+    return create_custom_template(
+        service=service,
+        user=user,
+        template_config_name='LETTERS_VOLUME_EMAIL_TEMPLATE_ID',
+        content=email_template_content,
+        subject="Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets",
+        template_type='email'
+    )
+
+
 def notify_service(notify_db, notify_db_session):
     user = create_user()
     service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])

tests/app/dao/notification_dao/test_notification_dao.py

@@ -1713,18 +1713,16 @@ def test_dao_get_letters_and_sheets_volume_by_postage(notify_db_session):
     second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704')
     first_template = create_template(service=first_service, template_type='letter', postage='second')
     second_template = create_template(service=second_service, template_type='letter', postage='second')
-    letters = [
+    create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30), postage='first'),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30), postage='first'),
+    create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30), postage='europe'),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30), postage='europe'),
+    create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30), postage='rest-of-world'),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30), postage='rest-of-world'),
+    create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30), billable_units=3),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30), billable_units=3),
+    create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)),
-        create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)),
+    create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30), postage='first'),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30), postage='first'),
+    create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31), postage='first'),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31), postage='first'),
+    create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)),
+    create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)),
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)),
+    create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34))
-        create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34))
-    ]
 
     results = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline=datetime(2020, 12, 1, 17, 30))