diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py index c26dc9370..a7a01672b 100644 --- a/app/celery/letters_pdf_tasks.py +++ b/app/celery/letters_pdf_tasks.py @@ -14,6 +14,7 @@ from app import encryption, notify_celery from app.aws import s3 from app.config import QueueNames, TaskNames from app.dao.notifications_dao import ( + dao_get_letters_and_sheets_volume_by_postage, dao_get_letters_to_be_printed, dao_get_notification_by_reference, dao_update_notification, @@ -21,6 +22,7 @@ from app.dao.notifications_dao import ( get_notification_by_id, update_notification_status_by_id, ) +from app.dao.templates_dao import dao_get_template_by_id from app.letters.utils import get_letter_pdf_filename from app.errors import VirusScanError from app.exceptions import NotificationTechnicalFailureException @@ -36,6 +38,8 @@ from app.letters.utils import ( ) from app.models import ( INTERNATIONAL_LETTERS, + INTERNATIONAL_POSTAGE_TYPES, + KEY_TYPE_NORMAL, KEY_TYPE_TEST, NOTIFICATION_CREATED, NOTIFICATION_DELIVERED, @@ -45,7 +49,9 @@ from app.models import ( NOTIFICATION_VIRUS_SCAN_FAILED, POSTAGE_TYPES, RESOLVE_POSTAGE_FOR_FILE_NAME, - INTERNATIONAL_POSTAGE_TYPES) + Service, +) + from app.cronitor import cronitor @@ -131,12 +137,15 @@ def collate_letter_pdfs_to_be_sent(): print_run_deadline = print_run_date.replace( hour=17, minute=30, second=0, microsecond=0 ) + _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline) + for postage in POSTAGE_TYPES: current_app.logger.info(f"starting collate-letter-pdfs-to-be-sent processing for postage class {postage}") letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage) for i, letters in enumerate(group_letters(letters_to_print)): filenames = [letter['Key'] for letter in letters] + service_id = letters[0]['ServiceId'] hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode() @@ -170,6 +179,58 @@ def collate_letter_pdfs_to_be_sent(): current_app.logger.info("finished collate-letter-pdfs-to-be-sent") +def _get_letters_and_sheets_volumes_and_send_to_dvla(print_run_deadline): + letters_volumes = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline) + send_letters_volume_email_to_dvla(letters_volumes) + + +def send_letters_volume_email_to_dvla(letters_volumes): + personalisation = { + 'total_volume': 0, + 'first_class_volume': 0, + 'second_class_volume': 0, + 'international_volume': 0, + 'total_sheets': 0, + 'first_class_sheets': 0, + "second_class_sheets": 0, + 'international_sheets': 0 + } + for item in letters_volumes: + personalisation['total_volume'] += item.letters_count + personalisation['total_sheets'] += item.sheets_count + if f"{item.postage}_class_volume" in personalisation: + personalisation[f"{item.postage}_class_volume"] = item.letters_count + personalisation[f"{item.postage}_class_sheets"] = item.sheets_count + else: + personalisation["international_volume"] += item.letters_count + personalisation["international_sheets"] += item.sheets_count + + template = dao_get_template_by_id(current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID']) + recipient = current_app.config['DVLA_EMAIL_ADDRESS'] + reply_to = template.service.get_default_reply_to_email_address() + service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) + + # avoid circular imports: + from app.notifications.process_notifications import ( + persist_notification, + send_notification_to_queue + ) + + saved_notification = persist_notification( + template_id=template.id, + template_version=template.version, + recipient=recipient, + service=service, + personalisation=personalisation, + notification_type=template.template_type, + api_key_id=None, + key_type=KEY_TYPE_NORMAL, + reply_to_text=reply_to + ) + + send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY) + + def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage): letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage) for letter in letters_awaiting_sending: diff --git a/app/config.py b/app/config.py index 1befe8863..3ca17b412 100644 --- a/app/config.py +++ b/app/config.py @@ -179,6 +179,8 @@ class Config(object): MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84' MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b' NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008' + LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12' + DVLA_EMAIL_ADDRESS = os.getenv('DVLA_EMAIL_ADDRESS') BROKER_URL = 'sqs://' BROKER_TRANSPORT_OPTIONS = { @@ -478,6 +480,7 @@ class Test(Development): FIRETEXT_URL = 'https://example.com/firetext' CBC_PROXY_ENABLED = True + DVLA_EMAIL_ADDRESS = 'some_email@example.com' class Preview(Config): diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index fc7efa14d..bc18a8f43 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -784,6 +784,8 @@ def dao_get_letters_and_sheets_volume_by_postage(print_run_deadline): Notification.key_type == KEY_TYPE_NORMAL, ).group_by( Notification.postage + ).order_by( + Notification.postage ).all() return notifications diff --git a/tests/app/celery/test_letters_pdf_tasks.py b/tests/app/celery/test_letters_pdf_tasks.py index de150a472..3326bfee5 100644 --- a/tests/app/celery/test_letters_pdf_tasks.py +++ b/tests/app/celery/test_letters_pdf_tasks.py @@ -1,6 +1,7 @@ from unittest.mock import call import boto3 +from collections import namedtuple from datetime import datetime, timedelta from moto import mock_s3 from flask import current_app @@ -23,10 +24,13 @@ from app.celery.letters_pdf_tasks import ( process_virus_scan_error, replay_letters_in_error, sanitise_letter, + send_letters_volume_email_to_dvla, update_billable_units_for_letter, _move_invalid_letter_and_update_status, ) from app.config import QueueNames, TaskNames +from app.dao.notifications_dao import get_notifications + from app.letters.utils import ScanErrorType from app.models import ( INTERNATIONAL_LETTERS, @@ -240,6 +244,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam assert results == [ { + 'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2, 'ServiceId': str(sample_letter_template.service_id) @@ -310,7 +315,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception( "2020-02-18 02:00:00", # the next day after midnight, before 5:30pm we expect the same results ]) def test_collate_letter_pdfs_to_be_sent( - notify_api, notify_db_session, mocker, time_to_run_task + notify_api, notify_db_session, mocker, time_to_run_task, letter_volumes_email_template ): with freeze_time("2020-02-17 18:00:00"): service_1 = create_service(service_name="service 1", service_id='f2fe37b0-1301-11eb-aba9-4c3275916899') @@ -381,11 +386,18 @@ def test_collate_letter_pdfs_to_be_sent( ]) mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task') + mock_send_email_to_dvla = mocker.patch( + 'app.celery.letters_pdf_tasks.send_letters_volume_email_to_dvla' + ) with set_config_values(notify_api, {'MAX_LETTER_PDF_COUNT_PER_ZIP': 2}): with freeze_time(time_to_run_task): collate_letter_pdfs_to_be_sent() + mock_send_email_to_dvla.assert_called_once_with([ + (1, 1, 'europe'), (1, 1, 'first'), (1, 1, 'rest-of-world'), (4, 4, 'second') + ]) + assert len(mock_celery.call_args_list) == 6 assert mock_celery.call_args_list[0] == call( name='zip-and-send-letter-pdfs', @@ -454,6 +466,36 @@ def test_collate_letter_pdfs_to_be_sent( ) +def test_send_letters_volume_email_to_dvla(notify_api, notify_db_session, mocker, letter_volumes_email_template): + MockVolume = namedtuple('LettersVolume', ['postage', 'letters_count', 'sheets_count']) + letters_volumes = [ + MockVolume('first', 5, 7), + MockVolume('second', 4, 12), + MockVolume('europe', 1, 3), + MockVolume('rest-of-world', 1, 2), + ] + send_mock = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async') + + send_letters_volume_email_to_dvla(letters_volumes) + + email_to_dvla = get_notifications().all()[0] + + send_mock.assert_called_once_with([str(email_to_dvla.id)], queue=QueueNames.NOTIFY) + + assert str(email_to_dvla.template_id) == current_app.config['LETTERS_VOLUME_EMAIL_TEMPLATE_ID'] + assert email_to_dvla.to == current_app.config['DVLA_EMAIL_ADDRESS'] + assert email_to_dvla.personalisation == { + 'total_volume': 11, + 'first_class_volume': 5, + 'second_class_volume': 4, + 'international_volume': 2, + 'total_sheets': 24, + 'first_class_sheets': 7, + "second_class_sheets": 12, + 'international_sheets': 5 + } + + def test_group_letters_splits_on_file_size(notify_api): letters = [ # ends under max but next one is too big diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 4099d65af..b1a5222e7 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -819,6 +819,34 @@ def create_custom_template(service, user, template_config_name, template_type, c return template +@pytest.fixture(scope='function') +def letter_volumes_email_template(notify_db, + notify_db_session): + service, user = notify_service(notify_db, notify_db_session) + + email_template_content = '\n'.join([ + "((total_volume)) letters (((total_sheets)) sheets) sent via Notify are coming in today''s batch. These include: ", # noqa + "", + "((first_class_volume)) first class letters (((first_class_sheets)) sheets).", + "((second_class_volume)) second class letters (((second_class_sheets)) sheets).", + "((international_volume)) international letters (((international_sheets)) sheets).", + "", + "Thanks", + "", + "GOV.​UK Notify team", + "https://www.gov.uk/notify" + ]) + + return create_custom_template( + service=service, + user=user, + template_config_name='LETTERS_VOLUME_EMAIL_TEMPLATE_ID', + content=email_template_content, + subject="Notify letter volume for ((date)): ((total_volume)) letters, ((total_sheets)) sheets", + template_type='email' + ) + + def notify_service(notify_db, notify_db_session): user = create_user() service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID']) diff --git a/tests/app/dao/notification_dao/test_notification_dao.py b/tests/app/dao/notification_dao/test_notification_dao.py index cc6abb8b7..322600c28 100644 --- a/tests/app/dao/notification_dao/test_notification_dao.py +++ b/tests/app/dao/notification_dao/test_notification_dao.py @@ -1713,18 +1713,16 @@ def test_dao_get_letters_and_sheets_volume_by_postage(notify_db_session): second_service = create_service(service_name='second service', service_id='642bf33b-54b5-45f2-8c13-942a46616704') first_template = create_template(service=first_service, template_type='letter', postage='second') second_template = create_template(service=second_service, template_type='letter', postage='second') - letters = [ - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30), postage='first'), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30), postage='europe'), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30), postage='rest-of-world'), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30), billable_units=3), - create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30), postage='first'), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31), postage='first'), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)), - create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34)) - ] + create_notification(template=first_template, created_at=datetime(2020, 12, 1, 9, 30), postage='first'), + create_notification(template=first_template, created_at=datetime(2020, 12, 1, 12, 30), postage='europe'), + create_notification(template=first_template, created_at=datetime(2020, 12, 1, 13, 30), postage='rest-of-world'), + create_notification(template=first_template, created_at=datetime(2020, 12, 1, 14, 30), billable_units=3), + create_notification(template=first_template, created_at=datetime(2020, 12, 1, 15, 30)), + create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 30), postage='first'), + create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 31), postage='first'), + create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 32)), + create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 33)), + create_notification(template=second_template, created_at=datetime(2020, 12, 1, 8, 34)) results = dao_get_letters_and_sheets_volume_by_postage(print_run_deadline=datetime(2020, 12, 1, 17, 30))