Simplify putting letters in right postage folders

app/celery/letters_pdf_tasks.py

@@ -42,6 +42,7 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
+    POSTAGE_TYPES,
     RESOLVE_POSTAGE_FOR_FILE_NAME
 )
 from app.cronitor import cronitor
@@ -135,21 +136,18 @@ def collate_letter_pdfs_to_be_sent():
     print_run_deadline = print_run_date.replace(
         hour=17, minute=30, second=0, microsecond=0
     )
+    for postage in POSTAGE_TYPES:
+        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
 
-    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
+        for i, letters in enumerate(group_letters(letters_to_print)):
-
-    for zip_folder, letters_list in letters_to_print.items():
-        i = 0
-        for letters in group_letters(letters_list):
-            i += 1
             filenames = [letter['Key'] for letter in letters]
 
             hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
             # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
             dvla_filename = 'NOTIFY.{date}.{postage}.{num:03}.{hash}.ZIP'.format(
                 date=print_run_deadline.strftime("%Y-%m-%d"),
-                postage=RESOLVE_POSTAGE_FOR_FILE_NAME[zip_folder],
+                postage=RESOLVE_POSTAGE_FOR_FILE_NAME[postage],
-                num=i,
+                num=i + 1,
                 hash=hash
             )
 
@@ -171,9 +169,9 @@ def collate_letter_pdfs_to_be_sent():
             )
 
 
-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
-    letter_pdfs = {"first": [], "second": [], "europe": [], "rest-of-world": []}
+    letter_pdfs = []
     for letter in letters_awaiting_sending:
         try:
             letter_file_name = get_letter_pdf_filename(
@@ -183,7 +181,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
                 postage=letter.postage
             )
             letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
-            letter_pdfs[letter.postage].append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
+            letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
         except BotoClientError as e:
             current_app.logger.exception(
                 f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)

app/dao/notifications_dao.py

@@ -740,7 +740,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline):
+def dao_get_letters_to_be_printed(print_run_deadline, postage):
     """
     Return all letters created before the print run deadline that have not yet been sent
     """
@@ -748,7 +748,8 @@ def dao_get_letters_to_be_printed(print_run_deadline):
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL
+        Notification.key_type == KEY_TYPE_NORMAL,
+        Notification.postage == postage,
     ).order_by(
         Notification.created_at
     ).all()

tests/app/celery/test_letters_pdf_tasks.py

@@ -175,31 +175,6 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
         created_at=(datetime.now() - timedelta(days=2))
     )
 
-    # first class
-    create_notification(
-        template=sample_letter_template,
-        status='created',
-        reference='first_class',
-        created_at=(datetime.now() - timedelta(hours=4)),
-        postage="first"
-    )
-
-    # international
-    create_notification(
-        template=sample_letter_template,
-        status='created',
-        reference='international',
-        created_at=(datetime.now() - timedelta(days=3)),
-        postage="europe"
-    )
-    create_notification(
-        template=sample_letter_template,
-        status='created',
-        reference='international',
-        created_at=(datetime.now() - timedelta(days=4)),
-        postage="rest-of-world"
-    )
-
     # notifications we don't expect to get sent to print as they are in the wrong status
     for status in ['delivered', 'validation-failed', 'cancelled', 'sending']:
         create_notification(
@@ -227,48 +202,29 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
     )
 
     mock_s3 = mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[
-        {'ContentLength': 1},
-        {'ContentLength': 1},
         {'ContentLength': 2},
         {'ContentLength': 1},
         {'ContentLength': 3},
-        {'ContentLength': 1},
     ])
 
-    results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
+    results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second')
 
-    assert mock_s3.call_count == 6
+    assert mock_s3.call_count == 3
     mock_s3.assert_has_calls(
         [
-            call(current_app.config[
-                'LETTERS_PDF_BUCKET_NAME'
-            ], '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF'),
-            call(current_app.config[
-                'LETTERS_PDF_BUCKET_NAME'
-            ], '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF'),
-            call(current_app.config[
-                'LETTERS_PDF_BUCKET_NAME'
-            ], '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'),
         ]
     )
 
-    assert results == {
+    assert len(results) == 3
-        "first": [{'Key': '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF', 'Size': 1}],
+
-        "second": [
+    assert results == [
-            {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2},
+        {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2},
-            {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3},
+        {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1},
-            {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1},
+        {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 3},
-        ],
+    ]
-        "europe": [
-            {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1},
-        ],
-        "rest-of-world": [
-            {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1},
-        ]
-    }
 
 
 @freeze_time('2020-02-17 18:00:00')
@@ -300,7 +256,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
         ClientError(error_response, "File not found")
     ])
 
-    results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
+    results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30), postage='second')
 
     assert mock_head_s3_object.call_count == 2
     mock_head_s3_object.assert_has_calls(
@@ -310,7 +266,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
         ]
     )
 
-    assert results["second"] == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}]
+    assert results == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}]
 
 
 @pytest.mark.parametrize('time_to_run_task', [