Split letters into zip files based on postage class

We will split them into three categories:
- first class
- second class
- international - this zip file will have letters for both europe
and rest-of-world postage classes

app/celery/letters_pdf_tasks.py

@@ -137,39 +137,44 @@ def collate_letter_pdfs_to_be_sent():
 
     letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
 
-    for i, letters in enumerate(group_letters(letters_to_print)):
+    i = 0
-        filenames = [letter['Key'] for letter in letters]
+    for zip_folder, letters_list in letters_to_print.items():
+        for letters in group_letters(letters_list):
+            i += 1
+            filenames = [letter['Key'] for letter in letters]
 
-        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+            dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
-            date=print_run_deadline.strftime("%Y-%m-%d"),
+                date=print_run_deadline.strftime("%Y-%m-%d"),
-            num=i + 1,
+                num=i,
-            hash=hash
+                hash=hash
-        )
+            )
-
+
-        current_app.logger.info(
+            current_app.logger.info(
-            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                len(filenames),
+                    len(filenames),
-                dvla_filename,
+                    dvla_filename,
-                sum(letter['Size'] for letter in letters)
+                    sum(letter['Size'] for letter in letters)
+                )
+            )
+            notify_celery.send_task(
+                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+                kwargs={
+                    'filenames_to_zip': filenames,
+                    'upload_filename': dvla_filename
+                },
+                queue=QueueNames.PROCESS_FTP,
+                compression='zlib'
             )
-        )
-        notify_celery.send_task(
-            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-            kwargs={
-                'filenames_to_zip': filenames,
-                'upload_filename': dvla_filename
-            },
-            queue=QueueNames.PROCESS_FTP,
-            compression='zlib'
-        )
 
 
 def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
     letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
-
+    zip_folders_by_postage = {
-    letter_pdfs = []
+        "first": "first", "second": "second", "europe": "international", "rest-of-world": "international"
+    }
+    letter_pdfs = {"first": [], "second": [], "international": []}
     for letter in letters_awaiting_sending:
         try:
             letter_file_name = get_letter_pdf_filename(
@@ -179,7 +184,9 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
                 postage=letter.postage
             )
             letter_head = s3.head_s3_object(current_app.config['LETTERS_PDF_BUCKET_NAME'], letter_file_name)
-            letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
+            letter_pdfs[
+                zip_folders_by_postage[letter.postage]
+            ].append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
         except BotoClientError as e:
             current_app.logger.exception(
                 f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)

tests/app/celery/test_letters_pdf_tasks.py

@@ -155,20 +155,19 @@ def test_update_billable_units_for_letter_doesnt_update_if_sent_with_test_key(mo
 
 @freeze_time('2020-02-17 18:00:00')
 def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sample_letter_template):
+    # second class
     create_notification(
         template=sample_letter_template,
         status='created',
         reference='ref0',
         created_at=(datetime.now() - timedelta(hours=2))
     )
-
     create_notification(
         template=sample_letter_template,
         status='created',
         reference='ref1',
         created_at=(datetime.now() - timedelta(hours=3))
     )
-
     create_notification(
         template=sample_letter_template,
         status='created',
@@ -176,6 +175,31 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
         created_at=(datetime.now() - timedelta(days=2))
     )
 
+    # first class
+    create_notification(
+        template=sample_letter_template,
+        status='created',
+        reference='first_class',
+        created_at=(datetime.now() - timedelta(hours=4)),
+        postage="first"
+    )
+
+    # international
+    create_notification(
+        template=sample_letter_template,
+        status='created',
+        reference='international',
+        created_at=(datetime.now() - timedelta(days=3)),
+        postage="europe"
+    )
+    create_notification(
+        template=sample_letter_template,
+        status='created',
+        reference='international',
+        created_at=(datetime.now() - timedelta(days=4)),
+        postage="rest-of-world"
+    )
+
     # notifications we don't expect to get sent to print as they are in the wrong status
     for status in ['delivered', 'validation-failed', 'cancelled', 'sending']:
         create_notification(
@@ -203,28 +227,46 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print(notify_api, mocker, sam
     )
 
     mock_s3 = mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[
+        {'ContentLength': 1},
+        {'ContentLength': 1},
         {'ContentLength': 2},
         {'ContentLength': 1},
         {'ContentLength': 3},
+        {'ContentLength': 1},
     ])
 
     results = get_key_and_size_of_letters_to_be_sent_to_print(datetime.now() - timedelta(minutes=30))
 
-    assert mock_s3.call_count == 3
+    assert mock_s3.call_count == 6
     mock_s3.assert_has_calls(
         [
+            call(current_app.config[
+                'LETTERS_PDF_BUCKET_NAME'
+            ], '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF'),
+            call(current_app.config[
+                'LETTERS_PDF_BUCKET_NAME'
+            ], '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF'),
+            call(current_app.config[
+                'LETTERS_PDF_BUCKET_NAME'
+            ], '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'),
             call(current_app.config['LETTERS_PDF_BUCKET_NAME'], '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'),
         ]
     )
 
-    assert len(results) == 3
+    assert results == {
-    assert results == [
+        "first": [{'Key': '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF', 'Size': 1}],
-        {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2},
+        "second": [
-        {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 1},
+            {'Key': '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF', 'Size': 2},
-        {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 3},
+            {'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 3},
-    ]
+            {'Key': '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF', 'Size': 1},
+        ],
+        "international": [
+            {'Key': '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF', 'Size': 1},
+            {'Key': '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF', 'Size': 1},
+        ]
+    }
 
 
 @freeze_time('2020-02-17 18:00:00')
@@ -266,7 +308,7 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
         ]
     )
 
-    assert results == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}]
+    assert results["second"] == [{'Key': '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF', 'Size': 2}]
 
 
 @pytest.mark.parametrize('time_to_run_task', [
@@ -275,20 +317,19 @@ def test_get_key_and_size_of_letters_to_be_sent_to_print_catches_exception(
 ])
 def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mocker, time_to_run_task):
     with freeze_time("2020-02-17 18:00:00"):
+        # second class
         create_notification(
             template=sample_letter_template,
             status='created',
             reference='ref0',
             created_at=(datetime.now() - timedelta(hours=2))
         )
-
         create_notification(
             template=sample_letter_template,
             status='created',
             reference='ref1',
             created_at=(datetime.now() - timedelta(hours=3))
         )
-
         create_notification(
             template=sample_letter_template,
             status='created',
@@ -296,10 +337,38 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
             created_at=(datetime.now() - timedelta(days=2))
         )
 
+        # first class
+        create_notification(
+            template=sample_letter_template,
+            status='created',
+            reference='first_class',
+            created_at=(datetime.now() - timedelta(hours=4)),
+            postage="first"
+        )
+
+        # international
+        create_notification(
+            template=sample_letter_template,
+            status='created',
+            reference='international',
+            created_at=(datetime.now() - timedelta(days=3)),
+            postage="europe"
+        )
+        create_notification(
+            template=sample_letter_template,
+            status='created',
+            reference='international',
+            created_at=(datetime.now() - timedelta(days=4)),
+            postage="rest-of-world"
+        )
+
     mocker.patch('app.celery.tasks.s3.head_s3_object', side_effect=[
+        {'ContentLength': 1},
+        {'ContentLength': 1},
         {'ContentLength': 2},
         {'ContentLength': 1},
         {'ContentLength': 3},
+        {'ContentLength': 1},
     ])
 
     mock_celery = mocker.patch('app.celery.letters_pdf_tasks.notify_celery.send_task')
@@ -308,26 +377,49 @@ def test_collate_letter_pdfs_to_be_sent(notify_api, sample_letter_template, mock
         with freeze_time(time_to_run_task):
             collate_letter_pdfs_to_be_sent()
 
-    assert len(mock_celery.call_args_list) == 2
+    assert len(mock_celery.call_args_list) == 4
     assert mock_celery.call_args_list[0] == call(
         name='zip-and-send-letter-pdfs',
         kwargs={
             'filenames_to_zip': [
-                '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
+                '2020-02-17/NOTIFY.FIRST_CLASS.D.1.C.C.20200217140000.PDF'
-                '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'
             ],
-            'upload_filename': 'NOTIFY.2020-02-17.001.k3x_WqC5KhB6e2DWv9Ma.ZIP'
+            'upload_filename': 'NOTIFY.2020-02-17.001.kHh01fdUxT9iEIYUt5Wx.ZIP'
         },
         queue='process-ftp-tasks',
         compression='zlib'
     )
     assert mock_celery.call_args_list[1] == call(
+        name='zip-and-send-letter-pdfs',
+        kwargs={
+            'filenames_to_zip': [
+                '2020-02-16/NOTIFY.REF2.D.2.C.C.20200215180000.PDF',
+                '2020-02-17/NOTIFY.REF1.D.2.C.C.20200217150000.PDF'
+            ],
+            'upload_filename': 'NOTIFY.2020-02-17.002.k3x_WqC5KhB6e2DWv9Ma.ZIP'
+        },
+        queue='process-ftp-tasks',
+        compression='zlib'
+    )
+    assert mock_celery.call_args_list[2] == call(
         name='zip-and-send-letter-pdfs',
         kwargs={
             'filenames_to_zip': [
                 '2020-02-17/NOTIFY.REF0.D.2.C.C.20200217160000.PDF'
             ],
-            'upload_filename': 'NOTIFY.2020-02-17.002.J85cUw-FWlKuAIOcwdLS.ZIP'
+            'upload_filename': 'NOTIFY.2020-02-17.003.J85cUw-FWlKuAIOcwdLS.ZIP'
+        },
+        queue='process-ftp-tasks',
+        compression='zlib'
+    )
+    assert mock_celery.call_args_list[3] == call(
+        name='zip-and-send-letter-pdfs',
+        kwargs={
+            'filenames_to_zip': [
+                '2020-02-14/NOTIFY.INTERNATIONAL.D.N.C.C.20200213180000.PDF',
+                '2020-02-15/NOTIFY.INTERNATIONAL.D.E.C.C.20200214180000.PDF'
+            ],
+            'upload_filename': 'NOTIFY.2020-02-17.004.ArkHQVgyuvwCZA-dVExE.ZIP'
         },
         queue='process-ftp-tasks',
         compression='zlib'