Group the letters by postage class.

In this PR the letters are grouped and sent by postage type.
This means we can group the first class letters first and start sending them.

This PR is to share an idea... tests have not been written and an order to postage hasn't been added. There is also a PR in the works to add the new postage classes so we should consider that.

app/celery/letters_pdf_tasks.py

@@ -52,7 +52,7 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
-)
+    POSTAGE_TYPES)
 from app.cronitor import cronitor
 
 
@@ -136,39 +136,40 @@ def collate_letter_pdfs_to_be_sent():
         hour=17, minute=30, second=0, microsecond=0
     )
 
-    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
+    for postage in POSTAGE_TYPES:
+        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
 
-    for i, letters in enumerate(group_letters(letters_to_print)):
-        filenames = [letter['Key'] for letter in letters]
+        for i, letters in enumerate(group_letters(letters_to_print)):
+            filenames = [letter['Key'] for letter in letters]
 
-        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
-            date=print_run_deadline.strftime("%Y-%m-%d"),
-            num=i + 1,
-            hash=hash
-        )
-
-        current_app.logger.info(
-            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                len(filenames),
-                dvla_filename,
-                sum(letter['Size'] for letter in letters)
+            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+            dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+                date=print_run_deadline.strftime("%Y-%m-%d"),
+                num=i + 1,
+                hash=hash
+            )
+
+            current_app.logger.info(
+                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                    len(filenames),
+                    dvla_filename,
+                    sum(letter['Size'] for letter in letters)
+                )
+            )
+            notify_celery.send_task(
+                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+                kwargs={
+                    'filenames_to_zip': filenames,
+                    'upload_filename': dvla_filename
+                },
+                queue=QueueNames.PROCESS_FTP,
+                compression='zlib'
             )
-        )
-        notify_celery.send_task(
-            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-            kwargs={
-                'filenames_to_zip': filenames,
-                'upload_filename': dvla_filename
-            },
-            queue=QueueNames.PROCESS_FTP,
-            compression='zlib'
-        )
 
 
-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
 
     letter_pdfs = []
     for letter in letters_awaiting_sending:
@@ -183,7 +184,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
             letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
         except BotoClientError as e:
             current_app.logger.exception(
-                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
+                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}")
 
     return letter_pdfs
 

app/dao/notifications_dao.py

@@ -731,7 +731,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline):
+def dao_get_letters_to_be_printed(print_run_deadline, postage):
     """
     Return all letters created before the print run deadline that have not yet been sent
     """
@@ -739,7 +739,8 @@ def dao_get_letters_to_be_printed(print_run_deadline):
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL
+        Notification.key_type == KEY_TYPE_NORMAL,
+        Notification.postage == postage
     ).order_by(
         Notification.created_at
     ).all()