From 9dc9431550c4f549eb367c91beb4b38ed898f37c Mon Sep 17 00:00:00 2001
From: Rebecca Law <rebecca.law@digital.cabinet-office.gov.uk>
Date: Thu, 19 Mar 2020 09:32:50 +0000
Subject: [PATCH] Group the letters by postage class.

In this PR the letters are grouped and sent by postage type.
This means we can group the first class letters first and start sending them.

This PR is to share an idea... tests have not been written and an order to postage hasn't been added. There is also a PR in the works to add the new postage classes so we should consider that.
---
 app/celery/letters_pdf_tasks.py | 61 +++++++++++++++++----------------
 app/dao/notifications_dao.py    |  5 +--
 2 files changed, 34 insertions(+), 32 deletions(-)

diff --git a/app/celery/letters_pdf_tasks.py b/app/celery/letters_pdf_tasks.py
index 74fc22fcf..a5832ead9 100644
--- a/app/celery/letters_pdf_tasks.py
+++ b/app/celery/letters_pdf_tasks.py
@@ -52,7 +52,7 @@ from app.models import (
     NOTIFICATION_TECHNICAL_FAILURE,
     NOTIFICATION_VALIDATION_FAILED,
     NOTIFICATION_VIRUS_SCAN_FAILED,
-)
+    POSTAGE_TYPES)
 from app.cronitor import cronitor
 
 
@@ -136,39 +136,40 @@ def collate_letter_pdfs_to_be_sent():
         hour=17, minute=30, second=0, microsecond=0
     )
 
-    letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline)
+    for postage in POSTAGE_TYPES:
+        letters_to_print = get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage)
 
-    for i, letters in enumerate(group_letters(letters_to_print)):
-        filenames = [letter['Key'] for letter in letters]
+        for i, letters in enumerate(group_letters(letters_to_print)):
+            filenames = [letter['Key'] for letter in letters]
 
-        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
-        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
-        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
-            date=print_run_deadline.strftime("%Y-%m-%d"),
-            num=i + 1,
-            hash=hash
-        )
-
-        current_app.logger.info(
-            'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
-                len(filenames),
-                dvla_filename,
-                sum(letter['Size'] for letter in letters)
+            hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+            # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+            dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+                date=print_run_deadline.strftime("%Y-%m-%d"),
+                num=i + 1,
+                hash=hash
+            )
+
+            current_app.logger.info(
+                'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
+                    len(filenames),
+                    dvla_filename,
+                    sum(letter['Size'] for letter in letters)
+                )
+            )
+            notify_celery.send_task(
+                name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+                kwargs={
+                    'filenames_to_zip': filenames,
+                    'upload_filename': dvla_filename
+                },
+                queue=QueueNames.PROCESS_FTP,
+                compression='zlib'
             )
-        )
-        notify_celery.send_task(
-            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
-            kwargs={
-                'filenames_to_zip': filenames,
-                'upload_filename': dvla_filename
-            },
-            queue=QueueNames.PROCESS_FTP,
-            compression='zlib'
-        )
 
 
-def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
-    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline)
+def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline, postage):
+    letters_awaiting_sending = dao_get_letters_to_be_printed(print_run_deadline, postage)
 
     letter_pdfs = []
     for letter in letters_awaiting_sending:
@@ -183,7 +184,7 @@ def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline):
             letter_pdfs.append({"Key": letter_file_name, "Size": letter_head['ContentLength']})
         except BotoClientError as e:
             current_app.logger.exception(
-                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}", e)
+                f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}")
 
     return letter_pdfs
 
diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py
index 645bf9090..4c50c0972 100644
--- a/app/dao/notifications_dao.py
+++ b/app/dao/notifications_dao.py
@@ -731,7 +731,7 @@ def notifications_not_yet_sent(should_be_sending_after_seconds, notification_typ
     return notifications
 
 
-def dao_get_letters_to_be_printed(print_run_deadline):
+def dao_get_letters_to_be_printed(print_run_deadline, postage):
     """
     Return all letters created before the print run deadline that have not yet been sent
     """
@@ -739,7 +739,8 @@ def dao_get_letters_to_be_printed(print_run_deadline):
         Notification.created_at < convert_bst_to_utc(print_run_deadline),
         Notification.notification_type == LETTER_TYPE,
         Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type == KEY_TYPE_NORMAL
+        Notification.key_type == KEY_TYPE_NORMAL,
+        Notification.postage == postage
     ).order_by(
         Notification.created_at
     ).all()