add collate-letter-pdfs task

add collate-letter-pdfs task (name pending). This retrieves a list of
letter pdf files (just the metadata, not the actual data) from s3, and
loops through them, calling the ftp task zip-and-send-letter-pdfs. It
groups them up by adding them to lists while counting the total
filesize, if it gets over a certain filesize (currently set to 500mb)
it breaks at that chunk, sends off that list of files to the ftp app,
and then starts building up a new list.

DVLA have a hard 2gb limit on how big the zip files we can send is -
however we're going to be limited by the amount of memory on the ftp
app well before we get around to handling 2gb of pdf data - so the
limit is 500mb for now. We'll adjust it after we see how ftp performs.

app/celery/letters_pdf_tasks.py

@@ -1,15 +1,15 @@
-from flask import current_app
 import math
+
+from flask import current_app
 from requests import (
     post as requests_post,
     RequestException
 )
-
 from botocore.exceptions import ClientError as BotoClientError
 
 from app import notify_celery
 from app.aws import s3
-from app.config import QueueNames
+from app.config import QueueNames, TaskNames
 from app.dao.notifications_dao import (
     get_notification_by_id,
     update_notification_status_by_id,
@@ -79,3 +79,45 @@ def get_letters_pdf(template, contact_block, org_id, values):
     billable_units = math.ceil(int(resp.headers.get("X-pdf-page-count", 0)) / pages_per_sheet)
 
     return resp.content, billable_units
+
+
+@notify_celery.task(name='collate-letter-pdfs-for-day')
+def collate_letter_pdfs_for_day(date):
+    letter_pdfs = s3.get_s3_bucket_objects(
+        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        subfolder=date
+    )
+    for letters in group_letters(letter_pdfs):
+        filenames = [letter['Key'] for letter in letters]
+        current_app.logger.info(
+            'Calling task zip-and-send-letter-pdfs for {} pdfs of total size {:,} bytes'.format(
+                len(filenames),
+                sum(letter['Size'] for letter in letters)
+            )
+        )
+        notify_celery.send_task(
+            name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
+            kwargs={'filenames': filenames},
+            queue=QueueNames.PROCESS_FTP
+        )
+
+
+def group_letters(letter_pdfs):
+    """
+    Group letters in chunks of MAX_LETTER_PDF_ZIP_FILESIZE. Will add files to lists, never going over that size.
+    If a single file is (somehow) larger than MAX_LETTER_PDF_ZIP_FILESIZE that'll be in a list on it's own.
+    If there are no files, will just exit (rather than yielding an empty list).
+    """
+    running_filesize = 0
+    list_of_files = []
+    for letter in letter_pdfs:
+        if running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']:
+            yield list_of_files
+            running_filesize = 0
+            list_of_files = []
+
+        running_filesize += letter['Size']
+        list_of_files.append(letter)
+
+    if list_of_files:
+        yield list_of_files

app/config.py

@@ -57,6 +57,7 @@ class TaskNames(object):
     DVLA_JOBS = 'send-jobs-to-dvla'
     DVLA_NOTIFICATIONS = 'send-api-notifications-to-dvla'
     PROCESS_INCOMPLETE_JOBS = 'process-incomplete-jobs'
+    ZIP_AND_SEND_LETTER_PDFS = 'zip-and-send-letter-pdfs'
 
 
 class Config(object):
@@ -127,6 +128,8 @@ class Config(object):
     ONE_OFF_MESSAGE_FILENAME = 'Report'
     MAX_VERIFY_CODE_COUNT = 10
 
+    MAX_LETTER_PDF_ZIP_FILESIZE = 500 * 1024 * 1024  # 500mb
+
     CHECK_PROXY_HEADER = False
 
     NOTIFY_SERVICE_ID = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'