Group letters into a max number of files for sending to DVLA

Grouping the letters into a maximum number of files is necessary because
the SQS task needs to be under a certain size. We also compress the task
when sending.

app/celery/letters_pdf_tasks.py

@@ -98,7 +98,8 @@ def collate_letter_pdfs_for_day(date):
         notify_celery.send_task(
             name=TaskNames.ZIP_AND_SEND_LETTER_PDFS,
             kwargs={'filenames_to_zip': filenames},
-            queue=QueueNames.PROCESS_FTP
+            queue=QueueNames.PROCESS_FTP,
+            compression='zlib'
         )
 
 
@@ -112,7 +113,10 @@ def group_letters(letter_pdfs):
     list_of_files = []
     for letter in letter_pdfs:
         if letter['Key'].lower().endswith('.pdf'):
-            if running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE']:
+            if (
+                running_filesize + letter['Size'] > current_app.config['MAX_LETTER_PDF_ZIP_FILESIZE'] or
+                len(list_of_files) >= current_app.config['MAX_LETTER_PDF_COUNT_PER_ZIP']
+            ):
                 yield list_of_files
                 running_filesize = 0
                 list_of_files = []

app/config.py

@@ -129,6 +129,7 @@ class Config(object):
     MAX_VERIFY_CODE_COUNT = 10
 
     MAX_LETTER_PDF_ZIP_FILESIZE = 500 * 1024 * 1024  # 500mb
+    MAX_LETTER_PDF_COUNT_PER_ZIP = 5000
 
     CHECK_PROXY_HEADER = False