Merge pull request #2412 from alphagov/hash-filename

add a hash of letter filenames to the dvla zip file name

app/celery/letters_pdf_tasks.py

@@ -2,6 +2,8 @@ import io
 import math
 from datetime import datetime
 from uuid import UUID
+from hashlib import sha512
+from base64 import urlsafe_b64encode
 
 from PyPDF2.utils import PdfReadError
 from botocore.exceptions import ClientError as BotoClientError
@@ -115,14 +117,24 @@ def collate_letter_pdfs_for_day(date=None):
         # since it is triggered mid afternoon.
         date = datetime.utcnow().strftime("%Y-%m-%d")
 
-    letter_pdfs = s3.get_s3_bucket_objects(
+    letter_pdfs = sorted(
-        current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        s3.get_s3_bucket_objects(
-        subfolder=date
+            current_app.config['LETTERS_PDF_BUCKET_NAME'],
+            subfolder=date
+        ),
+        key=lambda letter: letter['Key']
     )
     for i, letters in enumerate(group_letters(letter_pdfs)):
-        # eg NOTIFY.2018-12-31.001.ZIP
-        dvla_filename = 'NOTIFY.{date}.{num:03}.ZIP'.format(date=date, num=i + 1)
         filenames = [letter['Key'] for letter in letters]
+
+        hash = urlsafe_b64encode(sha512(''.join(filenames).encode()).digest())[:20].decode()
+        # eg NOTIFY.2018-12-31.001.Wjrui5nAvObjPd-3GEL-.ZIP
+        dvla_filename = 'NOTIFY.{date}.{num:03}.{hash}.ZIP'.format(
+            date=date,
+            num=i + 1,
+            hash=hash
+        )
+
         current_app.logger.info(
             'Calling task zip-and-send-letter-pdfs for {} pdfs to upload {} with total size {:,} bytes'.format(
                 len(filenames),

tests/app/celery/test_letters_pdf_tasks.py

@@ -218,7 +218,11 @@ def test_create_letters_gets_the_right_logo_when_service_has_letter_branding_log
 
 
 def test_collate_letter_pdfs_for_day(notify_api, mocker):
-    mock_s3 = mocker.patch('app.celery.tasks.s3.get_s3_bucket_objects')
+    mock_s3 = mocker.patch('app.celery.tasks.s3.get_s3_bucket_objects', return_value=[
+        {'Key': 'B.pDf', 'Size': 2},
+        {'Key': 'A.PDF', 'Size': 1},
+        {'Key': 'C.pdf', 'Size': 3}
+    ])
     mock_group_letters = mocker.patch('app.celery.letters_pdf_tasks.group_letters', return_value=[
         [{'Key': 'A.PDF', 'Size': 1}, {'Key': 'B.pDf', 'Size': 2}],
         [{'Key': 'C.pdf', 'Size': 3}]
@@ -228,16 +232,22 @@ def test_collate_letter_pdfs_for_day(notify_api, mocker):
     collate_letter_pdfs_for_day('2017-01-02')
 
     mock_s3.assert_called_once_with('test-letters-pdf', subfolder='2017-01-02')
-    mock_group_letters.assert_called_once_with(mock_s3.return_value)
+    mock_group_letters.assert_called_once_with(sorted(mock_s3.return_value, key=lambda x: x['Key']))
     assert mock_celery.call_args_list[0] == call(
         name='zip-and-send-letter-pdfs',
-        kwargs={'filenames_to_zip': ['A.PDF', 'B.pDf'], 'upload_filename': 'NOTIFY.2017-01-02.001.ZIP'},
+        kwargs={
+            'filenames_to_zip': ['A.PDF', 'B.pDf'],
+            'upload_filename': 'NOTIFY.2017-01-02.001.oqdjIM2-NAUU9Sm5Slmi.ZIP'
+        },
         queue='process-ftp-tasks',
         compression='zlib'
     )
     assert mock_celery.call_args_list[1] == call(
         name='zip-and-send-letter-pdfs',
-        kwargs={'filenames_to_zip': ['C.pdf'], 'upload_filename': 'NOTIFY.2017-01-02.002.ZIP'},
+        kwargs={
+            'filenames_to_zip': ['C.pdf'],
+            'upload_filename': 'NOTIFY.2017-01-02.002.tdr7hcdPieiqjkVoS4kU.ZIP'
+        },
         queue='process-ftp-tasks',
         compression='zlib'
     )