Merge pull request #3938 from alphagov/save-original-pdf-uploads

Backup original precompiled uploads to S3

app/config.py

@@ -89,6 +89,7 @@ class Development(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-tools'
     MOU_BUCKET_NAME = 'notify.tools-mou'
     TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
+    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'development-letters-precompiled-originals-backup'
 
     ADMIN_CLIENT_SECRET = 'dev-notify-secret-key'
     API_HOST_NAME = 'http://localhost:6011'
@@ -112,6 +113,7 @@ class Test(Development):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-test'
     MOU_BUCKET_NAME = 'test-mou'
     TRANSIENT_UPLOADED_LETTERS = 'test-transient-uploaded-letters'
+    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'test-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'test'
     API_HOST_NAME = 'http://you-forgot-to-mock-an-api-call-to'
     TEMPLATE_PREVIEW_API_HOST = 'http://localhost:9999'
@@ -131,6 +133,7 @@ class Preview(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-preview'
     MOU_BUCKET_NAME = 'notify.works-mou'
     TRANSIENT_UPLOADED_LETTERS = 'preview-transient-uploaded-letters'
+    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'preview-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'preview'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notify.works'
@@ -148,6 +151,7 @@ class Staging(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-staging'
     MOU_BUCKET_NAME = 'staging-notify.works-mou'
     TRANSIENT_UPLOADED_LETTERS = 'staging-transient-uploaded-letters'
+    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'staging-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'staging'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.staging-notify.works'
@@ -162,6 +166,7 @@ class Live(Config):
     LOGO_UPLOAD_BUCKET_NAME = 'public-logos-production'
     MOU_BUCKET_NAME = 'notifications.service.gov.uk-mou'
     TRANSIENT_UPLOADED_LETTERS = 'production-transient-uploaded-letters'
+    PRECOMPILED_ORIGINALS_BACKUP_LETTERS = 'production-letters-precompiled-originals-backup'
     NOTIFY_ENVIRONMENT = 'live'
     CHECK_PROXY_HEADER = False
     ASSET_DOMAIN = 'static.notifications.service.gov.uk'

app/main/views/uploads.py

@@ -39,6 +39,7 @@ from app.main import main
 from app.main.forms import CsvUploadForm, LetterUploadPostageForm, PDFUploadForm
 from app.models.contact_list import ContactList
 from app.s3_client.s3_letter_upload_client import (
+    backup_original_letter_to_s3,
     get_letter_metadata,
     get_letter_pdf_and_metadata,
     get_transient_letter_file_location,
@@ -216,6 +217,11 @@ def upload_letter(service_id):
                 filename=original_filename,
                 recipient=recipient)
 
+            backup_original_letter_to_s3(
+                pdf_file_bytes,
+                upload_id=upload_id,
+            )
+
         return redirect(
             url_for(
                 'main.uploaded_letter_preview',

app/s3_client/s3_letter_upload_client.py

@@ -10,6 +10,18 @@ def get_transient_letter_file_location(service_id, upload_id):
     return 'service-{}/{}.pdf'.format(service_id, upload_id)
 
 
+def backup_original_letter_to_s3(
+    data,
+    upload_id,
+):
+    utils_s3upload(
+        filedata=data,
+        region=current_app.config['AWS_REGION'],
+        bucket_name=current_app.config['PRECOMPILED_ORIGINALS_BACKUP_LETTERS'],
+        file_location=f'{upload_id}.pdf',
+    )
+
+
 def upload_letter_to_s3(
     data,
     *,

tests/app/main/views/uploads/test_upload_letter.py

@@ -42,7 +42,8 @@ def test_post_upload_letter_redirects_for_valid_file(
             json=lambda: {'file': 'VGhlIHNhbml0aXNlZCBjb250ZW50', 'recipient_address': 'The Queen'}
         )
     )
-    mock_s3 = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mock_s3_upload = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mock_s3_backup = mocker.patch('app.main.views.uploads.backup_original_letter_to_s3')
     mocker.patch('app.main.views.uploads.get_letter_metadata', return_value=LetterMetadata({
         'filename': 'tests/test_pdf_files/one_page_pdf.pdf',
         'page_count': '1',
@@ -64,7 +65,7 @@ def test_post_upload_letter_redirects_for_valid_file(
         )
     assert antivirus_mock.called
 
-    mock_s3.assert_called_once_with(
+    mock_s3_upload.assert_called_once_with(
         b'The sanitised content',
         file_location='service-{}/{}.pdf'.format(SERVICE_ONE_ID, fake_uuid),
         status='valid',
@@ -72,6 +73,13 @@ def test_post_upload_letter_redirects_for_valid_file(
         filename='tests/test_pdf_files/one_page_pdf.pdf',
         recipient='The Queen',
     )
+
+    with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
+        mock_s3_backup.assert_called_once_with(
+            file.read(),
+            upload_id=ANY,
+        )
+
     mock_sanitise.assert_called_once_with(
         ANY,
         allow_international_letters=expected_allow_international,
@@ -108,6 +116,7 @@ def test_post_upload_letter_shows_letter_preview_for_valid_file(
         )
     )
     mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mocker.patch('app.main.views.uploads.backup_original_letter_to_s3')
     mocker.patch('app.main.views.uploads.pdf_page_count', return_value=3)
     mocker.patch('app.main.views.uploads.get_letter_metadata', return_value=LetterMetadata({
         'filename': 'tests/test_pdf_files/one_page_pdf.pdf',
@@ -167,6 +176,7 @@ def test_upload_international_letter_shows_preview_with_no_choice_of_postage(
         json=lambda: {'file': 'VGhlIHNhbml0aXNlZCBjb250ZW50', 'recipient_address': 'The Queen'}
     ))
     mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mocker.patch('app.main.views.uploads.backup_original_letter_to_s3')
     mocker.patch('app.main.views.uploads.pdf_page_count', return_value=3)
     mocker.patch('app.main.views.uploads.get_letter_metadata', return_value=LetterMetadata({
         'filename': 'tests/test_pdf_files/one_page_pdf.pdf',
@@ -230,6 +240,7 @@ def test_post_upload_letter_shows_error_when_no_file_uploaded(client_request):
 
 def test_post_upload_letter_shows_error_when_file_contains_virus(mocker, client_request):
     mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=False)
+    mock_s3_backup = mocker.patch('app.main.views.uploads.backup_original_letter_to_s3')
 
     with open('tests/test_pdf_files/one_page_pdf.pdf', 'rb') as file:
         page = client_request.post(
@@ -240,6 +251,7 @@ def test_post_upload_letter_shows_error_when_file_contains_virus(mocker, client_
         )
     assert page.find('div', class_='banner-dangerous').find('h1').text == 'Your file contains a virus'
     assert normalize_spaces(page.find('label', class_='file-upload-button').text) == 'Upload your file again'
+    mock_s3_backup.assert_not_called()
 
 
 def test_post_choose_upload_file_when_file_is_too_big(mocker, client_request):
@@ -277,7 +289,8 @@ def test_post_choose_upload_file_when_file_is_malformed(mocker, client_request):
 def test_post_upload_letter_with_invalid_file(mocker, client_request, fake_uuid):
     mocker.patch('uuid.uuid4', return_value=fake_uuid)
     mocker.patch('app.main.views.uploads.antivirus_client.scan', return_value=True)
-    mock_s3 = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mock_s3_upload = mocker.patch('app.main.views.uploads.upload_letter_to_s3')
+    mock_s3_backup = mocker.patch('app.main.views.uploads.backup_original_letter_to_s3')
 
     mock_sanitise_response = Mock()
     mock_sanitise_response.raise_for_status.side_effect = RequestException(response=Mock(status_code=400))
@@ -302,7 +315,7 @@ def test_post_upload_letter_with_invalid_file(mocker, client_request, fake_uuid)
             _follow_redirects=True
         )
 
-        mock_s3.assert_called_once_with(
+        mock_s3_upload.assert_called_once_with(
             file_contents,
             file_location='service-{}/{}.pdf'.format(SERVICE_ONE_ID, fake_uuid),
             status='invalid',
@@ -312,6 +325,7 @@ def test_post_upload_letter_with_invalid_file(mocker, client_request, fake_uuid)
             message='content-outside-printable-area'
         )
 
+    mock_s3_backup.assert_not_called()
     assert page.find('div', class_='banner-dangerous').find('h1', {"data-error-type": 'content-outside-printable-area'})
     assert not page.find('button', {'class': 'page-footer__button', 'type': 'submit'})
 

tests/app/s3_client/test_s3_letter_upload_client.py

@@ -1,13 +1,32 @@
 import urllib
+import uuid
 
 from flask import current_app
 
 from app.s3_client.s3_letter_upload_client import (
     LetterMetadata,
+    backup_original_letter_to_s3,
     upload_letter_to_s3,
 )
 
 
+def test_backup_original_letter_to_s3(mocker, notify_admin):
+    s3_mock = mocker.patch('app.s3_client.s3_letter_upload_client.utils_s3upload')
+    upload_id = uuid.uuid4()
+
+    backup_original_letter_to_s3(
+        'pdf_data',
+        upload_id=upload_id,
+    )
+
+    s3_mock.assert_called_once_with(
+        bucket_name=current_app.config['PRECOMPILED_ORIGINALS_BACKUP_LETTERS'],
+        file_location=f'{str(upload_id)}.pdf',
+        filedata='pdf_data',
+        region=current_app.config['AWS_REGION']
+    )
+
+
 def test_upload_letter_to_s3(mocker):
     s3_mock = mocker.patch('app.s3_client.s3_letter_upload_client.utils_s3upload')