Merge pull request #3063 from alphagov/letter-retention-fixes-third-approach

Do not let us delete letters that have not reached a final state

tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py

@@ -212,13 +212,79 @@ def test_delete_notifications_delete_notification_type_for_default_time_if_no_da
     assert Notification.query.filter_by(notification_type='email').count() == 1
 
 
-def test_delete_notifications_does_try_to_delete_from_s3_when_letter_has_not_been_sent(sample_service, mocker):
+@pytest.mark.parametrize(
+    'notification_status', ['validation-failed', 'virus-scan-failed']
+)
+def test_delete_notifications_deletes_letters_not_sent_and_in_final_state_from_table_but_not_s3(
+    sample_service, mocker, notification_status
+):
     mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
     letter_template = create_template(service=sample_service, template_type='letter')
+    create_notification(
+        template=letter_template,
+        status=notification_status,
+        reference='LETTER_REF',
+        created_at=datetime.utcnow() - timedelta(days=14)
+    )
+    assert Notification.query.count() == 1
+    assert NotificationHistory.query.count() == 0
 
-    create_notification(template=letter_template, status='sending',
-                        reference='LETTER_REF')
-    delete_notifications_older_than_retention_by_type('email', qry_limit=1)
+    delete_notifications_older_than_retention_by_type('letter')
+
+    assert Notification.query.count() == 0
+    assert NotificationHistory.query.count() == 1
+    mock_get_s3.assert_not_called()
+
+
+@pytest.mark.parametrize('notification_status', ['delivered', 'returned-letter', 'technical-failure'])
+def test_delete_notifications_deletes_letters_sent_and_in_final_state_from_table_and_s3(
+    sample_service, mocker, notification_status
+):
+    mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
+    letter_template = create_template(service=sample_service, template_type='letter')
+    eight_days_ago = datetime.utcnow() - timedelta(days=8)
+    create_notification(
+        template=letter_template,
+        status=notification_status,
+        reference='LETTER_REF',
+        created_at=eight_days_ago,
+        sent_at=eight_days_ago
+    )
+    assert Notification.query.count() == 1
+    assert NotificationHistory.query.count() == 0
+
+    delete_notifications_older_than_retention_by_type('letter')
+
+    assert Notification.query.count() == 0
+    assert NotificationHistory.query.count() == 1
+    mock_get_s3.assert_called_once_with(
+        bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
+        subfolder="{}/NOTIFY.LETTER_REF.D.2.C.C.{}.PDF".format(
+            str(eight_days_ago.date()),
+            eight_days_ago.strftime('%Y%m%d%H%M%S')
+        )
+    )
+
+
+@pytest.mark.parametrize('notification_status', ['pending-virus-check', 'created', 'sending'])
+def test_delete_notifications_does_not_delete_letters_not_yet_in_final_state(
+    sample_service, mocker, notification_status
+):
+    mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_bucket_objects")
+    letter_template = create_template(service=sample_service, template_type='letter')
+    create_notification(
+        template=letter_template,
+        status=notification_status,
+        reference='LETTER_REF',
+        created_at=datetime.utcnow() - timedelta(days=8),
+    )
+    assert Notification.query.count() == 1
+    assert NotificationHistory.query.count() == 0
+
+    delete_notifications_older_than_retention_by_type('letter')
+
+    assert Notification.query.count() == 1
+    assert NotificationHistory.query.count() == 0
     mock_get_s3.assert_not_called()
 
 

tests/app/db.py

@@ -256,7 +256,7 @@ def create_notification(
     if to_field is None:
         to_field = '+447700900855' if template.template_type == SMS_TYPE else 'test@example.com'
 
-    if status != 'created':
+    if status not in ('created', 'validation-failed', 'virus-scan-failed', 'pending-virus-check'):
         sent_at = sent_at or datetime.utcnow()
         updated_at = updated_at or datetime.utcnow()