darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-16 20:48:37 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2031 from alphagov/fix_data_retention

Browse Source 
Use days of retention when deleting notifications in the nightly job.
This commit is contained in:
Rebecca Law
2018-08-10 15:35:42 +01:00
committed by GitHub
parent 0e284701b9 f844a39ea6
commit 85a0fd3b46
9 changed files with 287 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -11,7 +11,6 @@ from requests import RequestException
from sqlalchemy.orm.exc import NoResultFound
from app.errors import VirusScanError
from app.variables import Retention
from app.celery.letters_pdf_tasks import (
create_letters_pdf,
get_letters_pdf,
@@ -112,8 +111,7 @@ def test_create_letters_pdf_calls_s3upload(mocker, sample_letter_notification):
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'],
tags={Retention.KEY: Retention.ONE_WEEK}
region=current_app.config['AWS_REGION']
)

100
tests/app/dao/notification_dao/test_notification_dao.py
View File

@@ -54,11 +54,9 @@ from app.models import (
from tests.app.conftest import (
sample_notification,
sample_template as create_sample_template,
sample_email_template,
sample_service,
sample_job,
sample_notification_history as create_notification_history,
sample_letter_template
)
from tests.app.db import (
create_job,
@@ -607,104 +605,6 @@ def test_update_notification_sets_status(sample_notification):
assert notification_from_db.status == 'failed'
@pytest.mark.parametrize('month, delete_run_time',
[(4, '2016-04-10 23:40'), (1, '2016-01-11 00:40')])
@pytest.mark.parametrize(
'notification_type, expected_sms_count, expected_email_count, expected_letter_count',
[('sms', 7, 10, 10),
('email', 10, 7, 10),
('letter', 10, 10, 7)]
)
def test_should_delete_notifications_by_type_after_seven_days(
sample_service,
month,
delete_run_time,
notification_type,
expected_sms_count,
expected_email_count,
expected_letter_count
):
assert len(Notification.query.all()) == 0
email_template = create_template(service=sample_service, template_type='email')
sms_template = create_template(service=sample_service)
letter_template = create_template(service=sample_service, template_type='letter')
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
for i in range(1, 11):
past_date = '2016-0{0}-{1:02d} {1:02d}:00:00.000000'.format(month, i)
with freeze_time(past_date):
create_notification(template=email_template, created_at=datetime.utcnow(), status="permanent-failure")
create_notification(template=sms_template, created_at=datetime.utcnow(), status="delivered")
create_notification(template=letter_template, created_at=datetime.utcnow(), status="temporary-failure")
all_notifications = Notification.query.all()
assert len(all_notifications) == 30
# Records from before 3rd should be deleted
with freeze_time(delete_run_time):
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
remaining_sms_notifications = Notification.query.filter_by(notification_type='sms').all()
remaining_letter_notifications = Notification.query.filter_by(notification_type='letter').all()
remaining_email_notifications = Notification.query.filter_by(notification_type='email').all()
assert len(remaining_sms_notifications) == expected_sms_count
assert len(remaining_email_notifications) == expected_email_count
assert len(remaining_letter_notifications) == expected_letter_count
if notification_type == 'sms':
notifications_to_check = remaining_sms_notifications
if notification_type == 'email':
notifications_to_check = remaining_email_notifications
if notification_type == 'letter':
notifications_to_check = remaining_letter_notifications
for notification in notifications_to_check:
assert notification.created_at.date() >= date(2016, month, 3)
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
@freeze_time("2016-01-10 12:00:00.000000")
def test_should_not_delete_notification_history(notify_db, notify_db_session, sample_service, notification_type):
with freeze_time('2016-01-01 12:00'):
email_template = sample_email_template(notify_db, notify_db_session, service=sample_service)
sms_template = create_sample_template(notify_db, notify_db_session, service=sample_service)
letter_template = sample_letter_template(sample_service)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=email_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=sms_template
)
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.utcnow(),
status="failed",
service=sample_service,
template=letter_template
)
assert Notification.query.count() == 3
assert NotificationHistory.query.count() == 3
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert Notification.query.count() == 2
assert NotificationHistory.query.count() == 3
@freeze_time("2016-01-10")
def test_should_limit_notifications_return_by_day_limit_plus_one(sample_template):
assert len(Notification.query.all()) == 0

183
tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py Normal file
View File

@@ -0,0 +1,183 @@
from datetime import (
datetime,
date,
timedelta
)
import pytest
from flask import current_app
from freezegun import freeze_time
from app.dao.notifications_dao import delete_notifications_created_more_than_a_week_ago_by_type
from app.models import Notification, NotificationHistory
from tests.app.db import (
create_template,
create_notification,
create_service_data_retention,
create_service
)
@pytest.mark.parametrize('month, delete_run_time',
[(4, '2016-04-10 23:40'), (1, '2016-01-11 00:40')])
@pytest.mark.parametrize(
'notification_type, expected_sms_count, expected_email_count, expected_letter_count',
[('sms', 7, 10, 10),
('email', 10, 7, 10),
('letter', 10, 10, 7)]
)
def test_should_delete_notifications_by_type_after_seven_days(
sample_service,
mocker,
month,
delete_run_time,
notification_type,
expected_sms_count,
expected_email_count,
expected_letter_count
):
mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
assert len(Notification.query.all()) == 0
email_template, letter_template, sms_template = _create_templates(sample_service)
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
for i in range(1, 11):
past_date = '2016-0{0}-{1:02d} {1:02d}:00:00.000000'.format(month, i)
with freeze_time(past_date):
create_notification(template=email_template, created_at=datetime.utcnow(), status="permanent-failure")
create_notification(template=sms_template, created_at=datetime.utcnow(), status="delivered")
create_notification(template=letter_template, created_at=datetime.utcnow(), status="temporary-failure")
all_notifications = Notification.query.all()
assert len(all_notifications) == 30
# Records from before 3rd should be deleted
with freeze_time(delete_run_time):
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
remaining_sms_notifications = Notification.query.filter_by(notification_type='sms').all()
remaining_letter_notifications = Notification.query.filter_by(notification_type='letter').all()
remaining_email_notifications = Notification.query.filter_by(notification_type='email').all()
assert len(remaining_sms_notifications) == expected_sms_count
assert len(remaining_email_notifications) == expected_email_count
assert len(remaining_letter_notifications) == expected_letter_count
if notification_type == 'sms':
notifications_to_check = remaining_sms_notifications
if notification_type == 'email':
notifications_to_check = remaining_email_notifications
if notification_type == 'letter':
notifications_to_check = remaining_letter_notifications
for notification in notifications_to_check:
assert notification.created_at.date() >= date(2016, month, 3)
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
@freeze_time("2016-01-10 12:00:00.000000")
def test_should_not_delete_notification_history(sample_service, notification_type, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
with freeze_time('2016-01-01 12:00'):
email_template, letter_template, sms_template = _create_templates(sample_service)
create_notification(template=email_template, status='permanent-failure')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='permanent-failure')
assert Notification.query.count() == 3
assert NotificationHistory.query.count() == 3
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert Notification.query.count() == 2
assert NotificationHistory.query.count() == 3
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
service_with_default_data_retention = create_service(service_name='default data retention')
email_template, letter_template, sms_template = _create_templates(sample_service)
default_email_template, default_letter_template, default_sms_template = _create_templates(
service_with_default_data_retention)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow())
create_notification(template=email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow(),
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=default_email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow(),
created_at=datetime.utcnow() - timedelta(days=8))
create_service_data_retention(service_id=sample_service.id, notification_type=notification_type)
assert len(Notification.query.all()) == 9
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert len(Notification.query.all()) == 7
assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 1
if notification_type == 'letter':
mock_get_s3.assert_called_with(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
prefix="{}NOTIFY.LETTER_REF.D.2.C.C".format(str(datetime.utcnow().date()))
)
assert mock_get_s3.call_count == 2
else:
mock_get_s3.assert_not_called()
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
def test_delete_notifications_keep_data_for_days_of_retention_is_longer(sample_service, notification_type, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
create_service_data_retention(service_id=sample_service.id, notification_type=notification_type,
days_of_retention=15)
email_template, letter_template, sms_template = _create_templates(sample_service)
service_with_default_data_retention = create_service(service_name='default data retention')
default_email_template, default_letter_template, default_sms_template = _create_templates(
service_with_default_data_retention)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='temporary-failure')
create_notification(template=email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=letter_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=default_email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_letter_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=8))
assert len(Notification.query.all()) == 9
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert len(Notification.query.filter_by().all()) == 8
assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 2
if notification_type == 'letter':
assert mock_get_s3.called
else:
mock_get_s3.assert_not_called()
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type(
sample_service, mocker
):
create_service_data_retention(service_id=sample_service.id, notification_type='sms',
days_of_retention=15)
email_template, letter_template, sms_template = _create_templates(sample_service)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='temporary-failure')
create_notification(template=email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=14))
create_notification(template=letter_template, status='temporary-failure',
created_at=datetime.utcnow() - timedelta(days=14))
assert len(Notification.query.all()) == 6
delete_notifications_created_more_than_a_week_ago_by_type('email')
assert len(Notification.query.filter_by().all()) == 5
assert len(Notification.query.filter_by(notification_type='email').all()) == 1
def _create_templates(sample_service):
email_template = create_template(service=sample_service, template_type='email')
sms_template = create_template(service=sample_service)
letter_template = create_template(service=sample_service, template_type='letter')
return email_template, letter_template, sms_template

22
tests/app/dao/test_service_data_retention_dao.py
View File

@@ -8,10 +8,11 @@ from app.dao.service_data_retention_dao import (
fetch_service_data_retention,
insert_service_data_retention,
update_service_data_retention,
fetch_service_data_retention_by_id
fetch_service_data_retention_by_id,
fetch_service_data_retention_by_notification_type
)
from app.models import ServiceDataRetention
from tests.app.db import create_service
from tests.app.db import create_service, create_service_data_retention
def test_fetch_service_data_retention(sample_service):
@@ -131,3 +132,20 @@ def test_update_service_data_retention_does_not_update_row_if_data_retention_is_
service_id=uuid.uuid4(),
days_of_retention=5)
assert updated_count == 0
@pytest.mark.parametrize('notification_type, alternate',
[('sms', 'email'),
('email', 'sms'),
('letter', 'email'),
('letter', 'sms')]
)
def test_fetch_service_data_retention_by_notification_type(sample_service, notification_type, alternate):
data_retention = create_service_data_retention(service_id=sample_service.id, notification_type=notification_type)
create_service_data_retention(service_id=sample_service.id, notification_type=alternate)
result = fetch_service_data_retention_by_notification_type(sample_service.id, notification_type)
assert result == data_retention
def test_fetch_service_data_retention_by_notification_type_returns_none_when_no_rows(sample_service):
assert not fetch_service_data_retention_by_notification_type(sample_service.id, 'email')

4
tests/app/letters/test_letter_utils.py
View File

@@ -15,7 +15,6 @@ from app.letters.utils import (
ScanErrorType, move_failed_pdf, get_folder_name
)
from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEST, PRECOMPILED_TEMPLATE_NAME
from app.variables import Retention
FROZEN_DATE_TIME = "2018-03-14 17:00:00"
@@ -135,8 +134,7 @@ def test_upload_letter_pdf_to_correct_bucket(
bucket_name=current_app.config[bucket_config_name],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'],
tags={Retention.KEY: Retention.ONE_WEEK}
region=current_app.config['AWS_REGION']
)