darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 09:15:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

If the notifications that are being deleted are letters then we need to delete the letter from s3 as well.

Browse Source
This commit is contained in:
Rebecca Law
2018-08-08 16:20:25 +01:00
parent 43000b2253
commit d0e9ab4972
6 changed files with 90 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/aws/s3.py
View File

@@ -66,6 +66,17 @@ def get_s3_bucket_objects(bucket_name, subfolder='', older_than=7, limit_days=2)
return all_objects_in_bucket
def get_s3_object_by_prefix(bucket_name, prefix):
boto_client = client('s3', current_app.config['AWS_REGION'])
paginator = boto_client.get_paginator('list_objects_v2')
page_iterator = paginator.paginate(
Bucket=bucket_name,
Prefix=prefix
)
return page_iterator
def filter_s3_bucket_objects_within_date_range(bucket_objects, older_than=7, limit_days=2):
"""
S3 returns the Object['LastModified'] as an 'offset-aware' timestamp so the

80
app/dao/notifications_dao.py
View File

@@ -22,6 +22,8 @@ from sqlalchemy.sql import functions
from notifications_utils.international_billing_rates import INTERNATIONAL_BILLING_RATES
from app import db, create_uuid
from app.aws.s3 import get_s3_object_by_prefix
from app.letters.utils import LETTERS_PDF_FILE_LOCATION_STRUCTURE
from app.utils import midnight_n_days_ago, escape_special_characters
from app.errors import InvalidRequest
from app.models import (
@@ -235,18 +237,18 @@ def get_notifications(filter_dict=None):
@statsd(namespace="dao")
def get_notifications_for_service(
service_id,
filter_dict=None,
page=1,
page_size=None,
limit_days=None,
key_type=None,
personalisation=False,
include_jobs=False,
include_from_test_key=False,
older_than=None,
client_reference=None,
include_one_off=True
service_id,
filter_dict=None,
page=1,
page_size=None,
limit_days=None,
key_type=None,
personalisation=False,
include_jobs=False,
include_from_test_key=False,
older_than=None,
client_reference=None,
include_one_off=True
):
if page_size is None:
page_size = current_app.config['PAGE_SIZE']
@@ -317,21 +319,43 @@ def delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
deleted = 0
for f in flexible_data_retention:
days_of_retention = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=f.days_of_retention)
deleted += db.session.query(Notification).filter(
query = db.session.query(Notification).filter(
func.date(Notification.created_at) < days_of_retention,
Notification.notification_type == f.notification_type,
Notification.service_id == f.service_id
).delete(synchronize_session='fetch')
Notification.notification_type == f.notification_type, Notification.service_id == f.service_id)
_delete_letters_from_s3(notification_type, query)
deleted += query.delete(synchronize_session='fetch')
seven_days_ago = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=7)
services_with_data_retention = [x.service_id for x in flexible_data_retention]
deleted = db.session.query(Notification).filter(
func.date(Notification.created_at) < seven_days_ago,
Notification.notification_type == notification_type,
Notification.service_id.notin_(services_with_data_retention)
).delete(synchronize_session='fetch')
query = db.session.query(Notification).filter(func.date(Notification.created_at) < seven_days_ago,
Notification.notification_type == notification_type,
Notification.service_id.notin_(
services_with_data_retention))
_delete_letters_from_s3(notification_type=notification_type, query=query)
deleted = query.delete(synchronize_session='fetch')
return deleted
def _delete_letters_from_s3(notification_type, query):
if notification_type == LETTER_TYPE:
letters_to_delete_from_s3 = query.all()
for letter in letters_to_delete_from_s3:
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
sent_at = str(letter.sent_at.date())
prefix = LETTERS_PDF_FILE_LOCATION_STRUCTURE.format(
folder=sent_at,
reference=letter.reference,
duplex="D",
letter_class="2",
colour="C",
crown="C" if letter.service.crown else "N",
date=''
).upper()[:-5]
s3_objects = get_s3_object_by_prefix(bucket_name=bucket_name, prefix=prefix)
for s3_object in s3_objects:
s3_object.delete()
@statsd(namespace="dao")
@transactional
def dao_delete_notifications_and_history_by_id(notification_id):
@@ -344,7 +368,6 @@ def dao_delete_notifications_and_history_by_id(notification_id):
def _timeout_notifications(current_statuses, new_status, timeout_start, updated_at):
notifications = Notification.query.filter(
Notification.created_at < timeout_start,
Notification.status.in_(current_statuses),
@@ -407,12 +430,12 @@ def get_total_sent_notifications_in_date_range(start_date, end_date, notificatio
def is_delivery_slow_for_provider(
sent_at,
provider,
threshold,
delivery_time,
service_id,
template_id
sent_at,
provider,
threshold,
delivery_time,
service_id,
template_id
):
count = db.session.query(Notification).filter(
Notification.service_id == service_id,
@@ -447,7 +470,6 @@ def dao_update_notifications_by_reference(references, update_dict):
@statsd(namespace="dao")
def dao_get_notifications_by_to_field(service_id, search_term, notification_type=None, statuses=None):
if notification_type is None:
notification_type = guess_notification_type(search_term)

4
app/letters/utils.py
View File

@@ -8,7 +8,6 @@ from notifications_utils.s3 import s3upload
from app.models import KEY_TYPE_TEST
from app.utils import convert_utc_to_bst
from app.variables import Retention
class ScanErrorType(Enum):
@@ -83,8 +82,7 @@ def upload_letter_pdf(notification, pdf_data, precompiled=False):
filedata=pdf_data,
region=current_app.config['AWS_REGION'],
bucket_name=bucket_name,
file_location=upload_file_name,
tags={Retention.KEY: Retention.ONE_WEEK}
file_location=upload_file_name
)
current_app.logger.info("Uploaded letters PDF {} to {} for notification id {}".format(

4
tests/app/celery/test_letters_pdf_tasks.py
View File

@@ -11,7 +11,6 @@ from requests import RequestException
from sqlalchemy.orm.exc import NoResultFound
from app.errors import VirusScanError
from app.variables import Retention
from app.celery.letters_pdf_tasks import (
create_letters_pdf,
get_letters_pdf,
@@ -112,8 +111,7 @@ def test_create_letters_pdf_calls_s3upload(mocker, sample_letter_notification):
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'],
tags={Retention.KEY: Retention.ONE_WEEK}
region=current_app.config['AWS_REGION']
)

30
tests/app/dao/notification_dao/test_notification_dao_delete_notifications.py
View File

@@ -4,6 +4,7 @@ from datetime import (
timedelta
)
import pytest
from flask import current_app
from freezegun import freeze_time
from app.dao.notifications_dao import delete_notifications_created_more_than_a_week_ago_by_type
from app.models import Notification, NotificationHistory
@@ -25,6 +26,7 @@ from tests.app.db import (
)
def test_should_delete_notifications_by_type_after_seven_days(
sample_service,
mocker,
month,
delete_run_time,
notification_type,
@@ -32,6 +34,7 @@ def test_should_delete_notifications_by_type_after_seven_days(
expected_email_count,
expected_letter_count
):
mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
assert len(Notification.query.all()) == 0
email_template, letter_template, sms_template = _create_templates(sample_service)
# create one notification a day between 1st and 10th from 11:00 to 19:00 of each type
@@ -65,7 +68,8 @@ def test_should_delete_notifications_by_type_after_seven_days(
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
@freeze_time("2016-01-10 12:00:00.000000")
def test_should_not_delete_notification_history(sample_service, notification_type):
def test_should_not_delete_notification_history(sample_service, notification_type, mocker):
mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
with freeze_time('2016-01-01 12:00'):
email_template, letter_template, sms_template = _create_templates(sample_service)
create_notification(template=email_template, status='permanent-failure')
@@ -79,35 +83,47 @@ def test_should_not_delete_notification_history(sample_service, notification_typ
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
def test_delete_notifications_for_days_of_retention(sample_service, notification_type):
def test_delete_notifications_for_days_of_retention(sample_service, notification_type, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
service_with_default_data_retention = create_service(service_name='default data retention')
email_template, letter_template, sms_template = _create_templates(sample_service)
default_email_template, default_letter_template, default_sms_template = _create_templates(
service_with_default_data_retention)
create_notification(template=email_template, status='delivered')
create_notification(template=sms_template, status='permanent-failure')
create_notification(template=letter_template, status='temporary-failure')
create_notification(template=letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow())
create_notification(template=email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow(),
created_at=datetime.utcnow() - timedelta(days=4))
create_notification(template=default_email_template, status='delivered',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_sms_template, status='permanent-failure',
created_at=datetime.utcnow() - timedelta(days=8))
create_notification(template=default_letter_template, status='temporary-failure',
reference='LETTER_REF', sent_at=datetime.utcnow(),
created_at=datetime.utcnow() - timedelta(days=8))
create_service_data_retention(service_id=sample_service.id, notification_type=notification_type)
assert len(Notification.query.all()) == 9
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert len(Notification.query.all()) == 7
assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 1
if notification_type == 'letter':
mock_get_s3.assert_called_with(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
prefix="{}NOTIFY.LETTER_REF.D.2.C.C".format(str(datetime.utcnow().date()))
)
assert mock_get_s3.call_count == 2
else:
mock_get_s3.assert_not_called()
@pytest.mark.parametrize('notification_type', ['sms', 'email', 'letter'])
def test_delete_notifications_keep_data_for_days_of_retention_is_longer(sample_service, notification_type):
def test_delete_notifications_keep_data_for_days_of_retention_is_longer(sample_service, notification_type, mocker):
mock_get_s3 = mocker.patch("app.dao.notifications_dao.get_s3_object_by_prefix")
create_service_data_retention(service_id=sample_service.id, notification_type=notification_type,
days_of_retention=15)
email_template, letter_template, sms_template = _create_templates(sample_service)
@@ -133,10 +149,14 @@ def test_delete_notifications_keep_data_for_days_of_retention_is_longer(sample_s
delete_notifications_created_more_than_a_week_ago_by_type(notification_type)
assert len(Notification.query.filter_by().all()) == 8
assert len(Notification.query.filter_by(notification_type=notification_type).all()) == 2
if notification_type == 'letter':
assert mock_get_s3.called
else:
mock_get_s3.assert_not_called()
def test_delete_notifications_delete_notification_type_for_default_time_if_no_days_of_retention_for_type(
sample_service
sample_service, mocker
):
create_service_data_retention(service_id=sample_service.id, notification_type='sms',
days_of_retention=15)

4
tests/app/letters/test_letter_utils.py
View File

@@ -15,7 +15,6 @@ from app.letters.utils import (
ScanErrorType, move_failed_pdf, get_folder_name
)
from app.models import KEY_TYPE_NORMAL, KEY_TYPE_TEST, PRECOMPILED_TEMPLATE_NAME
from app.variables import Retention
FROZEN_DATE_TIME = "2018-03-14 17:00:00"
@@ -135,8 +134,7 @@ def test_upload_letter_pdf_to_correct_bucket(
bucket_name=current_app.config[bucket_config_name],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'],
tags={Retention.KEY: Retention.ONE_WEEK}
region=current_app.config['AWS_REGION']
)