darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-02 08:55:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Remove letters-related code (#175)

Browse Source 
This deletes a big ol' chunk of code related to letters. It's not everything—there are still a few things that might be tied to sms/email—but it's the the heart of letters function. SMS and email function should be untouched by this.

Areas affected:

- Things obviously about letters
- PDF tasks, used for precompiling letters
- Virus scanning, used for those PDFs
- FTP, used to send letters to the printer
- Postage stuff
This commit is contained in:
Steven Reilly
2023-03-02 20:20:31 -05:00
committed by GitHub
parent b07b95f795
commit ff4190a8eb
141 changed files with 1108 additions and 12083 deletions
Download Patch File Download Diff File Expand all files Collapse all files

328
tests/app/celery/test_ftp_update_tasks.py
View File

@@ -1,328 +0,0 @@
import os
from collections import defaultdict, namedtuple
from datetime import date, datetime
import pytest
from flask import current_app
from freezegun import freeze_time
from app.celery.tasks import (
check_billable_units,
get_local_billing_date_from_filename,
persist_daily_sorted_letter_counts,
process_updates_from_file,
record_daily_sorted_counts,
update_letter_notifications_statuses,
update_letter_notifications_to_error,
update_letter_notifications_to_sent_to_dvla,
)
from app.dao.daily_sorted_letter_dao import (
dao_get_daily_sorted_letter_by_billing_day,
)
from app.exceptions import DVLAException, NotificationTechnicalFailureException
from app.models import (
NOTIFICATION_CREATED,
NOTIFICATION_DELIVERED,
NOTIFICATION_SENDING,
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
DailySortedLetter,
NotificationHistory,
)
from tests.app.db import (
create_notification,
create_notification_history,
create_service_callback_api,
)
from tests.conftest import set_config
@pytest.fixture
def notification_update():
"""
Returns a namedtuple to use as the argument for the check_billable_units function
"""
NotificationUpdate = namedtuple('NotificationUpdate', ['reference', 'status', 'page_count', 'cost_threshold'])
return NotificationUpdate('REFERENCE_ABC', 'sent', '1', 'cost')
def test_update_letter_notifications_statuses_raises_for_invalid_format(notify_api, mocker):
invalid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=invalid_file)
with pytest.raises(DVLAException) as e:
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
assert 'DVLA response file: {} has an invalid format'.format('NOTIFY-20170823160812-RSP.TXT') in str(e.value)
def test_update_letter_notification_statuses_when_notification_does_not_exist_updates_notification_history(
sample_letter_template,
mocker
):
valid_file = 'ref-foo|Sent|1|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
notification = create_notification_history(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
billable_units=1)
update_letter_notifications_statuses(filename="NOTIFY-20170823160812-RSP.TXT")
updated_history = NotificationHistory.query.filter_by(id=notification.id).one()
assert updated_history.status == NOTIFICATION_DELIVERED
def test_update_letter_notifications_statuses_raises_dvla_exception(notify_api, mocker, sample_letter_template):
valid_file = 'ref-foo|Failed|1|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
billable_units=0)
with pytest.raises(DVLAException) as e:
update_letter_notifications_statuses(filename="failed.txt")
failed = ["ref-foo"]
assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
filename="failed.txt", failures=failed
) in str(e.value)
def test_update_letter_notifications_statuses_calls_with_correct_bucket_location(notify_api, mocker):
s3_mock = mocker.patch('app.celery.tasks.s3.get_s3_object')
with set_config(notify_api, 'NOTIFY_EMAIL_DOMAIN', 'foo.bar'):
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
s3_mock.assert_called_with('{}-ftp'.format(
current_app.config['NOTIFY_EMAIL_DOMAIN']),
'NOTIFY-20170823160812-RSP.TXT',
os.environ.get('AWS_ACCESS_KEY_ID'),
os.environ.get('AWS_SECRET_ACCESS_KEY'),
os.environ.get('AWS_REGION'),
)
def test_update_letter_notifications_statuses_builds_updates_from_content(notify_api, mocker):
valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
update_mock = mocker.patch('app.celery.tasks.process_updates_from_file')
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
update_mock.assert_called_with('ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted')
def test_update_letter_notifications_statuses_builds_updates_list(notify_api, mocker):
valid_file = 'ref-foo|Sent|1|Unsorted\nref-bar|Sent|2|Sorted'
updates = process_updates_from_file(valid_file)
assert len(updates) == 2
assert updates[0].reference == 'ref-foo'
assert updates[0].status == 'Sent'
assert updates[0].page_count == '1'
assert updates[0].cost_threshold == 'Unsorted'
assert updates[1].reference == 'ref-bar'
assert updates[1].status == 'Sent'
assert updates[1].page_count == '2'
assert updates[1].cost_threshold == 'Sorted'
def test_update_letter_notifications_statuses_persisted(notify_api, mocker, sample_letter_template):
sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
billable_units=1)
failed_letter = create_notification(sample_letter_template, reference='ref-bar', status=NOTIFICATION_SENDING,
billable_units=2)
create_service_callback_api(service=sample_letter_template.service, url="https://original_url.com")
valid_file = '{}|Sent|1|Unsorted\n{}|Failed|2|Sorted'.format(
sent_letter.reference, failed_letter.reference)
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
with pytest.raises(expected_exception=DVLAException) as e:
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
assert sent_letter.status == NOTIFICATION_DELIVERED
assert sent_letter.billable_units == 1
assert sent_letter.updated_at
assert failed_letter.status == NOTIFICATION_TEMPORARY_FAILURE
assert failed_letter.billable_units == 2
assert failed_letter.updated_at
assert "DVLA response file: {filename} has failed letters with notification.reference {failures}".format(
filename="NOTIFY-20170823160812-RSP.TXT", failures=[format(failed_letter.reference)]) in str(e.value)
def test_update_letter_notifications_does_not_call_send_callback_if_no_db_entry(notify_api, mocker,
sample_letter_template):
sent_letter = create_notification(sample_letter_template, reference='ref-foo', status=NOTIFICATION_SENDING,
billable_units=0)
valid_file = '{}|Sent|1|Unsorted\n'.format(sent_letter.reference)
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
update_letter_notifications_statuses(filename='NOTIFY-20170823160812-RSP.TXT')
send_mock.assert_not_called()
def test_update_letter_notifications_to_sent_to_dvla_updates_based_on_notification_references(
client,
sample_letter_template
):
first = create_notification(sample_letter_template, reference='first ref')
second = create_notification(sample_letter_template, reference='second ref')
dt = datetime.utcnow()
with freeze_time(dt):
update_letter_notifications_to_sent_to_dvla([first.reference])
assert first.status == NOTIFICATION_SENDING
assert first.sent_by == 'dvla'
assert first.sent_at == dt
assert first.updated_at == dt
assert second.status == NOTIFICATION_CREATED
def test_update_letter_notifications_to_error_updates_based_on_notification_references(
sample_letter_template
):
first = create_notification(sample_letter_template, reference='first ref')
second = create_notification(sample_letter_template, reference='second ref')
create_service_callback_api(service=sample_letter_template.service, url="https://original_url.com")
dt = datetime.utcnow()
with freeze_time(dt):
with pytest.raises(NotificationTechnicalFailureException) as e:
update_letter_notifications_to_error([first.reference])
assert first.reference in str(e.value)
assert first.status == NOTIFICATION_TECHNICAL_FAILURE
assert first.sent_by is None
assert first.sent_at is None
assert first.updated_at == dt
assert second.status == NOTIFICATION_CREATED
def test_check_billable_units_when_billable_units_matches_page_count(
client,
sample_letter_template,
mocker,
notification_update
):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=1)
check_billable_units(notification_update)
mock_logger.assert_not_called()
def test_check_billable_units_when_billable_units_does_not_match_page_count(
client,
sample_letter_template,
mocker,
notification_update
):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
notification = create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=3)
check_billable_units(notification_update)
mock_logger.assert_called_once_with(
'Notification with id {} has 3 billable_units but DVLA says page count is 1'.format(notification.id)
)
@pytest.mark.parametrize('filename_date, billing_date', [
('20170820000000', date(2017, 8, 19)),
('20170120230000', date(2017, 1, 20))
])
def test_get_local_billing_date_from_filename(filename_date, billing_date):
filename = 'NOTIFY-{}-RSP.TXT'.format(filename_date)
result = get_local_billing_date_from_filename(filename)
assert result == billing_date
@freeze_time("2018-01-11 09:00:00")
def test_persist_daily_sorted_letter_counts_saves_sorted_and_unsorted_values(client, notify_db_session):
letter_counts = defaultdict(int, **{'unsorted': 5, 'sorted': 1})
persist_daily_sorted_letter_counts(date.today(), "test.txt", letter_counts)
day = dao_get_daily_sorted_letter_by_billing_day(date.today())
assert day.unsorted_count == 5
assert day.sorted_count == 1
def test_record_daily_sorted_counts_persists_daily_sorted_letter_count(
notify_api,
notify_db_session,
mocker,
):
valid_file = 'Letter1|Sent|1|uNsOrTeD\nLetter2|Sent|2|SORTED\nLetter3|Sent|2|Sorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
assert DailySortedLetter.query.count() == 0
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
daily_sorted_counts = DailySortedLetter.query.all()
assert len(daily_sorted_counts) == 1
assert daily_sorted_counts[0].sorted_count == 2
assert daily_sorted_counts[0].unsorted_count == 1
def test_record_daily_sorted_counts_raises_dvla_exception_with_unknown_sorted_status(
notify_api,
mocker,
):
file_contents = 'ref-foo|Failed|1|invalid\nrow_2|Failed|1|MM'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=file_contents)
filename = "failed.txt"
with pytest.raises(DVLAException) as e:
record_daily_sorted_counts(filename=filename)
assert "DVLA response file: {} contains unknown Sorted status".format(filename) in e.value.message
assert "'mm'" in e.value.message
assert "'invalid'" in e.value.message
def test_record_daily_sorted_counts_persists_daily_sorted_letter_count_with_no_sorted_values(
notify_api,
mocker,
notify_db_session
):
valid_file = 'Letter1|Sent|1|Unsorted\nLetter2|Sent|2|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
assert daily_sorted_letter.unsorted_count == 2
assert daily_sorted_letter.sorted_count == 0
def test_record_daily_sorted_counts_can_run_twice_for_same_file(
notify_api,
mocker,
notify_db_session
):
valid_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=valid_file)
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
assert daily_sorted_letter.unsorted_count == 1
assert daily_sorted_letter.sorted_count == 1
updated_file = 'Letter1|Sent|1|sorted\nLetter2|Sent|2|Unsorted\nLetter3|Sent|2|Unsorted'
mocker.patch('app.celery.tasks.s3.get_s3_file', return_value=updated_file)
record_daily_sorted_counts(filename='NOTIFY-20170823160812-RSP.TXT')
daily_sorted_letter = dao_get_daily_sorted_letter_by_billing_day(date(2017, 8, 23))
assert daily_sorted_letter.unsorted_count == 2
assert daily_sorted_letter.sorted_count == 1

225
tests/app/celery/test_nightly_tasks.py
View File

@@ -2,30 +2,20 @@ from datetime import date, datetime, timedelta
from unittest.mock import ANY, call
import pytest
import pytz
from flask import current_app
from freezegun import freeze_time
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
from app.celery import nightly_tasks
from app.celery.nightly_tasks import (
_delete_notifications_older_than_retention_by_type,
delete_email_notifications_older_than_retention,
delete_inbound_sms,
delete_letter_notifications_older_than_retention,
delete_sms_notifications_older_than_retention,
get_letter_notifications_still_sending_when_they_shouldnt_be,
letter_raise_alert_if_no_ack_file_for_zip,
raise_alert_if_letter_notifications_still_sending,
remove_letter_csv_files,
remove_sms_email_csv_files,
s3,
save_daily_notification_processing_time,
timeout_notifications,
)
from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, FactProcessingTime
from app.models import EMAIL_TYPE, SMS_TYPE, FactProcessingTime
from tests.app.db import (
create_job,
create_notification,
@@ -122,27 +112,6 @@ def test_will_remove_csv_files_for_jobs_older_than_retention_period(
], any_order=True)
@freeze_time('2017-01-01 10:00:00')
def test_remove_csv_files_filters_by_type(mocker, sample_service):
mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3')
"""
Jobs older than seven days are deleted, but only two day's worth (two-day window)
"""
letter_template = create_template(service=sample_service, template_type=LETTER_TYPE)
sms_template = create_template(service=sample_service, template_type=SMS_TYPE)
eight_days_ago = datetime.utcnow() - timedelta(days=8)
job_to_delete = create_job(template=letter_template, created_at=eight_days_ago)
create_job(template=sms_template, created_at=eight_days_ago)
remove_letter_csv_files()
assert s3.remove_job_from_s3.call_args_list == [
call(job_to_delete.service_id, job_to_delete.id),
]
def test_delete_sms_notifications_older_than_retention_calls_child_task(notify_api, mocker):
mocked = mocker.patch('app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type')
delete_sms_notifications_older_than_retention()
@@ -156,23 +125,6 @@ def test_delete_email_notifications_older_than_retentions_calls_child_task(notif
mocked_notifications.assert_called_once_with('email')
def test_delete_letter_notifications_older_than_retention_calls_child_task(notify_api, mocker):
mocked = mocker.patch('app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type')
delete_letter_notifications_older_than_retention()
mocked.assert_called_once_with('letter')
def test_should_not_update_status_of_letter_notifications(client, sample_letter_template):
created_at = datetime.utcnow() - timedelta(days=5)
not1 = create_notification(template=sample_letter_template, status='sending', created_at=created_at)
not2 = create_notification(template=sample_letter_template, status='created', created_at=created_at)
timeout_notifications()
assert not1.status == 'sending'
assert not2.status == 'created'
@freeze_time("2021-12-13T10:00")
def test_timeout_notifications(mocker, sample_notification):
mock_update = mocker.patch('app.celery.nightly_tasks.check_and_queue_callback_task')
@@ -195,181 +147,6 @@ def test_delete_inbound_sms_calls_child_task(notify_api, mocker):
assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1
def test_create_ticket_if_letter_notifications_still_sending(notify_api, mocker):
mock_get_letters = mocker.patch(
"app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be"
)
mock_get_letters.return_value = 1, date(2018, 1, 15)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
raise_alert_if_letter_notifications_still_sending()
mock_create_ticket.assert_called_once_with(
ANY,
subject='[test] Letters still sending',
email_ccs=current_app.config['DVLA_EMAIL_ADDRESSES'],
message=(
"There are 1 letters in the 'sending' state from Monday 15 January. Resolve using "
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-letters-still-in-sending"
),
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
def test_dont_create_ticket_if_letter_notifications_not_still_sending(notify_api, mocker):
mock_get_letters = mocker.patch(
"app.celery.nightly_tasks.get_letter_notifications_still_sending_when_they_shouldnt_be"
)
mock_get_letters.return_value = 0, None
mock_send_ticket_to_zendesk = mocker.patch(
"app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk",
autospec=True
)
raise_alert_if_letter_notifications_still_sending()
mock_send_ticket_to_zendesk.assert_not_called()
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_no_letters_if_sent_a_day_ago(
sample_letter_template
):
today = datetime.utcnow()
one_day_ago = today - timedelta(days=1)
create_notification(template=sample_letter_template, status='sending', sent_at=one_day_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 0
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_only_finds_letters_still_in_sending_status(
sample_letter_template
):
two_days_ago = datetime(2018, 1, 15, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=two_days_ago)
create_notification(template=sample_letter_template, status='delivered', sent_at=two_days_ago)
create_notification(template=sample_letter_template, status='failed', sent_at=two_days_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 1
assert expected_sent_date == date(2018, 1, 15)
@freeze_time("Thursday 17th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_letters_older_than_offset(
sample_letter_template
):
three_days_ago = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=three_days_ago)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 1
assert expected_sent_date == date(2018, 1, 15)
@freeze_time("Sunday 14th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_be_finds_no_letters_on_weekend(
sample_letter_template
):
yesterday = datetime(2018, 1, 13, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday)
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 0
@freeze_time("Monday 15th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_thursday_letters_when_run_on_monday(
sample_letter_template
):
thursday = datetime(2018, 1, 11, 13, 30)
yesterday = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='first')
create_notification(template=sample_letter_template, status='sending', sent_at=thursday, postage='second')
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='second')
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 2
assert expected_sent_date == date(2018, 1, 11)
@freeze_time("Tuesday 16th January 2018 17:00")
def test_get_letter_notifications_still_sending_when_they_shouldnt_finds_friday_letters_when_run_on_tuesday(
sample_letter_template
):
friday = datetime(2018, 1, 12, 13, 30)
yesterday = datetime(2018, 1, 14, 13, 30)
create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='first')
create_notification(template=sample_letter_template, status='sending', sent_at=friday, postage='second')
create_notification(template=sample_letter_template, status='sending', sent_at=yesterday, postage='first')
count, expected_sent_date = get_letter_notifications_still_sending_when_they_shouldnt_be()
assert count == 2
assert expected_sent_date == date(2018, 1, 12)
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_no_ack_file_for_zip_does_not_raise_when_files_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_match)
letter_raise_alert_if_no_ack_file_for_zip()
yesterday = datetime.now(tz=pytz.utc) - timedelta(days=1) # Datatime format on AWS
subfoldername = datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent'
assert mock_file_list.call_count == 2
assert mock_file_list.call_args_list == [
call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=subfoldername, suffix='.TXT'),
call(bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch',
suffix='.ACK.txt', last_modified=yesterday),
]
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_raise_alert_if_ack_files_not_match_zip_list(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=mock_s3_get_list_diff)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.nightly_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
mock_create_ticket.assert_called_once_with(
ANY,
subject="Letter acknowledge error",
message=ANY,
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
assert "['NOTIFY.2018-01-11175009', 'NOTIFY.2018-01-11175010']" in mock_create_ticket.call_args[1]['message']
assert '2018-01-11/zips_sent' in mock_create_ticket.call_args[1]['message']
@freeze_time('2018-01-11T23:00:00')
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db_session):
mock_file_list = mocker.patch("app.aws.s3.get_list_of_files_by_suffix", side_effect=None)
letter_raise_alert_if_no_ack_file_for_zip()
assert mock_file_list.call_count == 2
@freeze_time('2021-01-18T02:00')
@pytest.mark.parametrize('date_provided', [None, '2021-1-17'])
def test_save_daily_notification_processing_time(mocker, sample_template, date_provided):

201
tests/app/celery/test_reporting_tasks.py
View File

@@ -18,7 +18,6 @@ from app.models import (
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_TYPES,
SMS_TYPE,
FactBilling,
@@ -26,7 +25,6 @@ from app.models import (
Notification,
)
from tests.app.db import (
create_letter_rate,
create_notification,
create_notification_history,
create_rate,
@@ -36,11 +34,9 @@ from tests.app.db import (
def mocker_get_rate(
non_letter_rates, letter_rates, notification_type, local_date, crown=None, rate_multiplier=None, post_class="second"
non_letter_rates, notification_type, local_date, crown=None, rate_multiplier=None
):
if notification_type == LETTER_TYPE:
return Decimal(2.1)
elif notification_type == SMS_TYPE:
if notification_type == SMS_TYPE:
return Decimal(1.33)
elif notification_type == EMAIL_TYPE:
return Decimal(0)
@@ -87,10 +83,8 @@ def test_create_nightly_notification_status_triggers_tasks(
@freeze_time('2019-08-01T00:30')
@pytest.mark.parametrize('notification_date, expected_types_aggregated', [
('2019-08-01', set()),
('2019-07-31', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
('2019-07-28', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
('2019-07-27', {LETTER_TYPE}),
('2019-07-22', {LETTER_TYPE}),
('2019-07-31', {EMAIL_TYPE, SMS_TYPE}),
('2019-07-28', {EMAIL_TYPE, SMS_TYPE}),
('2019-07-21', set()),
])
def test_create_nightly_notification_status_triggers_relevant_tasks(
@@ -117,7 +111,7 @@ def test_create_nightly_notification_status_triggers_relevant_tasks(
@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
def test_create_nightly_billing_for_day_checks_history(
sample_service,
sample_letter_template,
sample_sms_template,
mocker
):
yesterday = datetime.now() - timedelta(days=1)
@@ -125,13 +119,13 @@ def test_create_nightly_billing_for_day_checks_history(
create_notification(
created_at=yesterday,
template=sample_letter_template,
template=sample_sms_template,
status='sending',
)
create_notification_history(
created_at=yesterday,
template=sample_letter_template,
template=sample_sms_template,
status='delivered',
)
@@ -143,7 +137,7 @@ def test_create_nightly_billing_for_day_checks_history(
assert len(records) == 1
record = records[0]
assert record.notification_type == LETTER_TYPE
assert record.notification_type == SMS_TYPE
assert record.notifications_sent == 2
@@ -291,116 +285,6 @@ def test_create_nightly_billing_for_day_different_sent_by(
assert record.rate_multiplier == 1.0
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_create_nightly_billing_for_day_different_letter_postage(
notify_db_session,
sample_letter_template,
mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
for _ in range(2):
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=2,
postage='first'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=2,
postage='second'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=1,
postage='europe'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=3,
postage='rest-of-world'
)
records = FactBilling.query.all()
assert len(records) == 0
create_nightly_billing_for_day(str(yesterday.date()))
records = FactBilling.query.order_by('postage').all()
assert len(records) == 4
assert records[0].notification_type == LETTER_TYPE
assert records[0].local_date == datetime.date(yesterday)
assert records[0].postage == 'europe'
assert records[0].notifications_sent == 1
assert records[0].billable_units == 1
assert records[1].notification_type == LETTER_TYPE
assert records[1].local_date == datetime.date(yesterday)
assert records[1].postage == 'first'
assert records[1].notifications_sent == 2
assert records[1].billable_units == 4
assert records[2].notification_type == LETTER_TYPE
assert records[2].local_date == datetime.date(yesterday)
assert records[2].postage == 'rest-of-world'
assert records[2].notifications_sent == 1
assert records[2].billable_units == 3
assert records[3].notification_type == LETTER_TYPE
assert records[3].local_date == datetime.date(yesterday)
assert records[3].postage == 'second'
assert records[3].notifications_sent == 1
assert records[3].billable_units == 2
@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
def test_create_nightly_billing_for_day_letter(
sample_service,
sample_letter_template,
mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
international=False,
rate_multiplier=2.0,
billable_units=2,
)
records = FactBilling.query.all()
assert len(records) == 0
create_nightly_billing_for_day(str(yesterday.date()))
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == 1
record = records[0]
assert record.notification_type == LETTER_TYPE
assert record.local_date == datetime.date(yesterday)
assert record.rate == Decimal(2.1)
assert record.billable_units == 2
assert record.rate_multiplier == 2.0
@pytest.mark.skip(reason="Needs updating for TTS: Timezone handling")
def test_create_nightly_billing_for_day_null_sent_by_sms(
sample_service,
@@ -436,38 +320,16 @@ def test_create_nightly_billing_for_day_null_sent_by_sms(
assert record.provider == 'unknown'
def test_get_rate_for_letter_latest(notify_db_session):
# letter rates should be passed into the get_rate function as a tuple of start_date, crown, sheet_count,
# rate and post_class
new = create_letter_rate(datetime(2017, 12, 1), crown=True, sheet_count=1, rate=0.33, post_class='second')
old = create_letter_rate(datetime(2016, 12, 1), crown=True, sheet_count=1, rate=0.30, post_class='second')
letter_rates = [new, old]
rate = get_rate([], letter_rates, LETTER_TYPE, date(2018, 1, 1), True, 1)
assert rate == Decimal('0.33')
def test_get_rate_for_letter_latest_if_crown_is_none(notify_db_session):
# letter rates should be passed into the get_rate function as a tuple of start_date, crown, sheet_count,
# rate and post_class
crown = create_letter_rate(datetime(2017, 12, 1), crown=True, sheet_count=1, rate=0.33, post_class='second')
non_crown = create_letter_rate(datetime(2017, 12, 1), crown=False, sheet_count=1, rate=0.35, post_class='second')
letter_rates = [crown, non_crown]
rate = get_rate([], letter_rates, LETTER_TYPE, date(2018, 1, 1), crown=None, letter_page_count=1)
assert rate == Decimal('0.33')
def test_get_rate_for_sms_and_email(notify_db_session):
non_letter_rates = [
create_rate(datetime(2017, 12, 1), 0.15, SMS_TYPE),
create_rate(datetime(2017, 12, 1), 0, EMAIL_TYPE)
]
rate = get_rate(non_letter_rates, [], SMS_TYPE, date(2018, 1, 1))
rate = get_rate(non_letter_rates, SMS_TYPE, date(2018, 1, 1))
assert rate == Decimal(0.15)
rate = get_rate(non_letter_rates, [], EMAIL_TYPE, date(2018, 1, 1))
rate = get_rate(non_letter_rates, EMAIL_TYPE, date(2018, 1, 1))
assert rate == Decimal(0)
@@ -568,7 +430,6 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
third_template = create_template(service=second_service, template_type='letter')
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.max)):
@@ -576,25 +437,23 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
create_notification(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
create_notification(template=second_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
create_notification(template=second_template, status='sending', key_type=KEY_TYPE_TEST)
# historical notifications are included
create_notification_history(template=third_template, status='delivered')
create_notification_history(template=second_template, status='delivered')
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.max)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
assert len(FactNotificationStatus.query.all()) == 0
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'letter')
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_type,
@@ -603,7 +462,23 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
assert len(new_fact_data) == 4
email_failure_row = new_fact_data[0]
email_delivered_row = new_fact_data[0]
assert email_delivered_row.template_id == second_template.id
assert email_delivered_row.service_id == second_service.id
assert email_delivered_row.notification_type == 'email'
assert email_delivered_row.notification_status == 'delivered'
assert email_delivered_row.notification_count == 1
assert email_delivered_row.key_type == KEY_TYPE_NORMAL
email_sending_row = new_fact_data[1]
assert email_sending_row.template_id == second_template.id
assert email_sending_row.service_id == second_service.id
assert email_sending_row.notification_type == 'email'
assert email_sending_row.notification_status == 'sending'
assert email_sending_row.notification_count == 1
assert email_sending_row.key_type == KEY_TYPE_TEAM
email_failure_row = new_fact_data[2]
assert email_failure_row.local_date == process_day
assert email_failure_row.template_id == second_template.id
assert email_failure_row.service_id == second_service.id
@@ -613,22 +488,6 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
assert email_failure_row.notification_count == 1
assert email_failure_row.key_type == KEY_TYPE_NORMAL
letter_delivered_row = new_fact_data[1]
assert letter_delivered_row.template_id == third_template.id
assert letter_delivered_row.service_id == second_service.id
assert letter_delivered_row.notification_type == 'letter'
assert letter_delivered_row.notification_status == 'delivered'
assert letter_delivered_row.notification_count == 1
assert letter_delivered_row.key_type == KEY_TYPE_NORMAL
letter_sending_row = new_fact_data[2]
assert letter_sending_row.template_id == third_template.id
assert letter_sending_row.service_id == second_service.id
assert letter_sending_row.notification_type == 'letter'
assert letter_sending_row.notification_status == 'sending'
assert letter_sending_row.notification_count == 1
assert letter_sending_row.key_type == KEY_TYPE_TEAM
sms_delivered_row = new_fact_data[3]
assert sms_delivered_row.template_id == first_template.id
assert sms_delivered_row.service_id == first_service.id

120
tests/app/celery/test_research_mode_tasks.py
View File

@@ -1,21 +1,18 @@
import uuid
from unittest.mock import ANY, call
from unittest.mock import ANY
import pytest
import requests_mock
from flask import current_app, json
from freezegun import freeze_time
from flask import json
from app.celery.research_mode_tasks import (
HTTPError,
create_fake_letter_response_file,
send_email_response,
send_sms_response,
ses_notification_callback,
sns_callback,
)
from app.config import QueueNames
from tests.conftest import Matcher, set_config_values
from tests.conftest import Matcher
dvla_response_file_matcher = Matcher(
'dvla_response_file',
@@ -96,114 +93,3 @@ def test_temp_failure_sns_callback():
assert data['status'] == "4"
assert data['reference'] == "sns_reference"
assert data['CID'] == "1234"
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_uploads_response_file_s3(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
with requests_mock.Mocker() as request_mock:
request_mock.post(
'http://localhost:6011/notifications/letter/dvla',
content=b'{}',
status_code=200
)
create_fake_letter_response_file('random-ref')
mock_s3upload.assert_called_once_with(
filedata='random-ref|Sent|0|Sorted',
region=current_app.config['AWS_REGION'],
bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
file_location=dvla_response_file_matcher
)
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_calls_dvla_callback_on_development(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
mocker.patch('app.celery.research_mode_tasks.s3upload')
with set_config_values(notify_api, {
'NOTIFY_ENVIRONMENT': 'development'
}):
with requests_mock.Mocker() as request_mock:
request_mock.post(
'http://localhost:6011/notifications/letter/dvla',
content=b'{}',
status_code=200
)
create_fake_letter_response_file('random-ref')
assert request_mock.last_request.json() == {
"Type": "Notification",
"MessageId": "some-message-id",
"Message": ANY
}
assert json.loads(request_mock.last_request.json()['Message']) == {
"Records": [
{
"s3": {
"object": {
"key": dvla_response_file_matcher
}
}
}
]
}
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_does_not_call_dvla_callback_on_preview(
notify_api, mocker):
mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=False)
mocker.patch('app.celery.research_mode_tasks.s3upload')
with set_config_values(notify_api, {
'NOTIFY_ENVIRONMENT': 'preview'
}):
with requests_mock.Mocker() as request_mock:
create_fake_letter_response_file('random-ref')
assert request_mock.last_request is None
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_tries_to_create_files_with_other_filenames(notify_api, mocker):
mock_file_exists = mocker.patch('app.celery.research_mode_tasks.file_exists', side_effect=[True, True, False])
mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
create_fake_letter_response_file('random-ref')
assert mock_file_exists.mock_calls == [
call('test.notify.com-ftp', dvla_response_file_matcher),
call('test.notify.com-ftp', dvla_response_file_matcher),
call('test.notify.com-ftp', dvla_response_file_matcher),
]
mock_s3upload.assert_called_once_with(
filedata=ANY,
region=ANY,
bucket_name=ANY,
file_location=dvla_response_file_matcher
)
@freeze_time("2018-01-25 14:00:30")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_create_fake_letter_response_file_gives_up_after_thirty_times(notify_api, mocker):
mock_file_exists = mocker.patch('app.celery.research_mode_tasks.file_exists', return_value=True)
mock_s3upload = mocker.patch('app.celery.research_mode_tasks.s3upload')
with pytest.raises(ValueError):
create_fake_letter_response_file('random-ref')
assert len(mock_file_exists.mock_calls) == 30
assert not mock_s3upload.called

182
tests/app/celery/test_scheduled_tasks.py
View File

@@ -4,7 +4,6 @@ from unittest import mock
from unittest.mock import ANY, call
import pytest
from freezegun import freeze_time
from notifications_utils.clients.zendesk.zendesk_client import (
NotifySupportTicket,
)
@@ -13,23 +12,19 @@ from app.celery import scheduled_tasks
from app.celery.scheduled_tasks import (
check_for_missing_rows_in_completed_jobs,
check_for_services_with_high_failure_rates_or_sending_to_tv_numbers,
check_if_letters_still_in_created,
check_if_letters_still_pending_virus_check,
check_job_status,
delete_invitations,
delete_verify_codes,
replay_created_notifications,
run_scheduled_jobs,
)
from app.config import QueueNames, TaskNames, Test
from app.config import QueueNames, Test
from app.dao.jobs_dao import dao_get_job_by_id
from app.models import (
JOB_STATUS_ERROR,
JOB_STATUS_FINISHED,
JOB_STATUS_IN_PROGRESS,
JOB_STATUS_PENDING,
NOTIFICATION_DELIVERED,
NOTIFICATION_PENDING_VIRUS_CHECK,
)
from tests.app import load_example_csv
from tests.app.db import create_job, create_notification, create_template
@@ -259,28 +254,6 @@ def test_replay_created_notifications(notify_db_session, sample_service, mocker)
queue="send-sms-tasks")
def test_replay_created_notifications_get_pdf_for_templated_letter_tasks_for_letters_not_ready_to_send(
sample_letter_template, mocker
):
mock_task = mocker.patch('app.celery.scheduled_tasks.get_pdf_for_templated_letter.apply_async')
create_notification(template=sample_letter_template, billable_units=0,
created_at=datetime.utcnow() - timedelta(hours=4))
create_notification(template=sample_letter_template, billable_units=0,
created_at=datetime.utcnow() - timedelta(minutes=20))
notification_1 = create_notification(template=sample_letter_template, billable_units=0,
created_at=datetime.utcnow() - timedelta(hours=1, minutes=20))
notification_2 = create_notification(template=sample_letter_template, billable_units=0,
created_at=datetime.utcnow() - timedelta(hours=5))
replay_created_notifications()
calls = [call([str(notification_1.id)], queue=QueueNames.CREATE_LETTERS_PDF),
call([str(notification_2.id)], queue=QueueNames.CREATE_LETTERS_PDF),
]
mock_task.assert_has_calls(calls, any_order=True)
def test_check_job_status_task_does_not_raise_error(sample_template):
create_job(
template=sample_template,
@@ -299,159 +272,6 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
check_job_status()
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_restarts_scan_for_stuck_letters(
mocker,
sample_letter_template
):
mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=True)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task')
create_notification(
template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5401),
reference='one'
)
expected_filename = 'NOTIFY.ONE.D.2.C.20190530122959.PDF'
check_if_letters_still_pending_virus_check()
mock_file_exists.assert_called_once_with('test-letters-scan', expected_filename)
mock_celery.assert_called_once_with(
name=TaskNames.SCAN_FILE,
kwargs={'filename': expected_filename},
queue=QueueNames.ANTIVIRUS
)
assert mock_create_ticket.called is False
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_pending_virus_check_raises_zendesk_if_files_cant_be_found(
mocker,
sample_letter_template
):
mock_file_exists = mocker.patch('app.aws.s3.file_exists', return_value=False)
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_celery = mocker.patch('app.celery.scheduled_tasks.notify_celery.send_task')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5400))
create_notification(template=sample_letter_template,
status=NOTIFICATION_DELIVERED,
created_at=datetime.utcnow() - timedelta(seconds=6000))
notification_1 = create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=5401),
reference='one')
notification_2 = create_notification(template=sample_letter_template,
status=NOTIFICATION_PENDING_VIRUS_CHECK,
created_at=datetime.utcnow() - timedelta(seconds=70000),
reference='two')
check_if_letters_still_pending_virus_check()
assert mock_file_exists.call_count == 2
mock_file_exists.assert_has_calls([
call('test-letters-scan', 'NOTIFY.ONE.D.2.C.20190530122959.PDF'),
call('test-letters-scan', 'NOTIFY.TWO.D.2.C.20190529183320.PDF'),
], any_order=True)
assert mock_celery.called is False
mock_create_ticket.assert_called_once_with(
ANY,
subject='[test] Letters still pending virus check',
message=ANY,
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
assert '2 precompiled letters have been pending-virus-check' in mock_create_ticket.call_args.kwargs['message']
assert f'{(str(notification_1.id), notification_1.reference)}' in mock_create_ticket.call_args.kwargs['message']
assert f'{(str(notification_2.id), notification_2.reference)}' in mock_create_ticket.call_args.kwargs['message']
mock_send_ticket_to_zendesk.assert_called_once()
@freeze_time("2019-05-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_bst(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 1, 12, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 29))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 16, 30))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 29, 17, 29))
create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 5, 28, 10, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 5, 30, 10, 0))
check_if_letters_still_in_created()
message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created."
mock_logger.assert_called_once_with(message)
mock_create_ticket.assert_called_with(
ANY,
message=message,
subject="[test] Letters still in 'created' status",
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
@freeze_time("2019-01-30 14:00:00")
@pytest.mark.skip(reason="Skipping letter-related functionality for now")
def test_check_if_letters_still_in_created_during_utc(mocker, sample_letter_template):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
autospec=True,
)
create_notification(template=sample_letter_template, created_at=datetime(2018, 12, 1, 12, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 29))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 17, 30))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 29, 18, 29))
create_notification(template=sample_letter_template, status='delivered', created_at=datetime(2019, 1, 29, 10, 0))
create_notification(template=sample_letter_template, created_at=datetime(2019, 1, 30, 10, 0))
check_if_letters_still_in_created()
message = "2 letters were created before 17.30 yesterday and still have 'created' status. " \
"Follow runbook to resolve: " \
"https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#deal-with-Letters-still-in-created."
mock_logger.assert_called_once_with(message)
mock_create_ticket.assert_called_once_with(
ANY,
message=message,
subject="[test] Letters still in 'created' status",
ticket_type='incident',
technical_ticket=True,
ticket_categories=['notify_letters']
)
mock_send_ticket_to_zendesk.assert_called_once()
@pytest.mark.parametrize('offset', (
timedelta(days=1),
pytest.param(timedelta(hours=23, minutes=59), marks=pytest.mark.xfail),

535
tests/app/celery/test_tasks.py
View File

@@ -9,7 +9,6 @@ from celery.exceptions import Retry
from freezegun import freeze_time
from notifications_utils.recipients import Row
from notifications_utils.template import (
LetterPrintTemplate,
PlainTextEmailTemplate,
SMSMessageTemplate,
)
@@ -23,13 +22,11 @@ from app.celery.tasks import (
process_incomplete_job,
process_incomplete_jobs,
process_job,
process_returned_letters_list,
process_row,
s3,
save_api_email,
save_api_sms,
save_email,
save_letter,
save_sms,
send_inbound_sms_to_service,
)
@@ -41,13 +38,10 @@ from app.models import (
JOB_STATUS_FINISHED,
JOB_STATUS_IN_PROGRESS,
KEY_TYPE_NORMAL,
LETTER_TYPE,
NOTIFICATION_CREATED,
SMS_TYPE,
Job,
Notification,
NotificationHistory,
ReturnedLetter,
)
from app.serialised_models import SerialisedService, SerialisedTemplate
from app.utils import DATETIME_FORMAT
@@ -57,9 +51,7 @@ from tests.app.db import (
create_api_key,
create_inbound_sms,
create_job,
create_letter_contact,
create_notification,
create_notification_history,
create_reply_to_email,
create_service,
create_service_inbound_api,
@@ -67,7 +59,6 @@ from tests.app.db import (
create_template,
create_user,
)
from tests.conftest import set_config_values
class AnyStringWith(str):
@@ -91,7 +82,6 @@ def test_should_have_decorated_tasks_functions():
assert process_job.__wrapped__.__name__ == 'process_job'
assert save_sms.__wrapped__.__name__ == 'save_sms'
assert save_email.__wrapped__.__name__ == 'save_email'
assert save_letter.__wrapped__.__name__ == 'save_letter'
@pytest.fixture
@@ -298,41 +288,6 @@ def test_should_process_email_job_with_sender_id(email_job_with_placeholders, mo
)
@freeze_time("2016-01-01 11:09:00.061258")
def test_should_process_letter_job(sample_letter_job, mocker):
csv = """address_line_1,address_line_2,address_line_3,address_line_4,postcode,name
A1,A2,A3,A4,A_POST,Alice
"""
s3_mock = mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(csv, {"sender_id": None}))
process_row_mock = mocker.patch('app.celery.tasks.process_row')
mocker.patch('app.celery.tasks.create_uuid', return_value="uuid")
process_job(sample_letter_job.id)
s3_mock.assert_called_once_with(
service_id=str(sample_letter_job.service.id),
job_id=str(sample_letter_job.id)
)
row_call = process_row_mock.mock_calls[0][1]
assert row_call[0].index == 0
assert row_call[0].recipient == ['A1', 'A2', 'A3', 'A4', None, None, 'A_POST', None]
assert row_call[0].personalisation == {
'addressline1': 'A1',
'addressline2': 'A2',
'addressline3': 'A3',
'addressline4': 'A4',
'postcode': 'A_POST'
}
assert row_call[2] == sample_letter_job
assert row_call[3] == sample_letter_job.service
assert process_row_mock.call_count == 1
assert sample_letter_job.job_status == 'finished'
def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
mocker):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
@@ -365,8 +320,6 @@ def test_should_process_all_sms_job(sample_job_with_placeholdered_template,
(SMS_TYPE, True, 'save_sms', 'research-mode-tasks'),
(EMAIL_TYPE, False, 'save_email', 'database-tasks'),
(EMAIL_TYPE, True, 'save_email', 'research-mode-tasks'),
(LETTER_TYPE, False, 'save_letter', 'database-tasks'),
(LETTER_TYPE, True, 'save_letter', 'research-mode-tasks'),
])
def test_process_row_sends_letter_task(template_type, research_mode, expected_function, expected_queue, mocker):
mocker.patch('app.celery.tasks.create_uuid', return_value='noti_uuid')
@@ -930,279 +883,6 @@ def test_save_sms_does_not_send_duplicate_and_does_not_put_in_retry_queue(sample
assert not retry.called
@pytest.mark.parametrize('personalisation, expected_to, expected_normalised', (
({
'addressline1': 'Foo',
'addressline2': 'Bar',
'addressline3': 'Baz',
'addressline4': 'Wibble',
'addressline5': 'Wobble',
'addressline6': 'Wubble',
'postcode': 'SE1 2SA',
}, (
'Foo\n'
'Bar\n'
'Baz\n'
'Wibble\n'
'Wobble\n'
'Wubble\n'
'SE1 2SA'
), (
'foobarbazwibblewobblewubblese12sa'
)),
({
# The address isn’t normalised when we store it in the
# `personalisation` column, but is normalised for storing in the
# `to` column
'addressline2': ' Foo ',
'addressline4': 'Bar',
'addressline6': 'se12sa',
}, (
'Foo\n'
'Bar\n'
'SE1 2SA'
), (
'foobarse12sa'
)),
))
def test_save_letter_saves_letter_to_database(
mocker,
notify_db_session,
personalisation,
expected_to,
expected_normalised,
):
service = create_service()
contact_block = create_letter_contact(service=service, contact_block="Address contact", is_default=True)
template = create_template(service=service, template_type=LETTER_TYPE, reply_to=contact_block.id)
job = create_job(template=template)
mocker.patch('app.celery.tasks.create_random_identifier', return_value="this-is-random-in-real-life")
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
notification_json = _notification_json(
template=job.template,
to='This is ignored for letters',
personalisation=personalisation,
job_id=job.id,
row_number=1
)
notification_id = uuid.uuid4()
created_at = datetime.utcnow()
save_letter(
job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.id == notification_id
assert notification_db.to == expected_to
assert notification_db.normalised_to == expected_normalised
assert notification_db.job_id == job.id
assert notification_db.template_id == job.template.id
assert notification_db.template_version == job.template.version
assert notification_db.status == 'created'
assert notification_db.created_at >= created_at
assert notification_db.notification_type == 'letter'
assert notification_db.sent_at is None
assert notification_db.sent_by is None
assert notification_db.personalisation == personalisation
assert notification_db.reference == "this-is-random-in-real-life"
assert notification_db.reply_to_text == contact_block.contact_block
@pytest.mark.parametrize('last_line_of_address, postage, expected_postage, expected_international',
[('SW1 1AA', 'first', 'first', False),
('SW1 1AA', 'second', 'second', False),
('New Zealand', 'second', 'rest-of-world', True),
('France', 'first', 'europe', True)])
def test_save_letter_saves_letter_to_database_with_correct_postage(
mocker, notify_db_session, last_line_of_address, postage, expected_postage, expected_international
):
service = create_service(service_permissions=[LETTER_TYPE])
template = create_template(service=service, template_type=LETTER_TYPE, postage=postage)
letter_job = create_job(template=template)
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
notification_json = _notification_json(
template=letter_job.template,
to='Foo',
personalisation={'addressline1': 'Foo', 'addressline2': 'Bar', 'postcode': last_line_of_address},
job_id=letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
save_letter(
letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.id == notification_id
assert notification_db.postage == expected_postage
assert notification_db.international == expected_international
@pytest.mark.parametrize('reference_paceholder,', [None, 'ref2'])
def test_save_letter_saves_letter_to_database_with_correct_client_reference(
mocker, notify_db_session, reference_paceholder
):
service = create_service(service_permissions=[LETTER_TYPE])
template = create_template(service=service, template_type=LETTER_TYPE)
letter_job = create_job(template=template)
personalisation = {'addressline1': 'Foo', 'addressline2': 'Bar', 'postcode': 'SW1A 1AA'}
if reference_paceholder:
personalisation['reference'] = reference_paceholder
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
notification_json = _notification_json(
template=letter_job.template,
to='Foo',
personalisation=personalisation,
job_id=letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
save_letter(
letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.id == notification_id
assert notification_db.client_reference == reference_paceholder
def test_save_letter_saves_letter_to_database_with_formatted_postcode(mocker, notify_db_session):
service = create_service(service_permissions=[LETTER_TYPE])
template = create_template(service=service, template_type=LETTER_TYPE)
letter_job = create_job(template=template)
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
notification_json = _notification_json(
template=letter_job.template,
to='Foo',
personalisation={'addressline1': 'Foo', 'addressline2': 'Bar', 'postcode': 'se1 64sa'},
job_id=letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
save_letter(
letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.id == notification_id
assert notification_db.personalisation["postcode"] == "se1 64sa"
def test_save_letter_saves_letter_to_database_right_reply_to(mocker, notify_db_session):
service = create_service()
create_letter_contact(service=service, contact_block="Address contact", is_default=True)
template = create_template(service=service, template_type=LETTER_TYPE, reply_to=None)
job = create_job(template=template)
mocker.patch('app.celery.tasks.create_random_identifier', return_value="this-is-random-in-real-life")
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
personalisation = {
'addressline1': 'Foo',
'addressline2': 'Bar',
'addressline3': 'Baz',
'addressline4': 'Wibble',
'addressline5': 'Wobble',
'addressline6': 'Wubble',
'postcode': 'SE1 3WS',
}
notification_json = _notification_json(
template=job.template,
to='Foo',
personalisation=personalisation,
job_id=job.id,
row_number=1
)
notification_id = uuid.uuid4()
created_at = datetime.utcnow()
save_letter(
job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.id == notification_id
assert notification_db.to == (
'Foo\n'
'Bar\n'
'Baz\n'
'Wibble\n'
'Wobble\n'
'Wubble\n'
'SE1 3WS'
)
assert notification_db.job_id == job.id
assert notification_db.template_id == job.template.id
assert notification_db.template_version == job.template.version
assert notification_db.status == 'created'
assert notification_db.created_at >= created_at
assert notification_db.notification_type == 'letter'
assert notification_db.sent_at is None
assert notification_db.sent_by is None
assert notification_db.personalisation == personalisation
assert notification_db.reference == "this-is-random-in-real-life"
assert not notification_db.reply_to_text
def test_save_letter_uses_template_reply_to_text(mocker, notify_db_session):
service = create_service()
create_letter_contact(service=service, contact_block="Address contact", is_default=True)
template_contact = create_letter_contact(
service=service,
contact_block="Template address contact",
is_default=False
)
template = create_template(
service=service,
template_type=LETTER_TYPE,
reply_to=template_contact.id
)
job = create_job(template=template)
mocker.patch('app.celery.tasks.create_random_identifier', return_value="this-is-random-in-real-life")
mocker.patch('app.celery.tasks.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
personalisation = {
'addressline1': 'Foo',
'addressline2': 'Bar',
'postcode': 'Flob',
}
notification_json = _notification_json(
template=job.template,
to='Foo',
personalisation=personalisation,
job_id=job.id,
row_number=1
)
save_letter(
job.service_id,
uuid.uuid4(),
encryption.encrypt(notification_json),
)
notification_db = Notification.query.one()
assert notification_db.reply_to_text == "Template address contact"
def test_save_sms_uses_sms_sender_reply_to_text(mocker, notify_db_session):
service = create_service_with_defined_sms_sender(sms_sender_value='2028675309')
template = create_template(service=service)
@@ -1241,115 +921,6 @@ def test_save_sms_uses_non_default_sms_sender_reply_to_text_if_provided(mocker,
assert persisted_notification.reply_to_text == 'new-sender'
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
@pytest.mark.parametrize('env', ['staging', 'live'])
def test_save_letter_sets_delivered_letters_as_pdf_permission_in_research_mode_in_staging_live(
notify_api, mocker, notify_db_session, sample_letter_job, env):
sample_letter_job.service.research_mode = True
sample_reference = "this-is-random-in-real-life"
mock_create_fake_letter_response_file = mocker.patch(
'app.celery.research_mode_tasks.create_fake_letter_response_file.apply_async')
mocker.patch('app.celery.tasks.create_random_identifier', return_value=sample_reference)
personalisation = {
'addressline1': 'Foo',
'addressline2': 'Bar',
'postcode': 'Flob',
}
notification_json = _notification_json(
template=sample_letter_job.template,
to='Foo',
personalisation=personalisation,
job_id=sample_letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
with set_config_values(notify_api, {
'NOTIFY_ENVIRONMENT': env
}):
save_letter(
sample_letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
notification = Notification.query.filter(Notification.id == notification_id).one()
assert notification.status == 'delivered'
assert not mock_create_fake_letter_response_file.called
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
@pytest.mark.parametrize('env', ['development', 'preview'])
def test_save_letter_calls_create_fake_response_for_letters_in_research_mode_on_development_preview(
notify_api, mocker, notify_db_session, sample_letter_job, env):
sample_letter_job.service.research_mode = True
sample_reference = "this-is-random-in-real-life"
mock_create_fake_letter_response_file = mocker.patch(
'app.celery.research_mode_tasks.create_fake_letter_response_file.apply_async')
mocker.patch('app.celery.tasks.create_random_identifier', return_value=sample_reference)
personalisation = {
'addressline1': 'Foo',
'addressline2': 'Bar',
'postcode': 'Flob',
}
notification_json = _notification_json(
template=sample_letter_job.template,
to='Foo',
personalisation=personalisation,
job_id=sample_letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
with set_config_values(notify_api, {
'NOTIFY_ENVIRONMENT': env
}):
save_letter(
sample_letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
mock_create_fake_letter_response_file.assert_called_once_with(
(sample_reference,),
queue=QueueNames.RESEARCH_MODE
)
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_save_letter_calls_get_pdf_for_templated_letter_task_not_in_research(
mocker, notify_db_session, sample_letter_job):
mock_create_letters_pdf = mocker.patch('app.celery.letters_pdf_tasks.get_pdf_for_templated_letter.apply_async')
personalisation = {
'addressline1': 'Foo',
'addressline2': 'Bar',
'postcode': 'Flob',
}
notification_json = _notification_json(
template=sample_letter_job.template,
to='Foo',
personalisation=personalisation,
job_id=sample_letter_job.id,
row_number=1
)
notification_id = uuid.uuid4()
save_letter(
sample_letter_job.service_id,
notification_id,
encryption.encrypt(notification_json),
)
assert mock_create_letters_pdf.called
mock_create_letters_pdf.assert_called_once_with(
[str(notification_id)],
queue=QueueNames.CREATE_LETTERS_PDF
)
def test_should_cancel_job_if_service_is_inactive(sample_service,
sample_job,
mocker):
@@ -1402,49 +973,6 @@ def test_get_sms_template_instance(mocker, sample_template, sample_job):
]
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_get_letter_template_instance(mocker, sample_job):
mocker.patch(
'app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=('', {}),
)
sample_contact_block = create_letter_contact(
service=sample_job.service,
contact_block='((reference number))'
)
sample_template = create_template(
service=sample_job.service,
template_type=LETTER_TYPE,
reply_to=sample_contact_block.id,
)
sample_job.template_id = sample_template.id
(
recipient_csv,
template,
_sender_id,
) = get_recipient_csv_and_template_and_sender_id(sample_job)
assert isinstance(template, LetterPrintTemplate)
assert template.contact_block == (
'((reference number))'
)
assert template.placeholders == {
'reference number'
}
assert recipient_csv.placeholders == [
'reference number',
'address line 1',
'address line 2',
'address line 3',
'address line 4',
'address line 5',
'address line 6',
'postcode',
'address line 7',
]
def test_send_inbound_sms_to_service_post_https_request_to_service(notify_api, sample_service):
inbound_api = create_service_inbound_api(service=sample_service, url="https://some.service.gov.uk/",
bearer_token="something_unique")
@@ -1719,28 +1247,6 @@ def test_process_incomplete_job_email(mocker, sample_email_template):
assert mock_email_saver.call_count == 8 # There are 10 in the file and we've added two already
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_process_incomplete_job_letter(mocker, sample_letter_template):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(load_example_csv('multiple_letter'), {'sender_id': None}))
mock_letter_saver = mocker.patch('app.celery.tasks.save_letter.apply_async')
job = create_job(template=sample_letter_template, notification_count=10,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_ERROR)
create_notification(sample_letter_template, job, 0)
create_notification(sample_letter_template, job, 1)
assert Notification.query.filter(Notification.job_id == job.id).count() == 2
process_incomplete_job(str(job.id))
assert mock_letter_saver.call_count == 8
@freeze_time('2017-01-01')
def test_process_incomplete_jobs_sets_status_to_in_progress_and_resets_processing_started_time(mocker, sample_template):
mock_process_incomplete_job = mocker.patch('app.celery.tasks.process_incomplete_job')
@@ -1767,47 +1273,6 @@ def test_process_incomplete_jobs_sets_status_to_in_progress_and_resets_processin
assert mock_process_incomplete_job.mock_calls == [call(str(job1.id)), call(str(job2.id))]
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_process_returned_letters_list(sample_letter_template):
create_notification(sample_letter_template, reference='ref1')
create_notification(sample_letter_template, reference='ref2')
process_returned_letters_list(['ref1', 'ref2', 'unknown-ref'])
notifications = Notification.query.all()
assert [n.status for n in notifications] == ['returned-letter', 'returned-letter']
assert all(n.updated_at for n in notifications)
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_process_returned_letters_list_updates_history_if_notification_is_already_purged(
sample_letter_template
):
create_notification_history(sample_letter_template, reference='ref1')
create_notification_history(sample_letter_template, reference='ref2')
process_returned_letters_list(['ref1', 'ref2', 'unknown-ref'])
notifications = NotificationHistory.query.all()
assert [n.status for n in notifications] == ['returned-letter', 'returned-letter']
assert all(n.updated_at for n in notifications)
@pytest.mark.skip(reason="Needs updating for TTS: Remove mail")
def test_process_returned_letters_populates_returned_letters_table(
sample_letter_template
):
create_notification_history(sample_letter_template, reference='ref1')
create_notification_history(sample_letter_template, reference='ref2')
process_returned_letters_list(['ref1', 'ref2', 'unknown-ref'])
returned_letters = ReturnedLetter.query.all()
assert len(returned_letters) == 2
@freeze_time('2020-03-25 14:30')
@pytest.mark.parametrize('notification_type', ['sms', 'email'])
def test_save_api_email_or_sms(mocker, sample_service, notification_type):