darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-08-25 00:33:41 -04:00
Code Issues Packages Projects Releases Wiki Activity

reformat

Browse Source
This commit is contained in:
Kenneth Kehl
2023-08-29 14:54:30 -07:00
parent 19dcd7a48b
commit 1ecb747c6d
588 changed files with 34100 additions and 23589 deletions
Download Patch File Download Diff File Expand all files Collapse all files

334
tests/app/celery/test_nightly_tasks.py
View File

@@ -27,29 +27,39 @@ from tests.app.db import (
)
def mock_s3_get_list_match(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.2018-01-11175007.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/dispatch/NOTIFY.2018-01-11175007.ACK.txt', 'root/dispatch/NOTIFY.2018-01-11175008.ACK.txt']
def mock_s3_get_list_match(bucket_name, subfolder="", suffix="", last_modified=None):
if subfolder == "2018-01-11/zips_sent":
return ["NOTIFY.2018-01-11175007.ZIP.TXT", "NOTIFY.2018-01-11175008.ZIP.TXT"]
if subfolder == "root/dispatch":
return [
"root/dispatch/NOTIFY.2018-01-11175007.ACK.txt",
"root/dispatch/NOTIFY.2018-01-11175008.ACK.txt",
]
def mock_s3_get_list_diff(bucket_name, subfolder='', suffix='', last_modified=None):
if subfolder == '2018-01-11/zips_sent':
return ['NOTIFY.2018-01-11175007p.ZIP.TXT', 'NOTIFY.2018-01-11175008.ZIP.TXT',
'NOTIFY.2018-01-11175009.ZIP.TXT', 'NOTIFY.2018-01-11175010.ZIP.TXT']
if subfolder == 'root/dispatch':
return ['root/disoatch/NOTIFY.2018-01-11175007p.ACK.TXT', 'root/disoatch/NOTIFY.2018-01-11175008.ACK.TXT']
def mock_s3_get_list_diff(bucket_name, subfolder="", suffix="", last_modified=None):
if subfolder == "2018-01-11/zips_sent":
return [
"NOTIFY.2018-01-11175007p.ZIP.TXT",
"NOTIFY.2018-01-11175008.ZIP.TXT",
"NOTIFY.2018-01-11175009.ZIP.TXT",
"NOTIFY.2018-01-11175010.ZIP.TXT",
]
if subfolder == "root/dispatch":
return [
"root/disoatch/NOTIFY.2018-01-11175007p.ACK.TXT",
"root/disoatch/NOTIFY.2018-01-11175008.ACK.TXT",
]
@freeze_time('2016-10-18T10:00:00')
@freeze_time("2016-10-18T10:00:00")
def test_will_remove_csv_files_for_jobs_older_than_seven_days(
notify_db_session, mocker, sample_template
notify_db_session, mocker, sample_template
):
"""
Jobs older than seven days are deleted, but only two day's worth (two-day window)
"""
mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3')
mocker.patch("app.celery.nightly_tasks.s3.remove_job_from_s3")
seven_days_ago = datetime.utcnow() - timedelta(days=7)
just_under_seven_days = seven_days_ago + timedelta(seconds=1)
@@ -74,23 +84,27 @@ def test_will_remove_csv_files_for_jobs_older_than_seven_days(
assert dont_delete_me_1.archived is False
@freeze_time('2016-10-18T10:00:00')
@freeze_time("2016-10-18T10:00:00")
def test_will_remove_csv_files_for_jobs_older_than_retention_period(
notify_db_session, mocker
):
"""
Jobs older than retention period are deleted, but only two day's worth (two-day window)
"""
mocker.patch('app.celery.nightly_tasks.s3.remove_job_from_s3')
service_1 = create_service(service_name='service 1')
service_2 = create_service(service_name='service 2')
create_service_data_retention(service=service_1, notification_type=SMS_TYPE, days_of_retention=3)
create_service_data_retention(service=service_2, notification_type=EMAIL_TYPE, days_of_retention=30)
mocker.patch("app.celery.nightly_tasks.s3.remove_job_from_s3")
service_1 = create_service(service_name="service 1")
service_2 = create_service(service_name="service 2")
create_service_data_retention(
service=service_1, notification_type=SMS_TYPE, days_of_retention=3
)
create_service_data_retention(
service=service_2, notification_type=EMAIL_TYPE, days_of_retention=30
)
sms_template_service_1 = create_template(service=service_1)
email_template_service_1 = create_template(service=service_1, template_type='email')
email_template_service_1 = create_template(service=service_1, template_type="email")
sms_template_service_2 = create_template(service=service_2)
email_template_service_2 = create_template(service=service_2, template_type='email')
email_template_service_2 = create_template(service=service_2, template_type="email")
four_days_ago = datetime.utcnow() - timedelta(days=4)
eight_days_ago = datetime.utcnow() - timedelta(days=8)
@@ -101,88 +115,107 @@ def test_will_remove_csv_files_for_jobs_older_than_retention_period(
create_job(email_template_service_1, created_at=four_days_ago)
create_job(email_template_service_2, created_at=eight_days_ago)
job3_to_delete = create_job(email_template_service_2, created_at=thirty_one_days_ago)
job3_to_delete = create_job(
email_template_service_2, created_at=thirty_one_days_ago
)
job4_to_delete = create_job(sms_template_service_2, created_at=eight_days_ago)
remove_sms_email_csv_files()
s3.remove_job_from_s3.assert_has_calls([
call(job1_to_delete.service_id, job1_to_delete.id),
call(job2_to_delete.service_id, job2_to_delete.id),
call(job3_to_delete.service_id, job3_to_delete.id),
call(job4_to_delete.service_id, job4_to_delete.id)
], any_order=True)
s3.remove_job_from_s3.assert_has_calls(
[
call(job1_to_delete.service_id, job1_to_delete.id),
call(job2_to_delete.service_id, job2_to_delete.id),
call(job3_to_delete.service_id, job3_to_delete.id),
call(job4_to_delete.service_id, job4_to_delete.id),
],
any_order=True,
)
def test_delete_sms_notifications_older_than_retention_calls_child_task(notify_api, mocker):
mocked = mocker.patch('app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type')
def test_delete_sms_notifications_older_than_retention_calls_child_task(
notify_api, mocker
):
mocked = mocker.patch(
"app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type"
)
delete_sms_notifications_older_than_retention()
mocked.assert_called_once_with('sms')
mocked.assert_called_once_with("sms")
def test_delete_email_notifications_older_than_retentions_calls_child_task(notify_api, mocker):
def test_delete_email_notifications_older_than_retentions_calls_child_task(
notify_api, mocker
):
mocked_notifications = mocker.patch(
'app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type')
"app.celery.nightly_tasks._delete_notifications_older_than_retention_by_type"
)
delete_email_notifications_older_than_retention()
mocked_notifications.assert_called_once_with('email')
mocked_notifications.assert_called_once_with("email")
@freeze_time("2021-12-13T10:00")
def test_timeout_notifications(mocker, sample_notification):
mock_update = mocker.patch('app.celery.nightly_tasks.check_and_queue_callback_task')
mock_dao = mocker.patch('app.celery.nightly_tasks.dao_timeout_notifications')
mock_update = mocker.patch("app.celery.nightly_tasks.check_and_queue_callback_task")
mock_dao = mocker.patch("app.celery.nightly_tasks.dao_timeout_notifications")
mock_dao.side_effect = [
[sample_notification], # first batch to time out
[sample_notification], # second batch
[] # nothing left to time out
[], # nothing left to time out
]
timeout_notifications()
mock_dao.assert_called_with(datetime.fromisoformat('2021-12-10T10:00'))
assert mock_update.mock_calls == [call(sample_notification), call(sample_notification)]
mock_dao.assert_called_with(datetime.fromisoformat("2021-12-10T10:00"))
assert mock_update.mock_calls == [
call(sample_notification),
call(sample_notification),
]
def test_delete_inbound_sms_calls_child_task(notify_api, mocker):
mocker.patch('app.celery.nightly_tasks.delete_inbound_sms_older_than_retention')
mocker.patch("app.celery.nightly_tasks.delete_inbound_sms_older_than_retention")
delete_inbound_sms()
assert nightly_tasks.delete_inbound_sms_older_than_retention.call_count == 1
def test_delete_inbound_sms_calls_child_task_db_error(notify_api, mocker):
mock_delete = mocker.patch('app.celery.nightly_tasks.delete_inbound_sms_older_than_retention')
mock_delete = mocker.patch(
"app.celery.nightly_tasks.delete_inbound_sms_older_than_retention"
)
mock_delete.side_effect = SQLAlchemyError
with pytest.raises(expected_exception=SQLAlchemyError):
delete_inbound_sms()
@freeze_time('2021-01-18T02:00')
@pytest.mark.parametrize('date_provided', [None, '2021-1-17'])
def test_save_daily_notification_processing_time(mocker, sample_template, date_provided):
@freeze_time("2021-01-18T02:00")
@pytest.mark.parametrize("date_provided", [None, "2021-1-17"])
def test_save_daily_notification_processing_time(
mocker, sample_template, date_provided
):
# notification created too early to be counted
create_notification(
sample_template,
created_at=datetime(2021, 1, 16, 23, 59),
sent_at=datetime(2021, 1, 16, 23, 59) + timedelta(seconds=5)
sent_at=datetime(2021, 1, 16, 23, 59) + timedelta(seconds=5),
)
# notification counted and sent within 10 seconds
create_notification(
sample_template,
created_at=datetime(2021, 1, 17, 00, 00),
sent_at=datetime(2021, 1, 17, 00, 00) + timedelta(seconds=5)
sent_at=datetime(2021, 1, 17, 00, 00) + timedelta(seconds=5),
)
# notification counted but not sent within 10 seconds
create_notification(
sample_template,
created_at=datetime(2021, 1, 17, 23, 59),
sent_at=datetime(2021, 1, 17, 23, 59) + timedelta(seconds=15)
sent_at=datetime(2021, 1, 17, 23, 59) + timedelta(seconds=15),
)
# notification created too late to be counted
create_notification(
sample_template,
created_at=datetime(2021, 1, 18, 00, 00),
sent_at=datetime(2021, 1, 18, 00, 00) + timedelta(seconds=5)
sent_at=datetime(2021, 1, 18, 00, 00) + timedelta(seconds=5),
)
save_daily_notification_processing_time(date_provided)
@@ -194,32 +227,34 @@ def test_save_daily_notification_processing_time(mocker, sample_template, date_p
assert persisted_to_db[0].messages_within_10_secs == 1
@freeze_time('2021-04-18T02:00')
@pytest.mark.parametrize('date_provided', [None, '2021-4-17'])
def test_save_daily_notification_processing_time_when_in_est(mocker, sample_template, date_provided):
@freeze_time("2021-04-18T02:00")
@pytest.mark.parametrize("date_provided", [None, "2021-4-17"])
def test_save_daily_notification_processing_time_when_in_est(
mocker, sample_template, date_provided
):
# notification created too early to be counted
create_notification(
sample_template,
created_at=datetime(2021, 4, 16, 22, 59),
sent_at=datetime(2021, 4, 16, 22, 59) + timedelta(seconds=15)
sent_at=datetime(2021, 4, 16, 22, 59) + timedelta(seconds=15),
)
# notification counted and sent within 10 seconds
create_notification(
sample_template,
created_at=datetime(2021, 4, 17, 4, 00),
sent_at=datetime(2021, 4, 17, 4, 00) + timedelta(seconds=5)
sent_at=datetime(2021, 4, 17, 4, 00) + timedelta(seconds=5),
)
# notification counted and sent within 10 seconds
create_notification(
sample_template,
created_at=datetime(2021, 4, 17, 22, 59),
sent_at=datetime(2021, 4, 17, 22, 59) + timedelta(seconds=5)
sent_at=datetime(2021, 4, 17, 22, 59) + timedelta(seconds=5),
)
# notification created too late to be counted
create_notification(
sample_template,
created_at=datetime(2021, 4, 18, 23, 00),
sent_at=datetime(2021, 4, 18, 23, 00) + timedelta(seconds=15)
sent_at=datetime(2021, 4, 18, 23, 00) + timedelta(seconds=15),
)
save_daily_notification_processing_time(date_provided)
@@ -231,109 +266,150 @@ def test_save_daily_notification_processing_time_when_in_est(mocker, sample_temp
assert persisted_to_db[0].messages_within_10_secs == 2
@freeze_time('2021-06-05 08:00')
def test_delete_notifications_task_calls_task_for_services_with_data_retention_of_same_type(notify_db_session, mocker):
sms_service = create_service(service_name='a')
email_service = create_service(service_name='b')
letter_service = create_service(service_name='c')
create_service_data_retention(sms_service, notification_type='sms')
create_service_data_retention(email_service, notification_type='email')
create_service_data_retention(letter_service, notification_type='letter')
mock_subtask = mocker.patch('app.celery.nightly_tasks.delete_notifications_for_service_and_type')
_delete_notifications_older_than_retention_by_type('sms')
mock_subtask.apply_async.assert_called_once_with(queue='reporting-tasks', kwargs={
'service_id': sms_service.id,
'notification_type': 'sms',
# three days of retention, its morn of 5th, so we want to keep all messages from 4th, 3rd and 2nd.
'datetime_to_delete_before': date(2021, 6, 2),
})
@freeze_time('2021-04-04 23:00')
def test_delete_notifications_task_calls_task_for_services_with_data_retention_by_looking_at_retention(
notify_db_session,
mocker
@freeze_time("2021-06-05 08:00")
def test_delete_notifications_task_calls_task_for_services_with_data_retention_of_same_type(
notify_db_session, mocker
):
service_14_days = create_service(service_name='a')
service_3_days = create_service(service_name='b')
sms_service = create_service(service_name="a")
email_service = create_service(service_name="b")
letter_service = create_service(service_name="c")
create_service_data_retention(sms_service, notification_type="sms")
create_service_data_retention(email_service, notification_type="email")
create_service_data_retention(letter_service, notification_type="letter")
mock_subtask = mocker.patch(
"app.celery.nightly_tasks.delete_notifications_for_service_and_type"
)
_delete_notifications_older_than_retention_by_type("sms")
mock_subtask.apply_async.assert_called_once_with(
queue="reporting-tasks",
kwargs={
"service_id": sms_service.id,
"notification_type": "sms",
# three days of retention, its morn of 5th, so we want to keep all messages from 4th, 3rd and 2nd.
"datetime_to_delete_before": date(2021, 6, 2),
},
)
@freeze_time("2021-04-04 23:00")
def test_delete_notifications_task_calls_task_for_services_with_data_retention_by_looking_at_retention(
notify_db_session, mocker
):
service_14_days = create_service(service_name="a")
service_3_days = create_service(service_name="b")
create_service_data_retention(service_14_days, days_of_retention=14)
create_service_data_retention(service_3_days, days_of_retention=3)
mock_subtask = mocker.patch('app.celery.nightly_tasks.delete_notifications_for_service_and_type')
mock_subtask = mocker.patch(
"app.celery.nightly_tasks.delete_notifications_for_service_and_type"
)
_delete_notifications_older_than_retention_by_type('sms')
_delete_notifications_older_than_retention_by_type("sms")
assert mock_subtask.apply_async.call_count == 2
mock_subtask.apply_async.assert_has_calls(any_order=True, calls=[
call(queue=ANY, kwargs={
'service_id': service_14_days.id,
'notification_type': 'sms',
'datetime_to_delete_before': date(2021, 3, 21)
}),
call(queue=ANY, kwargs={
'service_id': service_3_days.id,
'notification_type': 'sms',
'datetime_to_delete_before': date(2021, 4, 1)
}),
])
mock_subtask.apply_async.assert_has_calls(
any_order=True,
calls=[
call(
queue=ANY,
kwargs={
"service_id": service_14_days.id,
"notification_type": "sms",
"datetime_to_delete_before": date(2021, 3, 21),
},
),
call(
queue=ANY,
kwargs={
"service_id": service_3_days.id,
"notification_type": "sms",
"datetime_to_delete_before": date(2021, 4, 1),
},
),
],
)
@freeze_time('2021-04-02 23:00')
@freeze_time("2021-04-02 23:00")
def test_delete_notifications_task_calls_task_for_services_that_have_sent_notifications_recently(
notify_db_session,
mocker
notify_db_session, mocker
):
service_will_delete_1 = create_service(service_name='a')
service_will_delete_2 = create_service(service_name='b')
service_nothing_to_delete = create_service(service_name='c')
service_will_delete_1 = create_service(service_name="a")
service_will_delete_2 = create_service(service_name="b")
service_nothing_to_delete = create_service(service_name="c")
create_template(service_will_delete_1)
create_template(service_will_delete_2)
nothing_to_delete_sms_template = create_template(service_nothing_to_delete, template_type='sms')
nothing_to_delete_email_template = create_template(service_nothing_to_delete, template_type='email')
nothing_to_delete_sms_template = create_template(
service_nothing_to_delete, template_type="sms"
)
nothing_to_delete_email_template = create_template(
service_nothing_to_delete, template_type="email"
)
# will be deleted as service has no custom retention, but past our default 7 days
create_notification(service_will_delete_1.templates[0], created_at=datetime.utcnow() - timedelta(days=8))
create_notification(service_will_delete_2.templates[0], created_at=datetime.utcnow() - timedelta(days=8))
create_notification(
service_will_delete_1.templates[0],
created_at=datetime.utcnow() - timedelta(days=8),
)
create_notification(
service_will_delete_2.templates[0],
created_at=datetime.utcnow() - timedelta(days=8),
)
# will be kept as it's recent, and we won't run delete_notifications_for_service_and_type
create_notification(nothing_to_delete_sms_template, created_at=datetime.utcnow() - timedelta(days=2))
create_notification(
nothing_to_delete_sms_template, created_at=datetime.utcnow() - timedelta(days=2)
)
# this is an old notification, but for email not sms, so we won't run delete_notifications_for_service_and_type
create_notification(nothing_to_delete_email_template, created_at=datetime.utcnow() - timedelta(days=8))
create_notification(
nothing_to_delete_email_template,
created_at=datetime.utcnow() - timedelta(days=8),
)
mock_subtask = mocker.patch('app.celery.nightly_tasks.delete_notifications_for_service_and_type')
mock_subtask = mocker.patch(
"app.celery.nightly_tasks.delete_notifications_for_service_and_type"
)
_delete_notifications_older_than_retention_by_type('sms')
_delete_notifications_older_than_retention_by_type("sms")
assert mock_subtask.apply_async.call_count == 2
mock_subtask.apply_async.assert_has_calls(any_order=True, calls=[
call(queue=ANY, kwargs={
'service_id': service_will_delete_1.id,
'notification_type': 'sms',
'datetime_to_delete_before': date(2021, 3, 26)
}),
call(queue=ANY, kwargs={
'service_id': service_will_delete_2.id,
'notification_type': 'sms',
'datetime_to_delete_before': date(2021, 3, 26)
}),
])
mock_subtask.apply_async.assert_has_calls(
any_order=True,
calls=[
call(
queue=ANY,
kwargs={
"service_id": service_will_delete_1.id,
"notification_type": "sms",
"datetime_to_delete_before": date(2021, 3, 26),
},
),
call(
queue=ANY,
kwargs={
"service_id": service_will_delete_2.id,
"notification_type": "sms",
"datetime_to_delete_before": date(2021, 3, 26),
},
),
],
)
def test_cleanup_unfinished_jobs(mocker):
mock_s3 = mocker.patch('app.celery.nightly_tasks.remove_csv_object')
mock_dao_archive = mocker.patch('app.celery.nightly_tasks.dao_archive_job')
mock_dao = mocker.patch('app.celery.nightly_tasks.dao_get_unfinished_jobs')
mock_s3 = mocker.patch("app.celery.nightly_tasks.remove_csv_object")
mock_dao_archive = mocker.patch("app.celery.nightly_tasks.dao_archive_job")
mock_dao = mocker.patch("app.celery.nightly_tasks.dao_get_unfinished_jobs")
mock_job_unfinished = Job()
mock_job_unfinished.processing_started = datetime(2023, 1, 1, 0, 0, 0)
mock_job_unfinished.original_file_name = "blah"
mock_dao.return_value = [mock_job_unfinished]
cleanup_unfinished_jobs()
mock_s3.assert_called_once_with('blah')
mock_s3.assert_called_once_with("blah")
mock_dao_archive.assert_called_once_with(mock_job_unfinished)

384
tests/app/celery/test_process_ses_receipts_tasks.py
View File

@@ -28,9 +28,9 @@ from tests.app.db import (
def test_notifications_ses_400_with_invalid_header(client):
data = json.dumps({"foo": "bar"})
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=data,
headers=[('Content-Type', 'application/json')]
headers=[("Content-Type", "application/json")],
)
assert response.status_code == 400
@@ -38,44 +38,68 @@ def test_notifications_ses_400_with_invalid_header(client):
def test_notifications_ses_400_with_invalid_message_type(client):
data = json.dumps({"foo": "bar"})
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=data,
headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'foo')]
headers=[
("Content-Type", "application/json"),
("x-amz-sns-message-type", "foo"),
],
)
assert response.status_code == 400
assert "SES-SNS callback failed: invalid message type" in response.get_data(as_text=True)
assert "SES-SNS callback failed: invalid message type" in response.get_data(
as_text=True
)
def test_notifications_ses_400_with_invalid_json(client):
data = "FOOO"
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=data,
headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
headers=[
("Content-Type", "application/json"),
("x-amz-sns-message-type", "Notification"),
],
)
assert response.status_code == 400
assert "SES-SNS callback failed: invalid JSON given" in response.get_data(as_text=True)
assert "SES-SNS callback failed: invalid JSON given" in response.get_data(
as_text=True
)
def test_notifications_ses_400_with_certificate(client):
data = json.dumps({"foo": "bar"})
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=data,
headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
headers=[
("Content-Type", "application/json"),
("x-amz-sns-message-type", "Notification"),
],
)
assert response.status_code == 400
assert "SES-SNS callback failed: validation failed" in response.get_data(as_text=True)
assert "SES-SNS callback failed: validation failed" in response.get_data(
as_text=True
)
def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True)
requests_mock = mocker.patch("requests.get")
data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo", "Message": "foo"})
data = json.dumps(
{
"Type": "SubscriptionConfirmation",
"SubscribeURL": "https://foo",
"Message": "foo",
}
)
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=data,
headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'SubscriptionConfirmation')]
headers=[
("Content-Type", "application/json"),
("x-amz-sns-message-type", "SubscriptionConfirmation"),
],
)
requests_mock.assert_called_once_with("https://foo")
@@ -83,38 +107,64 @@ def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
def test_notifications_ses_200_call_process_task(client, mocker):
process_mock = mocker.patch("app.notifications.notifications_ses_callback.process_ses_results.apply_async")
process_mock = mocker.patch(
"app.notifications.notifications_ses_callback.process_ses_results.apply_async"
)
mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True)
data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"}}
mocker.patch("app.notifications.sns_handlers.sns_notification_handler", return_value=data)
mocker.patch(
"app.notifications.sns_handlers.sns_notification_handler", return_value=data
)
json_data = json.dumps(data)
response = client.post(
path='/notifications/email/ses',
path="/notifications/email/ses",
data=json_data,
headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
headers=[
("Content-Type", "application/json"),
("x-amz-sns-message-type", "Notification"),
],
)
process_mock.assert_called_once_with([{'Message': {"mail": "baz"}}], queue='notify-internal-tasks')
process_mock.assert_called_once_with(
[{"Message": {"mail": "baz"}}], queue="notify-internal-tasks"
)
assert response.status_code == 200
def test_process_ses_results(sample_email_template):
create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
create_notification(
sample_email_template,
reference="ref1",
sent_at=datetime.utcnow(),
status="sending",
)
assert process_ses_results(response=ses_notification_callback(reference='ref1'))
assert process_ses_results(response=ses_notification_callback(reference="ref1"))
def test_process_ses_results_retry_called(sample_email_template, mocker):
create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED"))
mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
process_ses_results(response=ses_notification_callback(reference='ref1'))
create_notification(
sample_email_template,
reference="ref1",
sent_at=datetime.utcnow(),
status="sending",
)
mocker.patch(
"app.dao.notifications_dao._update_notification_status",
side_effect=Exception("EXPECTED"),
)
mocked = mocker.patch(
"app.celery.process_ses_receipts_tasks.process_ses_results.retry"
)
process_ses_results(response=ses_notification_callback(reference="ref1"))
assert mocked.call_count != 0
def test_process_ses_results_in_complaint(sample_email_template, mocker):
notification = create_notification(template=sample_email_template, reference='ref1')
mocked = mocker.patch("app.dao.notifications_dao.update_notification_status_by_reference")
notification = create_notification(template=sample_email_template, reference="ref1")
mocked = mocker.patch(
"app.dao.notifications_dao.update_notification_status_by_reference"
)
process_ses_results(response=ses_complaint_callback())
assert mocked.call_count == 0
complaints = Complaint.query.all()
@@ -123,212 +173,236 @@ def test_process_ses_results_in_complaint(sample_email_template, mocker):
def test_remove_emails_from_complaint():
test_json = json.loads(ses_complaint_callback()['Message'])
test_json = json.loads(ses_complaint_callback()["Message"])
remove_emails_from_complaint(test_json)
assert "recipient1@example.com" not in json.dumps(test_json)
def test_remove_email_from_bounce():
test_json = json.loads(ses_hard_bounce_callback(reference='ref1')['Message'])
test_json = json.loads(ses_hard_bounce_callback(reference="ref1")["Message"])
remove_emails_from_bounce(test_json)
assert "bounce@simulator.amazonses.com" not in json.dumps(test_json)
def test_ses_callback_should_update_notification_status(
client,
_notify_db,
notify_db_session,
sample_email_template,
mocker):
with freeze_time('2001-01-01T12:00:00'):
client, _notify_db, notify_db_session, sample_email_template, mocker
):
with freeze_time("2001-01-01T12:00:00"):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
create_service_callback_api(
service=sample_email_template.service,
url="https://original_url.com"
service=sample_email_template.service, url="https://original_url.com"
)
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(ses_notification_callback(reference="ref"))
assert get_notification_by_id(notification.id).status == "delivered"
send_mock.assert_called_once_with(
[str(notification.id), ANY], queue="service-callbacks"
)
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
send_mock.assert_called_once_with([str(notification.id), ANY], queue="service-callbacks")
# assert second arg is an encrypted string
assert isinstance(send_mock.call_args.args[0][1], str)
def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker):
mock_dup = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._duplicate_update_warning')
mock_upd = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._update_notification_status')
notification = create_notification(template=sample_email_template, reference='ref', status='delivered')
assert process_ses_results(ses_notification_callback(reference='ref')) is None
assert get_notification_by_id(notification.id).status == 'delivered'
mock_dup.assert_called_once_with(notification, 'delivered')
def test_ses_callback_should_not_update_notification_status_if_already_delivered(
sample_email_template, mocker
):
mock_dup = mocker.patch(
"app.celery.process_ses_receipts_tasks.notifications_dao._duplicate_update_warning"
)
mock_upd = mocker.patch(
"app.celery.process_ses_receipts_tasks.notifications_dao._update_notification_status"
)
notification = create_notification(
template=sample_email_template, reference="ref", status="delivered"
)
assert process_ses_results(ses_notification_callback(reference="ref")) is None
assert get_notification_by_id(notification.id).status == "delivered"
mock_dup.assert_called_once_with(notification, "delivered")
assert mock_upd.call_count == 0
def test_ses_callback_should_retry_if_notification_is_new(mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
with freeze_time('2017-11-17T12:14:03.646Z'):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
mock_retry = mocker.patch(
"app.celery.process_ses_receipts_tasks.process_ses_results.retry"
)
mock_logger = mocker.patch(
"app.celery.process_ses_receipts_tasks.current_app.logger.error"
)
with freeze_time("2017-11-17T12:14:03.646Z"):
assert process_ses_results(ses_notification_callback(reference="ref")) is None
assert mock_logger.call_count == 0
assert mock_retry.call_count == 1
def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning')
with freeze_time('2017-11-17T12:34:03.646Z'):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
mock_retry = mocker.patch(
"app.celery.process_ses_receipts_tasks.process_ses_results.retry"
)
mock_logger = mocker.patch(
"app.celery.process_ses_receipts_tasks.current_app.logger.warning"
)
with freeze_time("2017-11-17T12:34:03.646Z"):
assert process_ses_results(ses_notification_callback(reference="ref")) is None
assert mock_retry.call_count == 0
mock_logger.assert_called_once_with(
'Notification not found for reference: ref (while attempting update to delivered)'
"Notification not found for reference: ref (while attempting update to delivered)"
)
def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
with freeze_time('2017-11-21T12:14:03.646Z'):
assert process_ses_results(ses_notification_callback(reference='ref')) is None
mock_retry = mocker.patch(
"app.celery.process_ses_receipts_tasks.process_ses_results.retry"
)
mock_logger = mocker.patch(
"app.celery.process_ses_receipts_tasks.current_app.logger.error"
)
with freeze_time("2017-11-21T12:14:03.646Z"):
assert process_ses_results(ses_notification_callback(reference="ref")) is None
assert mock_logger.call_count == 0
assert mock_retry.call_count == 0
def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
client,
_notify_db,
notify_db_session,
sample_email_template,
mocker):
with freeze_time('2001-01-01T12:00:00'):
client, _notify_db, notify_db_session, sample_email_template, mocker
):
with freeze_time("2001-01-01T12:00:00"):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_notification_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'delivered'
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(ses_notification_callback(reference="ref"))
assert get_notification_by_id(notification.id).status == "delivered"
send_mock.assert_not_called()
def test_ses_callback_should_update_multiple_notification_status_sent(
client,
_notify_db,
notify_db_session,
sample_email_template,
mocker):
client, _notify_db, notify_db_session, sample_email_template, mocker
):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref1',
reference="ref1",
sent_at=datetime.utcnow(),
status='sending')
status="sending",
)
create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref2',
reference="ref2",
sent_at=datetime.utcnow(),
status='sending')
status="sending",
)
create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref3',
reference="ref3",
sent_at=datetime.utcnow(),
status='sending')
create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
assert process_ses_results(ses_notification_callback(reference='ref1'))
assert process_ses_results(ses_notification_callback(reference='ref2'))
assert process_ses_results(ses_notification_callback(reference='ref3'))
assert send_mock.called
def test_ses_callback_should_set_status_to_temporary_failure(client,
_notify_db,
notify_db_session,
sample_email_template,
mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
)
create_service_callback_api(service=notification.service, url="https://original_url.com")
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_soft_bounce_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'temporary-failure'
assert send_mock.called
def test_ses_callback_should_set_status_to_permanent_failure(client,
_notify_db,
notify_db_session,
sample_email_template,
mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference='ref',
status='sending',
sent_at=datetime.utcnow()
)
create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
assert get_notification_by_id(notification.id).status == 'sending'
assert process_ses_results(ses_hard_bounce_callback(reference='ref'))
assert get_notification_by_id(notification.id).status == 'permanent-failure'
assert send_mock.called
def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker):
send_mock = mocker.patch(
'app.celery.service_callback_tasks.send_complaint_to_service.apply_async'
status="sending",
)
create_service_callback_api(
service=sample_email_template.service, url="https://original_url.com", callback_type="complaint"
service=sample_email_template.service, url="https://original_url.com"
)
assert process_ses_results(ses_notification_callback(reference="ref1"))
assert process_ses_results(ses_notification_callback(reference="ref2"))
assert process_ses_results(ses_notification_callback(reference="ref3"))
assert send_mock.called
def test_ses_callback_should_set_status_to_temporary_failure(
client, _notify_db, notify_db_session, sample_email_template, mocker
):
send_mock = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
create_service_callback_api(
service=notification.service, url="https://original_url.com"
)
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(ses_soft_bounce_callback(reference="ref"))
assert get_notification_by_id(notification.id).status == "temporary-failure"
assert send_mock.called
def test_ses_callback_should_set_status_to_permanent_failure(
client, _notify_db, notify_db_session, sample_email_template, mocker
):
send_mock = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async"
)
notification = create_sample_notification(
_notify_db,
notify_db_session,
template=sample_email_template,
reference="ref",
status="sending",
sent_at=datetime.utcnow(),
)
create_service_callback_api(
service=sample_email_template.service, url="https://original_url.com"
)
assert get_notification_by_id(notification.id).status == "sending"
assert process_ses_results(ses_hard_bounce_callback(reference="ref"))
assert get_notification_by_id(notification.id).status == "permanent-failure"
assert send_mock.called
def test_ses_callback_should_send_on_complaint_to_user_callback_api(
sample_email_template, mocker
):
send_mock = mocker.patch(
"app.celery.service_callback_tasks.send_complaint_to_service.apply_async"
)
create_service_callback_api(
service=sample_email_template.service,
url="https://original_url.com",
callback_type="complaint",
)
notification = create_notification(
template=sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending'
template=sample_email_template,
reference="ref1",
sent_at=datetime.utcnow(),
status="sending",
)
response = ses_complaint_callback()
assert process_ses_results(response)
assert send_mock.call_count == 1
assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
'complaint_date': '2018-06-05T13:59:58.000000Z',
'complaint_id': str(Complaint.query.one().id),
'notification_id': str(notification.id),
'reference': None,
'service_callback_api_bearer_token': 'some_super_secret',
'service_callback_api_url': 'https://original_url.com',
'to': 'recipient1@example.com'
"complaint_date": "2018-06-05T13:59:58.000000Z",
"complaint_id": str(Complaint.query.one().id),
"notification_id": str(notification.id),
"reference": None,
"service_callback_api_bearer_token": "some_super_secret",
"service_callback_api_url": "https://original_url.com",
"to": "recipient1@example.com",
}

183
tests/app/celery/test_provider_tasks.py
View File

@@ -15,41 +15,47 @@ from app.exceptions import NotificationTechnicalFailureException
def test_should_have_decorated_tasks_functions():
assert deliver_sms.__wrapped__.__name__ == 'deliver_sms'
assert deliver_email.__wrapped__.__name__ == 'deliver_email'
assert deliver_sms.__wrapped__.__name__ == "deliver_sms"
assert deliver_email.__wrapped__.__name__ == "deliver_email"
def test_should_call_send_sms_to_provider_from_deliver_sms_task(
sample_notification,
mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
mocker.patch('app.celery.provider_tasks.check_sms_delivery_receipt')
sample_notification, mocker
):
mocker.patch("app.delivery.send_to_providers.send_sms_to_provider")
mocker.patch("app.celery.provider_tasks.check_sms_delivery_receipt")
deliver_sms(sample_notification.id)
app.delivery.send_to_providers.send_sms_to_provider.assert_called_with(sample_notification)
app.delivery.send_to_providers.send_sms_to_provider.assert_called_with(
sample_notification
)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_sms_task(
notify_db_session,
mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider')
mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
notify_db_session, mocker
):
mocker.patch("app.delivery.send_to_providers.send_sms_to_provider")
mocker.patch("app.celery.provider_tasks.deliver_sms.retry")
notification_id = app.create_uuid()
deliver_sms(notification_id)
app.delivery.send_to_providers.send_sms_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
app.celery.provider_tasks.deliver_sms.retry.assert_called_with(
queue="retry-tasks", countdown=0
)
def test_should_retry_and_log_warning_if_SmsClientResponseException_for_deliver_sms_task(sample_notification, mocker):
def test_should_retry_and_log_warning_if_SmsClientResponseException_for_deliver_sms_task(
sample_notification, mocker
):
mocker.patch(
'app.delivery.send_to_providers.send_sms_to_provider',
"app.delivery.send_to_providers.send_sms_to_provider",
side_effect=SmsClientResponseException("something went wrong"),
)
mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
mock_logger_warning = mocker.patch('app.celery.tasks.current_app.logger.warning')
assert sample_notification.status == 'created'
mocker.patch("app.celery.provider_tasks.deliver_sms.retry")
mock_logger_warning = mocker.patch("app.celery.tasks.current_app.logger.warning")
assert sample_notification.status == "created"
deliver_sms(sample_notification.id)
@@ -60,29 +66,46 @@ def test_should_retry_and_log_warning_if_SmsClientResponseException_for_deliver_
def test_should_retry_and_log_exception_for_non_SmsClientResponseException_exceptions_for_deliver_sms_task(
sample_notification, mocker
):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("something went wrong"))
mocker.patch('app.celery.provider_tasks.deliver_sms.retry')
mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
mocker.patch(
"app.delivery.send_to_providers.send_sms_to_provider",
side_effect=Exception("something went wrong"),
)
mocker.patch("app.celery.provider_tasks.deliver_sms.retry")
mock_logger_exception = mocker.patch(
"app.celery.tasks.current_app.logger.exception"
)
assert sample_notification.status == 'created'
assert sample_notification.status == "created"
deliver_sms(sample_notification.id)
assert provider_tasks.deliver_sms.retry.called is True
assert mock_logger_exception.called
def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(sample_notification, mocker):
mocker.patch('app.delivery.send_to_providers.send_sms_to_provider', side_effect=Exception("EXPECTED"))
mocker.patch('app.celery.provider_tasks.deliver_sms.retry', side_effect=MaxRetriesExceededError())
mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(
sample_notification, mocker
):
mocker.patch(
"app.delivery.send_to_providers.send_sms_to_provider",
side_effect=Exception("EXPECTED"),
)
mocker.patch(
"app.celery.provider_tasks.deliver_sms.retry",
side_effect=MaxRetriesExceededError(),
)
mock_logger_exception = mocker.patch(
"app.celery.tasks.current_app.logger.exception"
)
with pytest.raises(NotificationTechnicalFailureException) as e:
deliver_sms(sample_notification.id)
assert str(sample_notification.id) in str(e.value)
provider_tasks.deliver_sms.retry.assert_called_with(queue="retry-tasks", countdown=0)
provider_tasks.deliver_sms.retry.assert_called_with(
queue="retry-tasks", countdown=0
)
assert sample_notification.status == 'technical-failure'
assert sample_notification.status == "technical-failure"
assert mock_logger_exception.called
@@ -90,99 +113,125 @@ def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_sms_task(s
def test_should_call_send_email_to_provider_from_deliver_email_task(
sample_notification,
mocker):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')
sample_notification, mocker
):
mocker.patch("app.delivery.send_to_providers.send_email_to_provider")
deliver_email(sample_notification.id)
app.delivery.send_to_providers.send_email_to_provider.assert_called_with(sample_notification)
app.delivery.send_to_providers.send_email_to_provider.assert_called_with(
sample_notification
)
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task(mocker):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider')
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
def test_should_add_to_retry_queue_if_notification_not_found_in_deliver_email_task(
mocker,
):
mocker.patch("app.delivery.send_to_providers.send_email_to_provider")
mocker.patch("app.celery.provider_tasks.deliver_email.retry")
notification_id = app.create_uuid()
deliver_email(notification_id)
app.delivery.send_to_providers.send_email_to_provider.assert_not_called()
app.celery.provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
app.celery.provider_tasks.deliver_email.retry.assert_called_with(
queue="retry-tasks"
)
@pytest.mark.parametrize(
'exception_class', [
"exception_class",
[
Exception(),
AwsSesClientException(),
AwsSesClientThrottlingSendRateException(),
]
],
)
def test_should_go_into_technical_error_if_exceeds_retries_on_deliver_email_task(
sample_notification, mocker, exception_class
):
mocker.patch('app.delivery.send_to_providers.send_email_to_provider', side_effect=exception_class)
mocker.patch('app.celery.provider_tasks.deliver_email.retry', side_effect=MaxRetriesExceededError())
mocker.patch(
"app.delivery.send_to_providers.send_email_to_provider",
side_effect=exception_class,
)
mocker.patch(
"app.celery.provider_tasks.deliver_email.retry",
side_effect=MaxRetriesExceededError(),
)
with pytest.raises(NotificationTechnicalFailureException) as e:
deliver_email(sample_notification.id)
assert str(sample_notification.id) in str(e.value)
provider_tasks.deliver_email.retry.assert_called_with(queue="retry-tasks")
assert sample_notification.status == 'technical-failure'
assert sample_notification.status == "technical-failure"
def test_should_technical_error_and_not_retry_if_EmailClientNonRetryableException(sample_notification, mocker):
def test_should_technical_error_and_not_retry_if_EmailClientNonRetryableException(
sample_notification, mocker
):
mocker.patch(
'app.delivery.send_to_providers.send_email_to_provider',
side_effect=EmailClientNonRetryableException('bad email')
"app.delivery.send_to_providers.send_email_to_provider",
side_effect=EmailClientNonRetryableException("bad email"),
)
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
mocker.patch("app.celery.provider_tasks.deliver_email.retry")
deliver_email(sample_notification.id)
assert provider_tasks.deliver_email.retry.called is False
assert sample_notification.status == 'technical-failure'
assert sample_notification.status == "technical-failure"
def test_should_retry_and_log_exception_for_deliver_email_task(sample_notification, mocker):
def test_should_retry_and_log_exception_for_deliver_email_task(
sample_notification, mocker
):
error_response = {
'Error': {
'Code': 'SomeError',
'Message': 'some error message from amazon',
'Type': 'Sender'
"Error": {
"Code": "SomeError",
"Message": "some error message from amazon",
"Type": "Sender",
}
}
ex = ClientError(error_response=error_response, operation_name='opname')
mocker.patch('app.delivery.send_to_providers.send_email_to_provider', side_effect=AwsSesClientException(str(ex)))
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
ex = ClientError(error_response=error_response, operation_name="opname")
mocker.patch(
"app.delivery.send_to_providers.send_email_to_provider",
side_effect=AwsSesClientException(str(ex)),
)
mocker.patch("app.celery.provider_tasks.deliver_email.retry")
mock_logger_exception = mocker.patch(
"app.celery.tasks.current_app.logger.exception"
)
deliver_email(sample_notification.id)
assert provider_tasks.deliver_email.retry.called is True
assert sample_notification.status == 'created'
assert sample_notification.status == "created"
assert mock_logger_exception.called
def test_if_ses_send_rate_throttle_then_should_retry_and_log_warning(sample_notification, mocker):
def test_if_ses_send_rate_throttle_then_should_retry_and_log_warning(
sample_notification, mocker
):
error_response = {
'Error': {
'Code': 'Throttling',
'Message': 'Maximum sending rate exceeded.',
'Type': 'Sender'
"Error": {
"Code": "Throttling",
"Message": "Maximum sending rate exceeded.",
"Type": "Sender",
}
}
ex = ClientError(error_response=error_response, operation_name='opname')
ex = ClientError(error_response=error_response, operation_name="opname")
mocker.patch(
'app.delivery.send_to_providers.send_email_to_provider',
side_effect=AwsSesClientThrottlingSendRateException(str(ex))
"app.delivery.send_to_providers.send_email_to_provider",
side_effect=AwsSesClientThrottlingSendRateException(str(ex)),
)
mocker.patch("app.celery.provider_tasks.deliver_email.retry")
mock_logger_warning = mocker.patch("app.celery.tasks.current_app.logger.warning")
mock_logger_exception = mocker.patch(
"app.celery.tasks.current_app.logger.exception"
)
mocker.patch('app.celery.provider_tasks.deliver_email.retry')
mock_logger_warning = mocker.patch('app.celery.tasks.current_app.logger.warning')
mock_logger_exception = mocker.patch('app.celery.tasks.current_app.logger.exception')
deliver_email(sample_notification.id)
assert provider_tasks.deliver_email.retry.called is True
assert sample_notification.status == 'created'
assert sample_notification.status == "created"
assert not mock_logger_exception.called
assert mock_logger_warning.called

284
tests/app/celery/test_reporting_tasks.py
View File

@@ -42,21 +42,30 @@ def mocker_get_rate(
return Decimal(0)
@freeze_time('2019-08-01T05:30')
@pytest.mark.parametrize('day_start, expected_kwargs', [
(None, [f'2019-07-{31-i}' for i in range(10)]),
('2019-07-21', [f'2019-07-{21-i}' for i in range(10)]),
])
def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_start, expected_kwargs):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_billing_for_day')
@freeze_time("2019-08-01T05:30")
@pytest.mark.parametrize(
"day_start, expected_kwargs",
[
(None, [f"2019-07-{31-i}" for i in range(10)]),
("2019-07-21", [f"2019-07-{21-i}" for i in range(10)]),
],
)
def test_create_nightly_billing_triggers_tasks_for_days(
notify_api, mocker, day_start, expected_kwargs
):
mock_celery = mocker.patch(
"app.celery.reporting_tasks.create_nightly_billing_for_day"
)
create_nightly_billing(day_start)
assert mock_celery.apply_async.call_count == 10
for i in range(10):
assert mock_celery.apply_async.call_args_list[i][1]['kwargs'] == {'process_day': expected_kwargs[i]}
assert mock_celery.apply_async.call_args_list[i][1]["kwargs"] == {
"process_day": expected_kwargs[i]
}
@freeze_time('2019-08-01T00:30')
@freeze_time("2019-08-01T00:30")
def test_create_nightly_notification_status_triggers_tasks(
notify_api,
sample_service,
@@ -64,29 +73,32 @@ def test_create_nightly_notification_status_triggers_tasks(
mocker,
):
mock_celery = mocker.patch(
'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
"app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day"
).apply_async
create_notification(template=sample_template, created_at='2019-07-31')
create_notification(template=sample_template, created_at="2019-07-31")
create_nightly_notification_status()
mock_celery.assert_called_with(
kwargs={
'service_id': sample_service.id,
'process_day': '2019-07-31',
'notification_type': SMS_TYPE
"service_id": sample_service.id,
"process_day": "2019-07-31",
"notification_type": SMS_TYPE,
},
queue=QueueNames.REPORTING
queue=QueueNames.REPORTING,
)
@freeze_time('2019-08-01T00:30')
@pytest.mark.parametrize('notification_date, expected_types_aggregated', [
('2019-08-01', set()),
('2019-07-31', {EMAIL_TYPE, SMS_TYPE}),
('2019-07-28', {EMAIL_TYPE, SMS_TYPE}),
('2019-07-21', set()),
])
@freeze_time("2019-08-01T00:30")
@pytest.mark.parametrize(
"notification_date, expected_types_aggregated",
[
("2019-08-01", set()),
("2019-07-31", {EMAIL_TYPE, SMS_TYPE}),
("2019-07-28", {EMAIL_TYPE, SMS_TYPE}),
("2019-07-21", set()),
],
)
def test_create_nightly_notification_status_triggers_relevant_tasks(
notify_api,
sample_service,
@@ -95,7 +107,7 @@ def test_create_nightly_notification_status_triggers_relevant_tasks(
expected_types_aggregated,
):
mock_celery = mocker.patch(
'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
"app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day"
).apply_async
for notification_type in NOTIFICATION_TYPES:
@@ -104,28 +116,28 @@ def test_create_nightly_notification_status_triggers_relevant_tasks(
create_nightly_notification_status()
types = {call.kwargs['kwargs']['notification_type'] for call in mock_celery.mock_calls}
types = {
call.kwargs["kwargs"]["notification_type"] for call in mock_celery.mock_calls
}
assert types == expected_types_aggregated
def test_create_nightly_billing_for_day_checks_history(
sample_service,
sample_template,
mocker
sample_service, sample_template, mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_template,
status='sending',
status="sending",
)
create_notification_history(
created_at=yesterday,
template=sample_template,
status='delivered',
status="delivered",
)
records = FactBilling.query.all()
@@ -140,9 +152,10 @@ def test_create_nightly_billing_for_day_checks_history(
assert record.notifications_sent == 2
@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
[(1.0, 1, 2, [1]),
(2.0, 2, 1, [1, 2])])
@pytest.mark.parametrize(
"second_rate, records_num, billable_units, multiplier",
[(1.0, 1, 2, [1]), (2.0, 2, 1, [1, 2])],
)
def test_create_nightly_billing_for_day_sms_rate_multiplier(
sample_service,
sample_template,
@@ -150,19 +163,18 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
second_rate,
records_num,
billable_units,
multiplier
multiplier,
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
# These are sms notifications
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=1.0,
billable_units=1,
@@ -170,8 +182,8 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=second_rate,
billable_units=1,
@@ -181,7 +193,7 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
assert len(records) == 0
create_nightly_billing_for_day(str(yesterday.date()))
records = FactBilling.query.order_by('rate_multiplier').all()
records = FactBilling.query.order_by("rate_multiplier").all()
assert len(records) == records_num
for i, record in enumerate(records):
@@ -192,20 +204,17 @@ def test_create_nightly_billing_for_day_sms_rate_multiplier(
def test_create_nightly_billing_for_day_different_templates(
sample_service,
sample_template,
sample_email_template,
mocker
sample_service, sample_template, sample_email_template, mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=1.0,
billable_units=1,
@@ -213,8 +222,8 @@ def test_create_nightly_billing_for_day_different_templates(
create_notification(
created_at=yesterday,
template=sample_email_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=0,
billable_units=0,
@@ -224,7 +233,7 @@ def test_create_nightly_billing_for_day_different_templates(
assert len(records) == 0
create_nightly_billing_for_day(str(yesterday.date()))
records = FactBilling.query.order_by('rate_multiplier').all()
records = FactBilling.query.order_by("rate_multiplier").all()
assert len(records) == 2
multiplier = [0, 1]
billable_units = [0, 1]
@@ -238,21 +247,18 @@ def test_create_nightly_billing_for_day_different_templates(
def test_create_nightly_billing_for_day_same_sent_by(
sample_service,
sample_template,
sample_email_template,
mocker
sample_service, sample_template, sample_email_template, mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
# These are sms notifications
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=1.0,
billable_units=1,
@@ -260,8 +266,8 @@ def test_create_nightly_billing_for_day_same_sent_by(
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='sns',
status="delivered",
sent_by="sns",
international=False,
rate_multiplier=1.0,
billable_units=1,
@@ -271,7 +277,7 @@ def test_create_nightly_billing_for_day_same_sent_by(
assert len(records) == 0
create_nightly_billing_for_day(str(yesterday.date()))
records = FactBilling.query.order_by('rate_multiplier').all()
records = FactBilling.query.order_by("rate_multiplier").all()
assert len(records) == 1
for _, record in enumerate(records):
@@ -282,18 +288,16 @@ def test_create_nightly_billing_for_day_same_sent_by(
def test_create_nightly_billing_for_day_null_sent_by_sms(
sample_service,
sample_template,
mocker
sample_service, sample_template, mocker
):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
status="delivered",
sent_by=None,
international=False,
rate_multiplier=1.0,
@@ -312,13 +316,13 @@ def test_create_nightly_billing_for_day_null_sent_by_sms(
assert record.rate == Decimal(1.33)
assert record.billable_units == 1
assert record.rate_multiplier == 1
assert record.provider == 'unknown'
assert record.provider == "unknown"
def test_get_rate_for_sms_and_email(notify_db_session):
non_letter_rates = [
create_rate(datetime(2017, 12, 1), 0.15, SMS_TYPE),
create_rate(datetime(2017, 12, 1), 0, EMAIL_TYPE)
create_rate(datetime(2017, 12, 1), 0, EMAIL_TYPE),
]
rate = get_rate(non_letter_rates, SMS_TYPE, date(2018, 1, 1))
@@ -328,20 +332,18 @@ def test_get_rate_for_sms_and_email(notify_db_session):
assert rate == Decimal(0)
@freeze_time('2018-03-26T04:30:00')
@freeze_time("2018-03-26T04:30:00")
# summer time starts on 2018-03-25
def test_create_nightly_billing_for_day_use_BST(
sample_service,
sample_template,
mocker):
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
sample_service, sample_template, mocker
):
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
# too late
create_notification(
created_at=datetime(2018, 3, 26, 4, 1),
template=sample_template,
status='delivered',
status="delivered",
rate_multiplier=1.0,
billable_units=1,
)
@@ -349,7 +351,7 @@ def test_create_nightly_billing_for_day_use_BST(
create_notification(
created_at=datetime(2018, 3, 25, 23, 59),
template=sample_template,
status='delivered',
status="delivered",
rate_multiplier=1.0,
billable_units=2,
)
@@ -358,7 +360,7 @@ def test_create_nightly_billing_for_day_use_BST(
create_notification(
created_at=datetime(2018, 3, 24, 23, 59),
template=sample_template,
status='delivered',
status="delivered",
rate_multiplier=1.0,
billable_units=4,
)
@@ -366,7 +368,7 @@ def test_create_nightly_billing_for_day_use_BST(
assert Notification.query.count() == 3
assert FactBilling.query.count() == 0
create_nightly_billing_for_day('2018-03-25')
create_nightly_billing_for_day("2018-03-25")
records = FactBilling.query.order_by(FactBilling.local_date).all()
assert len(records) == 1
@@ -374,18 +376,16 @@ def test_create_nightly_billing_for_day_use_BST(
assert records[0].billable_units == 2
@freeze_time('2018-01-15T08:30:00')
@freeze_time("2018-01-15T08:30:00")
def test_create_nightly_billing_for_day_update_when_record_exists(
sample_service,
sample_template,
mocker):
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
sample_service, sample_template, mocker
):
mocker.patch("app.dao.fact_billing_dao.get_rate", side_effect=mocker_get_rate)
create_notification(
created_at=datetime.now() - timedelta(days=1),
template=sample_template,
status='delivered',
status="delivered",
sent_by=None,
international=False,
rate_multiplier=1.0,
@@ -395,7 +395,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
records = FactBilling.query.all()
assert len(records) == 0
create_nightly_billing_for_day('2018-01-14')
create_nightly_billing_for_day("2018-01-14")
records = FactBilling.query.order_by(FactBilling.local_date).all()
assert len(records) == 1
@@ -406,7 +406,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
create_notification(
created_at=datetime.now() - timedelta(days=1),
template=sample_template,
status='delivered',
status="delivered",
sent_by=None,
international=False,
rate_multiplier=1.0,
@@ -414,41 +414,51 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
)
# run again, make sure create_nightly_billing() updates with no error
create_nightly_billing_for_day('2018-01-14')
create_nightly_billing_for_day("2018-01-14")
assert len(records) == 1
assert records[0].billable_units == 2
assert records[0].updated_at
def test_create_nightly_notification_status_for_service_and_day(notify_db_session):
first_service = create_service(service_name='First Service')
first_service = create_service(service_name="First Service")
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
second_service = create_service(service_name="second Service")
second_template = create_template(service=second_service, template_type="email")
process_day = datetime.utcnow().date() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.max)):
create_notification(template=first_template, status='delivered')
create_notification(template=second_template, status='failed')
create_notification(template=first_template, status="delivered")
create_notification(template=second_template, status="failed")
# team API key notifications are included
create_notification(template=second_template, status='sending', key_type=KEY_TYPE_TEAM)
create_notification(
template=second_template, status="sending", key_type=KEY_TYPE_TEAM
)
# test notifications are ignored
create_notification(template=second_template, status='sending', key_type=KEY_TYPE_TEST)
create_notification(
template=second_template, status="sending", key_type=KEY_TYPE_TEST
)
# historical notifications are included
create_notification_history(template=second_template, status='delivered')
create_notification_history(template=second_template, status="delivered")
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(datetime.utcnow().date() - timedelta(days=4), time.max)):
with freeze_time(
datetime.combine(datetime.utcnow().date() - timedelta(days=4), time.max)
):
create_notification(template=first_template)
create_notification_history(template=second_template)
assert len(FactNotificationStatus.query.all()) == 0
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
create_nightly_notification_status_for_service_and_day(
str(process_day), first_service.id, "sms"
)
create_nightly_notification_status_for_service_and_day(
str(process_day), second_service.id, "email"
)
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_type,
@@ -460,16 +470,16 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
email_delivered_row = new_fact_data[0]
assert email_delivered_row.template_id == second_template.id
assert email_delivered_row.service_id == second_service.id
assert email_delivered_row.notification_type == 'email'
assert email_delivered_row.notification_status == 'delivered'
assert email_delivered_row.notification_type == "email"
assert email_delivered_row.notification_status == "delivered"
assert email_delivered_row.notification_count == 1
assert email_delivered_row.key_type == KEY_TYPE_NORMAL
email_sending_row = new_fact_data[1]
assert email_sending_row.template_id == second_template.id
assert email_sending_row.service_id == second_service.id
assert email_sending_row.notification_type == 'email'
assert email_sending_row.notification_status == 'failed'
assert email_sending_row.notification_type == "email"
assert email_sending_row.notification_status == "failed"
assert email_sending_row.notification_count == 1
assert email_sending_row.key_type == KEY_TYPE_NORMAL
@@ -477,40 +487,46 @@ def test_create_nightly_notification_status_for_service_and_day(notify_db_sessio
assert email_failure_row.local_date == process_day
assert email_failure_row.template_id == second_template.id
assert email_failure_row.service_id == second_service.id
assert email_failure_row.job_id == UUID('00000000-0000-0000-0000-000000000000')
assert email_failure_row.notification_type == 'email'
assert email_failure_row.notification_status == 'sending'
assert email_failure_row.job_id == UUID("00000000-0000-0000-0000-000000000000")
assert email_failure_row.notification_type == "email"
assert email_failure_row.notification_status == "sending"
assert email_failure_row.notification_count == 1
assert email_failure_row.key_type == KEY_TYPE_TEAM
sms_delivered_row = new_fact_data[3]
assert sms_delivered_row.template_id == first_template.id
assert sms_delivered_row.service_id == first_service.id
assert sms_delivered_row.notification_type == 'sms'
assert sms_delivered_row.notification_status == 'delivered'
assert sms_delivered_row.notification_type == "sms"
assert sms_delivered_row.notification_status == "delivered"
assert sms_delivered_row.notification_count == 1
assert sms_delivered_row.key_type == KEY_TYPE_NORMAL
def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
first_service = create_service(service_name='First Service')
def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(
notify_db_session,
):
first_service = create_service(service_name="First Service")
first_template = create_template(service=first_service)
process_day = datetime.utcnow().date()
# first run: one notification, expect one row (just one status)
notification = create_notification(template=first_template, status='sending')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
notification = create_notification(template=first_template, status="sending")
create_nightly_notification_status_for_service_and_day(
str(process_day), first_service.id, "sms"
)
new_fact_data = FactNotificationStatus.query.all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].notification_status == 'sending'
assert new_fact_data[0].notification_status == "sending"
# second run: status changed, still expect one row (one status)
notification.status = 'delivered'
create_notification(template=first_template, status='created')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
notification.status = "delivered"
create_notification(template=first_template, status="created")
create_nightly_notification_status_for_service_and_day(
str(process_day), first_service.id, "sms"
)
updated_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_status
@@ -518,25 +534,39 @@ def test_create_nightly_notification_status_for_service_and_day_overwrites_old_d
assert len(updated_fact_data) == 2
assert updated_fact_data[0].notification_count == 1
assert updated_fact_data[0].notification_status == 'created'
assert updated_fact_data[0].notification_status == "created"
assert updated_fact_data[1].notification_count == 1
assert updated_fact_data[1].notification_status == 'delivered'
assert updated_fact_data[1].notification_status == "delivered"
# the job runs at 04:30am EST time.
@freeze_time('2019-04-02T04:30')
def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template):
create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 2, 5, 0)) # too new
@freeze_time("2019-04-02T04:30")
def test_create_nightly_notification_status_for_service_and_day_respects_bst(
sample_template,
):
create_notification(
sample_template, status="delivered", created_at=datetime(2019, 4, 2, 5, 0)
) # too new
create_notification(sample_template, status='created', created_at=datetime(2019, 4, 2, 5, 59))
create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 4, 0))
create_notification(
sample_template, status="created", created_at=datetime(2019, 4, 2, 5, 59)
)
create_notification(
sample_template, status="created", created_at=datetime(2019, 4, 1, 4, 0)
)
create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 21, 17, 59)) # too old
create_notification(
sample_template, status="delivered", created_at=datetime(2019, 3, 21, 17, 59)
) # too old
create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms')
create_nightly_notification_status_for_service_and_day(
"2019-04-01", sample_template.service_id, "sms"
)
noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.local_date).all()
noti_status = FactNotificationStatus.query.order_by(
FactNotificationStatus.local_date
).all()
assert len(noti_status) == 1
assert noti_status[0].local_date == date(2019, 4, 1)
assert noti_status[0].notification_status == 'created'
assert noti_status[0].notification_status == "created"

429
tests/app/celery/test_scheduled_tasks.py
View File

@@ -30,228 +30,273 @@ from tests.app import load_example_csv
from tests.app.db import create_job, create_notification, create_template
def test_should_call_delete_codes_on_delete_verify_codes_task(notify_db_session, mocker):
mocker.patch('app.celery.scheduled_tasks.delete_codes_older_created_more_than_a_day_ago')
def test_should_call_delete_codes_on_delete_verify_codes_task(
notify_db_session, mocker
):
mocker.patch(
"app.celery.scheduled_tasks.delete_codes_older_created_more_than_a_day_ago"
)
delete_verify_codes()
assert scheduled_tasks.delete_codes_older_created_more_than_a_day_ago.call_count == 1
assert (
scheduled_tasks.delete_codes_older_created_more_than_a_day_ago.call_count == 1
)
def test_should_call_delete_invotations_on_delete_invitations_task(notify_db_session, mocker):
mocker.patch('app.celery.scheduled_tasks.delete_invitations_created_more_than_two_days_ago')
def test_should_call_delete_invotations_on_delete_invitations_task(
notify_db_session, mocker
):
mocker.patch(
"app.celery.scheduled_tasks.delete_invitations_created_more_than_two_days_ago"
)
delete_invitations()
assert scheduled_tasks.delete_invitations_created_more_than_two_days_ago.call_count == 1
assert (
scheduled_tasks.delete_invitations_created_more_than_two_days_ago.call_count
== 1
)
def test_should_update_scheduled_jobs_and_put_on_queue(mocker, sample_template):
mocked = mocker.patch('app.celery.tasks.process_job.apply_async')
mocked = mocker.patch("app.celery.tasks.process_job.apply_async")
one_minute_in_the_past = datetime.utcnow() - timedelta(minutes=1)
job = create_job(sample_template, job_status='scheduled', scheduled_for=one_minute_in_the_past)
job = create_job(
sample_template, job_status="scheduled", scheduled_for=one_minute_in_the_past
)
run_scheduled_jobs()
updated_job = dao_get_job_by_id(job.id)
assert updated_job.job_status == 'pending'
assert updated_job.job_status == "pending"
mocked.assert_called_with([str(job.id)], queue="job-tasks")
def test_should_update_all_scheduled_jobs_and_put_on_queue(sample_template, mocker):
mocked = mocker.patch('app.celery.tasks.process_job.apply_async')
mocked = mocker.patch("app.celery.tasks.process_job.apply_async")
one_minute_in_the_past = datetime.utcnow() - timedelta(minutes=1)
ten_minutes_in_the_past = datetime.utcnow() - timedelta(minutes=10)
twenty_minutes_in_the_past = datetime.utcnow() - timedelta(minutes=20)
job_1 = create_job(sample_template, job_status='scheduled', scheduled_for=one_minute_in_the_past)
job_2 = create_job(sample_template, job_status='scheduled', scheduled_for=ten_minutes_in_the_past)
job_3 = create_job(sample_template, job_status='scheduled', scheduled_for=twenty_minutes_in_the_past)
job_1 = create_job(
sample_template, job_status="scheduled", scheduled_for=one_minute_in_the_past
)
job_2 = create_job(
sample_template, job_status="scheduled", scheduled_for=ten_minutes_in_the_past
)
job_3 = create_job(
sample_template,
job_status="scheduled",
scheduled_for=twenty_minutes_in_the_past,
)
run_scheduled_jobs()
assert dao_get_job_by_id(job_1.id).job_status == 'pending'
assert dao_get_job_by_id(job_2.id).job_status == 'pending'
assert dao_get_job_by_id(job_2.id).job_status == 'pending'
assert dao_get_job_by_id(job_1.id).job_status == "pending"
assert dao_get_job_by_id(job_2.id).job_status == "pending"
assert dao_get_job_by_id(job_2.id).job_status == "pending"
mocked.assert_has_calls([
call([str(job_3.id)], queue="job-tasks"),
call([str(job_2.id)], queue="job-tasks"),
call([str(job_1.id)], queue="job-tasks")
])
mocked.assert_has_calls(
[
call([str(job_3.id)], queue="job-tasks"),
call([str(job_2.id)], queue="job-tasks"),
call([str(job_1.id)], queue="job-tasks"),
]
)
def test_check_job_status_task_calls_process_incomplete_jobs(mocker, sample_template):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
job = create_job(template=sample_template, notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS)
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS,
)
create_notification(template=sample_template, job=job)
check_job_status()
mock_celery.assert_called_once_with(
[[str(job.id)]],
queue=QueueNames.JOBS
)
mock_celery.assert_called_once_with([[str(job.id)]], queue=QueueNames.JOBS)
def test_check_job_status_task_calls_process_incomplete_jobs_when_scheduled_job_is_not_complete(
mocker, sample_template
):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
job = create_job(template=sample_template, notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS)
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS,
)
check_job_status()
mock_celery.assert_called_once_with(
[[str(job.id)]],
queue=QueueNames.JOBS
)
mock_celery.assert_called_once_with([[str(job.id)]], queue=QueueNames.JOBS)
def test_check_job_status_task_calls_process_incomplete_jobs_for_pending_scheduled_jobs(
mocker, sample_template
):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
job = create_job(template=sample_template, notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_PENDING)
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_PENDING,
)
check_job_status()
mock_celery.assert_called_once_with(
[[str(job.id)]],
queue=QueueNames.JOBS
)
mock_celery.assert_called_once_with([[str(job.id)]], queue=QueueNames.JOBS)
def test_check_job_status_task_does_not_call_process_incomplete_jobs_for_non_scheduled_pending_jobs(
mocker,
sample_template,
):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
job_status=JOB_STATUS_PENDING
job_status=JOB_STATUS_PENDING,
)
check_job_status()
assert not mock_celery.called
def test_check_job_status_task_calls_process_incomplete_jobs_for_multiple_jobs(mocker, sample_template):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
job = create_job(template=sample_template, notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS)
job_2 = create_job(template=sample_template, notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS)
check_job_status()
mock_celery.assert_called_once_with(
[[str(job.id), str(job_2.id)]],
queue=QueueNames.JOBS
)
def test_check_job_status_task_only_sends_old_tasks(mocker, sample_template):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
def test_check_job_status_task_calls_process_incomplete_jobs_for_multiple_jobs(
mocker, sample_template
):
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS
job_status=JOB_STATUS_IN_PROGRESS,
)
job_2 = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS,
)
check_job_status()
mock_celery.assert_called_once_with(
[[str(job.id), str(job_2.id)]], queue=QueueNames.JOBS
)
def test_check_job_status_task_only_sends_old_tasks(mocker, sample_template):
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS,
)
create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=29),
job_status=JOB_STATUS_IN_PROGRESS
job_status=JOB_STATUS_IN_PROGRESS,
)
create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=50),
scheduled_for=datetime.utcnow() - timedelta(minutes=29),
job_status=JOB_STATUS_PENDING
job_status=JOB_STATUS_PENDING,
)
check_job_status()
# jobs 2 and 3 were created less than 30 minutes ago, so are not sent to Celery task
mock_celery.assert_called_once_with(
[[str(job.id)]],
queue=QueueNames.JOBS
)
mock_celery.assert_called_once_with([[str(job.id)]], queue=QueueNames.JOBS)
def test_check_job_status_task_sets_jobs_to_error(mocker, sample_template):
mock_celery = mocker.patch('app.celery.tasks.process_incomplete_jobs.apply_async')
mock_celery = mocker.patch("app.celery.tasks.process_incomplete_jobs.apply_async")
job = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_IN_PROGRESS
job_status=JOB_STATUS_IN_PROGRESS,
)
job_2 = create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=29),
job_status=JOB_STATUS_IN_PROGRESS
job_status=JOB_STATUS_IN_PROGRESS,
)
check_job_status()
# job 2 not in celery task
mock_celery.assert_called_once_with(
[[str(job.id)]],
queue=QueueNames.JOBS
)
mock_celery.assert_called_once_with([[str(job.id)]], queue=QueueNames.JOBS)
assert job.job_status == JOB_STATUS_ERROR
assert job_2.job_status == JOB_STATUS_IN_PROGRESS
def test_replay_created_notifications(notify_db_session, sample_service, mocker):
email_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_email.apply_async')
sms_delivery_queue = mocker.patch('app.celery.provider_tasks.deliver_sms.apply_async')
email_delivery_queue = mocker.patch(
"app.celery.provider_tasks.deliver_email.apply_async"
)
sms_delivery_queue = mocker.patch(
"app.celery.provider_tasks.deliver_sms.apply_async"
)
sms_template = create_template(service=sample_service, template_type='sms')
email_template = create_template(service=sample_service, template_type='email')
sms_template = create_template(service=sample_service, template_type="sms")
email_template = create_template(service=sample_service, template_type="email")
older_than = (60 * 60) + (60 * 15) # 1 hour 15 minutes
# notifications expected to be resent
old_sms = create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than),
status='created')
old_email = create_notification(template=email_template,
created_at=datetime.utcnow() - timedelta(seconds=older_than),
status='created')
old_sms = create_notification(
template=sms_template,
created_at=datetime.utcnow() - timedelta(seconds=older_than),
status="created",
)
old_email = create_notification(
template=email_template,
created_at=datetime.utcnow() - timedelta(seconds=older_than),
status="created",
)
# notifications that are not to be resent
create_notification(template=sms_template, created_at=datetime.utcnow() - timedelta(seconds=older_than),
status='sending')
create_notification(template=email_template, created_at=datetime.utcnow() - timedelta(seconds=older_than),
status='delivered')
create_notification(template=sms_template, created_at=datetime.utcnow(),
status='created')
create_notification(template=email_template, created_at=datetime.utcnow(),
status='created')
create_notification(
template=sms_template,
created_at=datetime.utcnow() - timedelta(seconds=older_than),
status="sending",
)
create_notification(
template=email_template,
created_at=datetime.utcnow() - timedelta(seconds=older_than),
status="delivered",
)
create_notification(
template=sms_template, created_at=datetime.utcnow(), status="created"
)
create_notification(
template=email_template, created_at=datetime.utcnow(), status="created"
)
replay_created_notifications()
email_delivery_queue.assert_called_once_with([str(old_email.id)],
queue='send-email-tasks')
sms_delivery_queue.assert_called_once_with([str(old_sms.id)],
queue="send-sms-tasks")
email_delivery_queue.assert_called_once_with(
[str(old_email.id)], queue="send-email-tasks"
)
sms_delivery_queue.assert_called_once_with(
[str(old_sms.id)], queue="send-sms-tasks"
)
def test_check_job_status_task_does_not_raise_error(sample_template):
@@ -261,32 +306,39 @@ def test_check_job_status_task_does_not_raise_error(sample_template):
created_at=datetime.utcnow() - timedelta(hours=2),
scheduled_for=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_FINISHED)
job_status=JOB_STATUS_FINISHED,
)
create_job(
template=sample_template,
notification_count=3,
created_at=datetime.utcnow() - timedelta(minutes=31),
processing_started=datetime.utcnow() - timedelta(minutes=31),
job_status=JOB_STATUS_FINISHED)
job_status=JOB_STATUS_FINISHED,
)
check_job_status()
@pytest.mark.parametrize('offset', (
timedelta(days=1),
pytest.param(timedelta(hours=23, minutes=59), marks=pytest.mark.xfail),
pytest.param(timedelta(minutes=20), marks=pytest.mark.xfail),
timedelta(minutes=19),
))
@pytest.mark.parametrize(
"offset",
(
timedelta(days=1),
pytest.param(timedelta(hours=23, minutes=59), marks=pytest.mark.xfail),
pytest.param(timedelta(minutes=20), marks=pytest.mark.xfail),
timedelta(minutes=19),
),
)
def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
mocker,
sample_email_template,
offset,
):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
mocker.patch(
"app.celery.tasks.s3.get_job_and_metadata_from_s3",
return_value=(load_example_csv("multiple_email"), {"sender_id": None}),
)
mocker.patch("app.encryption.encrypt", return_value="something_encrypted")
process_row = mocker.patch("app.celery.scheduled_tasks.process_row")
job = create_job(
template=sample_email_template,
@@ -303,15 +355,19 @@ def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs(
def test_check_for_missing_rows_in_completed_jobs(mocker, sample_email_template):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(load_example_csv('multiple_email'), {"sender_id": None}))
mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
mocker.patch(
"app.celery.tasks.s3.get_job_and_metadata_from_s3",
return_value=(load_example_csv("multiple_email"), {"sender_id": None}),
)
mocker.patch("app.encryption.encrypt", return_value="something_encrypted")
process_row = mocker.patch("app.celery.scheduled_tasks.process_row")
job = create_job(template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20))
job = create_job(
template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20),
)
for i in range(0, 4):
create_notification(job=job, job_row_number=i)
@@ -322,17 +378,23 @@ def test_check_for_missing_rows_in_completed_jobs(mocker, sample_email_template)
)
def test_check_for_missing_rows_in_completed_jobs_calls_save_email(mocker, sample_email_template):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(load_example_csv('multiple_email'), {'sender_id': None}))
save_email_task = mocker.patch('app.celery.tasks.save_email.apply_async')
mocker.patch('app.encryption.encrypt', return_value="something_encrypted")
mocker.patch('app.celery.tasks.create_uuid', return_value='uuid')
def test_check_for_missing_rows_in_completed_jobs_calls_save_email(
mocker, sample_email_template
):
mocker.patch(
"app.celery.tasks.s3.get_job_and_metadata_from_s3",
return_value=(load_example_csv("multiple_email"), {"sender_id": None}),
)
save_email_task = mocker.patch("app.celery.tasks.save_email.apply_async")
mocker.patch("app.encryption.encrypt", return_value="something_encrypted")
mocker.patch("app.celery.tasks.create_uuid", return_value="uuid")
job = create_job(template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20))
job = create_job(
template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20),
)
for i in range(0, 4):
create_notification(job=job, job_row_number=i)
@@ -344,19 +406,25 @@ def test_check_for_missing_rows_in_completed_jobs_calls_save_email(mocker, sampl
"something_encrypted",
),
{},
queue="database-tasks"
queue="database-tasks",
)
def test_check_for_missing_rows_in_completed_jobs_uses_sender_id(mocker, sample_email_template, fake_uuid):
mocker.patch('app.celery.tasks.s3.get_job_and_metadata_from_s3',
return_value=(load_example_csv('multiple_email'), {'sender_id': fake_uuid}))
mock_process_row = mocker.patch('app.celery.scheduled_tasks.process_row')
def test_check_for_missing_rows_in_completed_jobs_uses_sender_id(
mocker, sample_email_template, fake_uuid
):
mocker.patch(
"app.celery.tasks.s3.get_job_and_metadata_from_s3",
return_value=(load_example_csv("multiple_email"), {"sender_id": fake_uuid}),
)
mock_process_row = mocker.patch("app.celery.scheduled_tasks.process_row")
job = create_job(template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20))
job = create_job(
template=sample_email_template,
notification_count=5,
job_status=JOB_STATUS_FINISHED,
processing_finished=datetime.utcnow() - timedelta(minutes=20),
)
for i in range(0, 4):
create_notification(job=job, job_row_number=i)
@@ -367,53 +435,58 @@ def test_check_for_missing_rows_in_completed_jobs_uses_sender_id(mocker, sample_
MockServicesSendingToTVNumbers = namedtuple(
'ServicesSendingToTVNumbers',
"ServicesSendingToTVNumbers",
[
'service_id',
'notification_count',
]
"service_id",
"notification_count",
],
)
MockServicesWithHighFailureRate = namedtuple(
'ServicesWithHighFailureRate',
"ServicesWithHighFailureRate",
[
'service_id',
'permanent_failure_rate',
]
"service_id",
"permanent_failure_rate",
],
)
@pytest.mark.parametrize("failure_rates, sms_to_tv_numbers, expected_message", [
@pytest.mark.parametrize(
"failure_rates, sms_to_tv_numbers, expected_message",
[
[MockServicesWithHighFailureRate("123", 0.3)],
[],
"1 service(s) have had high permanent-failure rates for sms messages in last "
"24 hours:\nservice: {}/services/{} failure rate: 0.3,\n".format(
Test.ADMIN_BASE_URL, "123"
)
[
[MockServicesWithHighFailureRate("123", 0.3)],
[],
"1 service(s) have had high permanent-failure rates for sms messages in last "
"24 hours:\nservice: {}/services/{} failure rate: 0.3,\n".format(
Test.ADMIN_BASE_URL, "123"
),
],
[
[],
[MockServicesSendingToTVNumbers("123", 300)],
"1 service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n"
"service: {}/services/{} count of sms to tv numbers: 300,\n".format(
Test.ADMIN_BASE_URL, "123"
),
],
],
[
[],
[MockServicesSendingToTVNumbers("123", 300)],
"1 service(s) have sent over 500 sms messages to tv numbers in last 24 hours:\n"
"service: {}/services/{} count of sms to tv numbers: 300,\n".format(
Test.ADMIN_BASE_URL, "123"
)
]
])
)
def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(
mocker, notify_db_session, failure_rates, sms_to_tv_numbers, expected_message
):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.warning')
mock_create_ticket = mocker.spy(NotifySupportTicket, '__init__')
mock_logger = mocker.patch("app.celery.tasks.current_app.logger.warning")
mock_create_ticket = mocker.spy(NotifySupportTicket, "__init__")
mock_send_ticket_to_zendesk = mocker.patch(
'app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk',
"app.celery.scheduled_tasks.zendesk_client.send_ticket_to_zendesk",
autospec=True,
)
mock_failure_rates = mocker.patch(
'app.celery.scheduled_tasks.dao_find_services_with_high_failure_rates', return_value=failure_rates
"app.celery.scheduled_tasks.dao_find_services_with_high_failure_rates",
return_value=failure_rates,
)
mock_sms_to_tv_numbers = mocker.patch(
'app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers', return_value=sms_to_tv_numbers
"app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers",
return_value=sms_to_tv_numbers,
)
zendesk_actions = "\nYou can find instructions for this ticket in our manual:\nhttps://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers" # noqa
@@ -427,7 +500,7 @@ def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers(
ANY,
message=expected_message + zendesk_actions,
subject="[test] High failure rates for sms spotted for services",
ticket_type='incident',
technical_ticket=True
ticket_type="incident",
technical_ticket=True,
)
mock_send_ticket_to_zendesk.assert_called_once()

181
tests/app/celery/test_service_callback_tasks.py
View File

@@ -22,23 +22,24 @@ from tests.app.db import (
@pytest.mark.parametrize("notification_type", ["email", "sms"])
def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data(
notify_db_session, notification_type):
notify_db_session, notification_type
):
callback_api, template = _set_up_test_data(notification_type, "delivery_status")
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
notification = create_notification(
template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status="sent",
)
encrypted_status_update = _set_up_data_for_status_update(callback_api, notification)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=200)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update)
request_mock.post(callback_api.url, json={}, status_code=200)
send_delivery_status_to_service(
notification.id, encrypted_status_update=encrypted_status_update
)
mock_data = {
"id": str(notification.id),
@@ -50,28 +51,34 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr
"sent_at": datestr.strftime(DATETIME_FORMAT),
"notification_type": notification_type,
"template_id": str(template.id),
"template_version": 1
"template_version": 1,
}
assert request_mock.call_count == 1
assert request_mock.request_history[0].url == callback_api.url
assert request_mock.request_history[0].method == 'POST'
assert request_mock.request_history[0].method == "POST"
assert request_mock.request_history[0].text == json.dumps(mock_data)
assert request_mock.request_history[0].headers["Content-type"] == "application/json"
assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
assert request_mock.request_history[0].headers[
"Authorization"
] == "Bearer {}".format(callback_api.bearer_token)
def test_send_complaint_to_service_posts_https_request_to_service_with_encrypted_data(notify_db_session):
with freeze_time('2001-01-01T12:00:00'):
callback_api, template = _set_up_test_data('email', "complaint")
def test_send_complaint_to_service_posts_https_request_to_service_with_encrypted_data(
notify_db_session,
):
with freeze_time("2001-01-01T12:00:00"):
callback_api, template = _set_up_test_data("email", "complaint")
notification = create_notification(template=template)
complaint = create_complaint(service=template.service, notification=notification)
complaint_data = _set_up_data_for_complaint(callback_api, complaint, notification)
complaint = create_complaint(
service=template.service, notification=notification
)
complaint_data = _set_up_data_for_complaint(
callback_api, complaint, notification
)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=200)
request_mock.post(callback_api.url, json={}, status_code=200)
send_complaint_to_service(complaint_data)
mock_data = {
@@ -79,99 +86,112 @@ def test_send_complaint_to_service_posts_https_request_to_service_with_encrypted
"complaint_id": str(complaint.id),
"reference": notification.client_reference,
"to": notification.to,
"complaint_date": datetime.utcnow().strftime(
DATETIME_FORMAT),
"complaint_date": datetime.utcnow().strftime(DATETIME_FORMAT),
}
assert request_mock.call_count == 1
assert request_mock.request_history[0].url == callback_api.url
assert request_mock.request_history[0].method == 'POST'
assert request_mock.request_history[0].method == "POST"
assert request_mock.request_history[0].text == json.dumps(mock_data)
assert request_mock.request_history[0].headers["Content-type"] == "application/json"
assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
assert (
request_mock.request_history[0].headers["Content-type"]
== "application/json"
)
assert request_mock.request_history[0].headers[
"Authorization"
] == "Bearer {}".format(callback_api.bearer_token)
@pytest.mark.parametrize("notification_type", ["email", "sms"])
@pytest.mark.parametrize('status_code', [429, 500, 503])
@pytest.mark.parametrize("status_code", [429, 500, 503])
def test__send_data_to_service_callback_api_retries_if_request_returns_error_code_with_encrypted_data(
notify_db_session,
mocker,
notification_type,
status_code
notify_db_session, mocker, notification_type, status_code
):
callback_api, template = _set_up_test_data(notification_type, "delivery_status")
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
notification = create_notification(
template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status="sent",
)
encrypted_data = _set_up_data_for_status_update(callback_api, notification)
mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
mocked = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.retry"
)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=status_code)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
request_mock.post(callback_api.url, json={}, status_code=status_code)
send_delivery_status_to_service(
notification.id, encrypted_status_update=encrypted_data
)
assert mocked.call_count == 1
assert mocked.call_args[1]['queue'] == 'service-callbacks-retry'
assert mocked.call_args[1]["queue"] == "service-callbacks-retry"
@pytest.mark.parametrize("notification_type", ["email", "sms"])
def test__send_data_to_service_callback_api_does_not_retry_if_request_returns_404_with_encrypted_data(
notify_db_session,
mocker,
notification_type
notify_db_session, mocker, notification_type
):
callback_api, template = _set_up_test_data(notification_type, "delivery_status")
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status='sent'
)
notification = create_notification(
template=template,
created_at=datestr,
updated_at=datestr,
sent_at=datestr,
status="sent",
)
encrypted_data = _set_up_data_for_status_update(callback_api, notification)
mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
mocked = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.retry"
)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=404)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
request_mock.post(callback_api.url, json={}, status_code=404)
send_delivery_status_to_service(
notification.id, encrypted_status_update=encrypted_data
)
assert mocked.call_count == 0
def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none(
notify_db_session,
mocker
notify_db_session, mocker
):
callback_api, template = _set_up_test_data('email', "delivery_status")
callback_api, template = _set_up_test_data("email", "delivery_status")
datestr = datetime(2017, 6, 20)
notification = create_notification(template=template,
created_at=datestr,
updated_at=datestr,
sent_at=None,
status='technical-failure'
)
notification = create_notification(
template=template,
created_at=datestr,
updated_at=datestr,
sent_at=None,
status="technical-failure",
)
encrypted_data = _set_up_data_for_status_update(callback_api, notification)
mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
mocked = mocker.patch(
"app.celery.service_callback_tasks.send_delivery_status_to_service.retry"
)
with requests_mock.Mocker() as request_mock:
request_mock.post(callback_api.url,
json={},
status_code=404)
send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_data)
request_mock.post(callback_api.url, json={}, status_code=404)
send_delivery_status_to_service(
notification.id, encrypted_status_update=encrypted_data
)
assert mocked.call_count == 0
def _set_up_test_data(notification_type, callback_type):
service = create_service(restricted=True)
template = create_template(service=service, template_type=notification_type, subject='Hello')
callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
bearer_token="something_unique", callback_type=callback_type)
template = create_template(
service=service, template_type=notification_type, subject="Hello"
)
callback_api = create_service_callback_api(
service=service,
url="https://some.service.gov.uk/",
bearer_token="something_unique",
callback_type=callback_type,
)
return callback_api, template
@@ -182,9 +202,12 @@ def _set_up_data_for_status_update(callback_api, notification):
"notification_to": notification.to,
"notification_status": notification.status,
"notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
"notification_updated_at": notification.updated_at.strftime(
DATETIME_FORMAT) if notification.updated_at else None,
"notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT) if notification.sent_at else None,
"notification_updated_at": notification.updated_at.strftime(DATETIME_FORMAT)
if notification.updated_at
else None,
"notification_sent_at": notification.sent_at.strftime(DATETIME_FORMAT)
if notification.sent_at
else None,
"notification_type": notification.notification_type,
"service_callback_api_url": callback_api.url,
"service_callback_api_bearer_token": callback_api.bearer_token,

1082
tests/app/celery/test_tasks.py
View File

File diff suppressed because it is too large Load Diff

44
tests/app/celery/test_test_key_tasks.py
View File

@@ -16,44 +16,42 @@ from app.models import NOTIFICATION_DELIVERED, NOTIFICATION_FAILED, Notification
from tests.conftest import Matcher
dvla_response_file_matcher = Matcher(
'dvla_response_file',
lambda x: 'NOTIFY-20180125140000-RSP.TXT' < x <= 'NOTIFY-20180125140030-RSP.TXT'
"dvla_response_file",
lambda x: "NOTIFY-20180125140000-RSP.TXT" < x <= "NOTIFY-20180125140030-RSP.TXT",
)
def test_make_sns_callback(notify_api, rmock, mocker):
endpoint = "http://localhost:6011/notifications/sms/sns"
get_notification_by_id = mocker.patch('app.celery.test_key_tasks.get_notification_by_id')
get_notification_by_id = mocker.patch(
"app.celery.test_key_tasks.get_notification_by_id"
)
n = Notification()
n.id = 1234
n.status = NOTIFICATION_DELIVERED
get_notification_by_id.return_value = n
rmock.request(
"POST",
endpoint,
json={"status": "success"},
status_code=200)
rmock.request("POST", endpoint, json={"status": "success"}, status_code=200)
send_sms_response("sns", "1234")
assert rmock.called
assert rmock.request_history[0].url == endpoint
assert json.loads(rmock.request_history[0].text)['status'] == 'delivered'
assert json.loads(rmock.request_history[0].text)["status"] == "delivered"
def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
endpoint = "http://localhost:6011/notifications/sms/sns"
get_notification_by_id = mocker.patch('app.celery.test_key_tasks.get_notification_by_id')
get_notification_by_id = mocker.patch(
"app.celery.test_key_tasks.get_notification_by_id"
)
n = Notification()
n.id = 1234
n.status = NOTIFICATION_FAILED
get_notification_by_id.return_value = n
rmock.request(
"POST",
endpoint,
json={"error": "something went wrong"},
status_code=500)
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
"POST", endpoint, json={"error": "something went wrong"}, status_code=500
)
mock_logger = mocker.patch("app.celery.tasks.current_app.logger.error")
with pytest.raises(HTTPError):
send_sms_response("sns", "1234")
@@ -61,27 +59,31 @@ def test_callback_logs_on_api_call_failure(notify_api, rmock, mocker):
assert rmock.called
assert rmock.request_history[0].url == endpoint
mock_logger.assert_called_once_with(
'API POST request on http://localhost:6011/notifications/sms/sns failed with status 500'
"API POST request on http://localhost:6011/notifications/sms/sns failed with status 500"
)
def test_make_ses_callback(notify_api, mocker):
mock_task = mocker.patch('app.celery.test_key_tasks.process_ses_results')
mock_task = mocker.patch("app.celery.test_key_tasks.process_ses_results")
some_ref = str(uuid.uuid4())
send_email_response(reference=some_ref, to="test@test.com")
mock_task.apply_async.assert_called_once_with(ANY, queue=QueueNames.SEND_EMAIL)
assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(some_ref)
assert mock_task.apply_async.call_args[0][0][0] == ses_notification_callback(
some_ref
)
def test_delivered_sns_callback(mocker):
get_notification_by_id = mocker.patch('app.celery.test_key_tasks.get_notification_by_id')
get_notification_by_id = mocker.patch(
"app.celery.test_key_tasks.get_notification_by_id"
)
n = Notification()
n.id = 1234
n.status = NOTIFICATION_DELIVERED
get_notification_by_id.return_value = n
data = json.loads(sns_callback("1234"))
assert data['status'] == "delivered"
assert data['CID'] == "1234"
assert data["status"] == "delivered"
assert data["CID"] == "1234"