2020-10-26 17:34:59 +00:00
import uuid
2016-06-20 13:33:53 +01:00
from datetime import datetime , timedelta
2019-01-16 17:32:19 +00:00
from unittest . mock import call
2017-06-07 16:31:51 +01:00
2018-11-15 17:24:37 +00:00
import pytest
2019-12-03 10:26:59 +00:00
from collections import namedtuple
2016-09-07 15:36:59 +01:00
from freezegun import freeze_time
2019-11-05 16:47:00 +00:00
from mock import mock
2017-06-07 16:31:51 +01:00
2016-06-20 13:33:53 +01:00
from app . celery import scheduled_tasks
2017-01-27 12:30:56 +00:00
from app . celery . scheduled_tasks import (
2017-10-16 12:32:44 +01:00
check_job_status ,
2017-06-02 14:28:52 +01:00
delete_invitations ,
2017-01-27 12:30:56 +00:00
delete_verify_codes ,
run_scheduled_jobs ,
2019-06-11 13:16:34 +01:00
replay_created_notifications ,
check_precompiled_letter_state ,
2019-06-11 15:13:06 +01:00
check_templated_letter_state ,
2019-11-13 17:01:03 +00:00
check_for_missing_rows_in_completed_jobs ,
2019-11-29 21:24:17 +00:00
check_for_services_with_high_failure_rates_or_sending_to_tv_numbers ,
2019-11-13 17:01:03 +00:00
switch_current_sms_provider_on_slow_delivery ,
2020-11-16 12:47:38 +00:00
trigger_link_tests ,
2017-11-09 14:13:42 +00:00
)
2020-07-22 17:00:20 +01:00
from app . config import QueueNames , Config
2016-08-24 17:03:56 +01:00
from app . dao . jobs_dao import dao_get_job_by_id
2019-11-14 10:52:24 +00:00
from app . dao . provider_details_dao import get_provider_details_by_identifier
2017-06-06 16:02:01 +01:00
from app . models import (
2017-09-26 12:03:06 +01:00
JOB_STATUS_IN_PROGRESS ,
2018-03-09 16:34:47 +00:00
JOB_STATUS_ERROR ,
2019-01-16 17:32:19 +00:00
JOB_STATUS_FINISHED ,
2019-06-11 13:16:34 +01:00
NOTIFICATION_DELIVERED ,
NOTIFICATION_PENDING_VIRUS_CHECK ,
2017-11-10 13:49:20 +00:00
)
2020-11-16 12:47:38 +00:00
from tests . conftest import set_config
2019-11-05 16:47:00 +00:00
from tests . app import load_example_csv
2018-11-19 17:09:27 +00:00
from tests . app . db import (
create_notification ,
create_template ,
create_job ,
2017-06-12 15:55:42 +01:00
)
2016-06-20 13:33:53 +01:00
2018-12-05 14:40:07 +00:00
def _create_slow_delivery_notification ( template , provider = ' mmg ' ) :
2017-02-13 15:46:06 +00:00
now = datetime . utcnow ( )
five_minutes_from_now = now + timedelta ( minutes = 5 )
2017-02-24 13:41:32 +00:00
create_notification (
2017-02-13 15:46:06 +00:00
template = template ,
status = ' delivered ' ,
sent_by = provider ,
2018-12-05 14:40:07 +00:00
updated_at = five_minutes_from_now ,
sent_at = now ,
2017-02-13 15:46:06 +00:00
)
2019-06-17 13:52:36 +01:00
def test_should_call_delete_codes_on_delete_verify_codes_task ( notify_db_session , mocker ) :
2016-06-20 13:33:53 +01:00
mocker . patch ( ' app.celery.scheduled_tasks.delete_codes_older_created_more_than_a_day_ago ' )
delete_verify_codes ( )
assert scheduled_tasks . delete_codes_older_created_more_than_a_day_ago . call_count == 1
2019-11-20 17:23:39 +00:00
def test_should_call_delete_invotations_on_delete_invitations_task ( notify_db_session , mocker ) :
2016-06-20 13:33:53 +01:00
mocker . patch ( ' app.celery.scheduled_tasks.delete_invitations_created_more_than_two_days_ago ' )
delete_invitations ( )
assert scheduled_tasks . delete_invitations_created_more_than_two_days_ago . call_count == 1
2019-10-30 10:51:07 +00:00
def test_should_update_scheduled_jobs_and_put_on_queue ( mocker , sample_template ) :
2016-08-24 17:03:56 +01:00
mocked = mocker . patch ( ' app.celery.tasks.process_job.apply_async ' )
one_minute_in_the_past = datetime . utcnow ( ) - timedelta ( minutes = 1 )
2019-10-30 10:51:07 +00:00
job = create_job ( sample_template , job_status = ' scheduled ' , scheduled_for = one_minute_in_the_past )
2016-08-24 17:03:56 +01:00
run_scheduled_jobs ( )
updated_job = dao_get_job_by_id ( job . id )
assert updated_job . job_status == ' pending '
2017-05-25 10:51:49 +01:00
mocked . assert_called_with ( [ str ( job . id ) ] , queue = " job-tasks " )
2016-08-24 17:03:56 +01:00
2019-10-30 10:51:07 +00:00
def test_should_update_all_scheduled_jobs_and_put_on_queue ( sample_template , mocker ) :
2016-08-24 17:03:56 +01:00
mocked = mocker . patch ( ' app.celery.tasks.process_job.apply_async ' )
one_minute_in_the_past = datetime . utcnow ( ) - timedelta ( minutes = 1 )
ten_minutes_in_the_past = datetime . utcnow ( ) - timedelta ( minutes = 10 )
twenty_minutes_in_the_past = datetime . utcnow ( ) - timedelta ( minutes = 20 )
2019-10-30 10:51:07 +00:00
job_1 = create_job ( sample_template , job_status = ' scheduled ' , scheduled_for = one_minute_in_the_past )
job_2 = create_job ( sample_template , job_status = ' scheduled ' , scheduled_for = ten_minutes_in_the_past )
job_3 = create_job ( sample_template , job_status = ' scheduled ' , scheduled_for = twenty_minutes_in_the_past )
2016-08-24 17:03:56 +01:00
run_scheduled_jobs ( )
assert dao_get_job_by_id ( job_1 . id ) . job_status == ' pending '
assert dao_get_job_by_id ( job_2 . id ) . job_status == ' pending '
assert dao_get_job_by_id ( job_2 . id ) . job_status == ' pending '
mocked . assert_has_calls ( [
2017-05-25 10:51:49 +01:00
call ( [ str ( job_3 . id ) ] , queue = " job-tasks " ) ,
call ( [ str ( job_2 . id ) ] , queue = " job-tasks " ) ,
call ( [ str ( job_1 . id ) ] , queue = " job-tasks " )
2016-08-24 17:03:56 +01:00
] )
2016-09-07 15:36:59 +01:00
2019-11-13 17:01:03 +00:00
@freeze_time ( ' 2017-05-01 14:00:00 ' )
def test_switch_current_sms_provider_on_slow_delivery_switches_when_one_provider_is_slow (
mocker ,
restore_provider_details ,
2017-02-13 15:46:06 +00:00
) :
2019-11-13 17:01:03 +00:00
is_slow_dict = { ' mmg ' : False , ' firetext ' : True }
mock_is_slow = mocker . patch ( ' app.celery.scheduled_tasks.is_delivery_slow_for_providers ' , return_value = is_slow_dict )
mock_reduce = mocker . patch ( ' app.celery.scheduled_tasks.dao_reduce_sms_provider_priority ' )
# updated_at times are older than the 10 minute window
get_provider_details_by_identifier ( ' mmg ' ) . updated_at = datetime ( 2017 , 5 , 1 , 13 , 49 )
get_provider_details_by_identifier ( ' firetext ' ) . updated_at = None
switch_current_sms_provider_on_slow_delivery ( )
mock_is_slow . assert_called_once_with (
threshold = 0.3 ,
created_at = datetime ( 2017 , 5 , 1 , 13 , 50 ) ,
delivery_time = timedelta ( minutes = 4 )
)
2019-11-20 17:23:39 +00:00
mock_reduce . assert_called_once_with ( ' firetext ' , time_threshold = timedelta ( minutes = 10 ) )
2019-11-13 17:01:03 +00:00
@freeze_time ( ' 2017-05-01 14:00:00 ' )
@pytest.mark.parametrize ( ' is_slow_dict ' , [
{ ' mmg ' : False , ' firetext ' : False } ,
{ ' mmg ' : True , ' firetext ' : True } ,
] )
def test_switch_current_sms_provider_on_slow_delivery_does_nothing_if_no_need (
mocker ,
restore_provider_details ,
is_slow_dict
) :
mocker . patch ( ' app.celery.scheduled_tasks.is_delivery_slow_for_providers ' , return_value = is_slow_dict )
mock_reduce = mocker . patch ( ' app.celery.scheduled_tasks.dao_reduce_sms_provider_priority ' )
get_provider_details_by_identifier ( ' mmg ' ) . updated_at = datetime ( 2017 , 5 , 1 , 13 , 51 )
switch_current_sms_provider_on_slow_delivery ( )
assert mock_reduce . called is False
2017-05-22 14:05:57 +01:00
2020-07-22 17:00:20 +01:00
def test_check_job_status_task_calls_process_incomplete_jobs ( mocker , sample_template ) :
mock_celery = mocker . patch ( ' app.celery.tasks.process_incomplete_jobs.apply_async ' )
2017-10-12 16:21:08 +01:00
job = create_job ( template = sample_template , notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS )
create_notification ( template = sample_template , job = job )
2020-07-22 17:00:20 +01:00
check_job_status ( )
2017-10-12 16:21:08 +01:00
2017-10-13 16:46:17 +01:00
mock_celery . assert_called_once_with (
2020-07-22 17:00:20 +01:00
[ [ str ( job . id ) ] ] ,
2017-10-13 16:46:17 +01:00
queue = QueueNames . JOBS
)
2017-10-12 16:21:08 +01:00
2020-07-22 17:00:20 +01:00
def test_check_job_status_task_calls_process_incomplete_jobs_when_scheduled_job_is_not_complete (
mocker , sample_template
) :
mock_celery = mocker . patch ( ' app.celery.tasks.process_incomplete_jobs.apply_async ' )
2017-10-12 16:21:08 +01:00
job = create_job ( template = sample_template , notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS )
2020-07-22 17:00:20 +01:00
check_job_status ( )
2017-10-12 16:21:08 +01:00
2017-10-13 16:46:17 +01:00
mock_celery . assert_called_once_with (
2020-07-22 17:00:20 +01:00
[ [ str ( job . id ) ] ] ,
2017-10-13 16:46:17 +01:00
queue = QueueNames . JOBS
)
2017-10-12 16:21:08 +01:00
2020-07-22 17:00:20 +01:00
def test_check_job_status_task_calls_process_incomplete_jobs_for_multiple_jobs ( mocker , sample_template ) :
mock_celery = mocker . patch ( ' app.celery.tasks.process_incomplete_jobs.apply_async ' )
2017-10-12 16:21:08 +01:00
job = create_job ( template = sample_template , notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS )
job_2 = create_job ( template = sample_template , notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS )
2020-07-22 17:00:20 +01:00
check_job_status ( )
2017-10-12 16:21:08 +01:00
2017-10-13 16:46:17 +01:00
mock_celery . assert_called_once_with (
2020-07-22 17:00:20 +01:00
[ [ str ( job . id ) , str ( job_2 . id ) ] ] ,
2017-10-13 16:46:17 +01:00
queue = QueueNames . JOBS
)
2017-11-09 10:32:39 +00:00
2018-03-09 16:30:50 +00:00
def test_check_job_status_task_only_sends_old_tasks ( mocker , sample_template ) :
2020-07-22 17:00:20 +01:00
mock_celery = mocker . patch ( ' app.celery.tasks.process_incomplete_jobs.apply_async ' )
2018-03-09 16:30:50 +00:00
job = create_job (
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS
)
2020-07-22 17:00:20 +01:00
create_job (
2018-03-09 16:30:50 +00:00
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 29 ) ,
job_status = JOB_STATUS_IN_PROGRESS
)
2020-07-22 17:00:20 +01:00
check_job_status ( )
2018-03-09 16:30:50 +00:00
# job 2 not in celery task
mock_celery . assert_called_once_with (
2020-07-22 17:00:20 +01:00
[ [ str ( job . id ) ] ] ,
2018-03-09 16:30:50 +00:00
queue = QueueNames . JOBS
)
2018-03-09 16:34:47 +00:00
def test_check_job_status_task_sets_jobs_to_error ( mocker , sample_template ) :
2020-07-22 17:00:20 +01:00
mock_celery = mocker . patch ( ' app.celery.tasks.process_incomplete_jobs.apply_async ' )
2018-03-09 16:34:47 +00:00
job = create_job (
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_IN_PROGRESS
)
job_2 = create_job (
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 29 ) ,
job_status = JOB_STATUS_IN_PROGRESS
)
2020-07-22 17:00:20 +01:00
check_job_status ( )
2018-03-09 16:34:47 +00:00
# job 2 not in celery task
mock_celery . assert_called_once_with (
2020-07-22 17:00:20 +01:00
[ [ str ( job . id ) ] ] ,
2018-03-09 16:34:47 +00:00
queue = QueueNames . JOBS
)
assert job . job_status == JOB_STATUS_ERROR
assert job_2 . job_status == JOB_STATUS_IN_PROGRESS
2018-03-23 15:38:35 +00:00
def test_replay_created_notifications ( notify_db_session , sample_service , mocker ) :
email_delivery_queue = mocker . patch ( ' app.celery.provider_tasks.deliver_email.apply_async ' )
sms_delivery_queue = mocker . patch ( ' app.celery.provider_tasks.deliver_sms.apply_async ' )
sms_template = create_template ( service = sample_service , template_type = ' sms ' )
email_template = create_template ( service = sample_service , template_type = ' email ' )
2019-11-21 15:51:27 +00:00
older_than = ( 60 * 60 ) + ( 60 * 15 ) # 1 hour 15 minutes
2018-03-23 15:38:35 +00:00
# notifications expected to be resent
old_sms = create_notification ( template = sms_template , created_at = datetime . utcnow ( ) - timedelta ( seconds = older_than ) ,
status = ' created ' )
old_email = create_notification ( template = email_template ,
created_at = datetime . utcnow ( ) - timedelta ( seconds = older_than ) ,
status = ' created ' )
# notifications that are not to be resent
create_notification ( template = sms_template , created_at = datetime . utcnow ( ) - timedelta ( seconds = older_than ) ,
status = ' sending ' )
create_notification ( template = email_template , created_at = datetime . utcnow ( ) - timedelta ( seconds = older_than ) ,
status = ' delivered ' )
create_notification ( template = sms_template , created_at = datetime . utcnow ( ) ,
status = ' created ' )
create_notification ( template = email_template , created_at = datetime . utcnow ( ) ,
status = ' created ' )
replay_created_notifications ( )
email_delivery_queue . assert_called_once_with ( [ str ( old_email . id ) ] ,
queue = ' send-email-tasks ' )
sms_delivery_queue . assert_called_once_with ( [ str ( old_sms . id ) ] ,
queue = " send-sms-tasks " )
2019-01-14 17:22:41 +00:00
2020-05-06 18:23:56 +01:00
def test_replay_created_notifications_get_pdf_for_templated_letter_tasks_for_letters_not_ready_to_send (
2019-11-13 16:39:59 +00:00
sample_letter_template , mocker
) :
2020-05-06 18:23:56 +01:00
mock_task = mocker . patch ( ' app.celery.scheduled_tasks.get_pdf_for_templated_letter.apply_async ' )
2019-11-13 16:39:59 +00:00
create_notification ( template = sample_letter_template , billable_units = 0 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 4 ) )
2019-11-21 15:51:27 +00:00
create_notification ( template = sample_letter_template , billable_units = 0 ,
created_at = datetime . utcnow ( ) - timedelta ( minutes = 20 ) )
2019-11-13 16:39:59 +00:00
notification_1 = create_notification ( template = sample_letter_template , billable_units = 0 ,
2019-11-21 15:51:27 +00:00
created_at = datetime . utcnow ( ) - timedelta ( hours = 1 , minutes = 20 ) )
2019-11-13 16:39:59 +00:00
notification_2 = create_notification ( template = sample_letter_template , billable_units = 0 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 5 ) )
replay_created_notifications ( )
2020-04-22 16:06:03 +01:00
calls = [ call ( [ str ( notification_1 . id ) ] , queue = QueueNames . CREATE_LETTERS_PDF ) ,
call ( [ str ( notification_2 . id ) ] , queue = QueueNames . CREATE_LETTERS_PDF ) ,
2019-11-19 16:04:21 +00:00
]
mock_task . assert_has_calls ( calls , any_order = True )
2019-11-13 16:39:59 +00:00
2019-01-14 17:22:41 +00:00
def test_check_job_status_task_does_not_raise_error ( sample_template ) :
create_job (
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( hours = 2 ) ,
scheduled_for = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_FINISHED )
create_job (
template = sample_template ,
notification_count = 3 ,
created_at = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
processing_started = datetime . utcnow ( ) - timedelta ( minutes = 31 ) ,
job_status = JOB_STATUS_FINISHED )
check_job_status ( )
2019-06-11 13:16:34 +01:00
@freeze_time ( " 2019-05-30 14:00:00 " )
def test_check_precompiled_letter_state ( mocker , sample_letter_template ) :
mock_logger = mocker . patch ( ' app.celery.tasks.current_app.logger.exception ' )
2019-06-17 13:52:36 +01:00
mock_create_ticket = mocker . patch ( ' app.celery.nightly_tasks.zendesk_client.create_ticket ' )
2019-06-11 13:16:34 +01:00
create_notification ( template = sample_letter_template ,
status = NOTIFICATION_PENDING_VIRUS_CHECK ,
created_at = datetime . utcnow ( ) - timedelta ( seconds = 5400 ) )
create_notification ( template = sample_letter_template ,
status = NOTIFICATION_DELIVERED ,
created_at = datetime . utcnow ( ) - timedelta ( seconds = 6000 ) )
2020-09-24 14:16:16 +01:00
notification_1 = create_notification ( template = sample_letter_template ,
status = NOTIFICATION_PENDING_VIRUS_CHECK ,
created_at = datetime . utcnow ( ) - timedelta ( seconds = 5401 ) ,
reference = ' one ' )
notification_2 = create_notification ( template = sample_letter_template ,
status = NOTIFICATION_PENDING_VIRUS_CHECK ,
created_at = datetime . utcnow ( ) - timedelta ( seconds = 70000 ) ,
reference = ' two ' )
2019-06-11 13:16:34 +01:00
check_precompiled_letter_state ( )
2020-09-24 14:16:16 +01:00
id_references = sorted ( [ ( str ( notification_1 . id ) , notification_1 . reference ) ,
( str ( notification_2 . id ) , notification_2 . reference ) ] )
2020-03-30 12:07:05 +01:00
message = """ 2 precompiled letters have been pending-virus-check for over 90 minutes. Follow runbook to resolve:
https : / / github . com / alphagov / notifications - manuals / wiki / Support - Runbook #Deal-with-letter-pending-virus-scan-for-90-minutes.
2020-09-24 14:16:16 +01:00
Notifications : { } """ .format(id_references)
2019-06-17 13:52:36 +01:00
mock_logger . assert_called_once_with ( message )
mock_create_ticket . assert_called_with (
message = message ,
subject = ' [test] Letters still pending virus check ' ,
ticket_type = ' incident '
2019-06-11 13:16:34 +01:00
)
2019-06-11 15:13:06 +01:00
@freeze_time ( " 2019-05-30 14:00:00 " )
def test_check_templated_letter_state_during_bst ( mocker , sample_letter_template ) :
mock_logger = mocker . patch ( ' app.celery.tasks.current_app.logger.exception ' )
2019-06-17 13:52:36 +01:00
mock_create_ticket = mocker . patch ( ' app.celery.nightly_tasks.zendesk_client.create_ticket ' )
2019-06-11 15:13:06 +01:00
2019-11-13 16:39:59 +00:00
noti_1 = create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 5 , 1 , 12 , 0 ) )
noti_2 = create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 5 , 29 , 16 , 29 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 5 , 29 , 16 , 30 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 5 , 29 , 17 , 29 ) )
create_notification ( template = sample_letter_template , status = ' delivered ' , created_at = datetime ( 2019 , 5 , 28 , 10 , 0 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 5 , 30 , 10 , 0 ) )
2019-06-11 15:13:06 +01:00
check_templated_letter_state ( )
2019-06-17 13:52:36 +01:00
message = " 2 letters were created before 17.30 yesterday and still have ' created ' status. " \
" Notifications: [ ' {} ' , ' {} ' ] " . format ( noti_1 . id , noti_2 . id )
mock_logger . assert_called_once_with ( message )
mock_create_ticket . assert_called_with (
message = message ,
subject = " [test] Letters still in ' created ' status " ,
ticket_type = ' incident '
2019-06-11 15:13:06 +01:00
)
@freeze_time ( " 2019-01-30 14:00:00 " )
def test_check_templated_letter_state_during_utc ( mocker , sample_letter_template ) :
mock_logger = mocker . patch ( ' app.celery.tasks.current_app.logger.exception ' )
2019-11-13 16:39:59 +00:00
mock_create_ticket = mocker . patch ( ' app.celery.scheduled_tasks.zendesk_client.create_ticket ' )
noti_1 = create_notification ( template = sample_letter_template , created_at = datetime ( 2018 , 12 , 1 , 12 , 0 ) )
noti_2 = create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 1 , 29 , 17 , 29 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 1 , 29 , 17 , 30 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 1 , 29 , 18 , 29 ) )
create_notification ( template = sample_letter_template , status = ' delivered ' , created_at = datetime ( 2019 , 1 , 29 , 10 , 0 ) )
create_notification ( template = sample_letter_template , created_at = datetime ( 2019 , 1 , 30 , 10 , 0 ) )
2019-06-11 15:13:06 +01:00
check_templated_letter_state ( )
2019-06-17 13:52:36 +01:00
message = " 2 letters were created before 17.30 yesterday and still have ' created ' status. " \
" Notifications: [ ' {} ' , ' {} ' ] " . format ( noti_1 . id , noti_2 . id )
mock_logger . assert_called_once_with ( message )
mock_create_ticket . assert_called_with (
message = message ,
subject = " [test] Letters still in ' created ' status " ,
ticket_type = ' incident '
2019-06-11 15:13:06 +01:00
)
2019-11-05 16:47:00 +00:00
2020-09-26 11:06:44 +01:00
@pytest.mark.parametrize ( ' offset ' , (
timedelta ( days = 1 ) ,
pytest . param ( timedelta ( hours = 23 , minutes = 59 ) , marks = pytest . mark . xfail ) ,
pytest . param ( timedelta ( minutes = 20 ) , marks = pytest . mark . xfail ) ,
timedelta ( minutes = 19 ) ,
) )
def test_check_for_missing_rows_in_completed_jobs_ignores_old_and_new_jobs (
mocker ,
sample_email_template ,
offset ,
) :
mocker . patch ( ' app.celery.tasks.s3.get_job_and_metadata_from_s3 ' ,
return_value = ( load_example_csv ( ' multiple_email ' ) , { " sender_id " : None } ) )
mocker . patch ( ' app.encryption.encrypt ' , return_value = " something_encrypted " )
process_row = mocker . patch ( ' app.celery.scheduled_tasks.process_row ' )
job = create_job (
template = sample_email_template ,
notification_count = 5 ,
job_status = JOB_STATUS_FINISHED ,
processing_finished = datetime . utcnow ( ) - offset ,
)
for i in range ( 0 , 4 ) :
create_notification ( job = job , job_row_number = i )
check_for_missing_rows_in_completed_jobs ( )
assert process_row . called is False
2019-11-05 16:47:00 +00:00
def test_check_for_missing_rows_in_completed_jobs ( mocker , sample_email_template ) :
2019-11-15 15:32:16 +00:00
mocker . patch ( ' app.celery.tasks.s3.get_job_and_metadata_from_s3 ' ,
return_value = ( load_example_csv ( ' multiple_email ' ) , { " sender_id " : None } ) )
2019-11-05 16:47:00 +00:00
mocker . patch ( ' app.encryption.encrypt ' , return_value = " something_encrypted " )
process_row = mocker . patch ( ' app.celery.scheduled_tasks.process_row ' )
job = create_job ( template = sample_email_template ,
notification_count = 5 ,
job_status = JOB_STATUS_FINISHED ,
2020-09-26 11:06:44 +01:00
processing_finished = datetime . utcnow ( ) - timedelta ( minutes = 20 ) )
2019-11-05 16:47:00 +00:00
for i in range ( 0 , 4 ) :
create_notification ( job = job , job_row_number = i )
check_for_missing_rows_in_completed_jobs ( )
process_row . assert_called_once_with (
2019-11-15 15:32:16 +00:00
mock . ANY , mock . ANY , job , job . service , sender_id = None
2019-11-05 16:47:00 +00:00
)
def test_check_for_missing_rows_in_completed_jobs_calls_save_email ( mocker , sample_email_template ) :
2019-11-15 15:32:16 +00:00
mocker . patch ( ' app.celery.tasks.s3.get_job_and_metadata_from_s3 ' ,
return_value = ( load_example_csv ( ' multiple_email ' ) , { ' sender_id ' : None } ) )
2019-11-05 16:47:00 +00:00
save_email_task = mocker . patch ( ' app.celery.tasks.save_email.apply_async ' )
mocker . patch ( ' app.encryption.encrypt ' , return_value = " something_encrypted " )
mocker . patch ( ' app.celery.tasks.create_uuid ' , return_value = ' uuid ' )
job = create_job ( template = sample_email_template ,
notification_count = 5 ,
job_status = JOB_STATUS_FINISHED ,
2020-09-26 11:06:44 +01:00
processing_finished = datetime . utcnow ( ) - timedelta ( minutes = 20 ) )
2019-11-05 16:47:00 +00:00
for i in range ( 0 , 4 ) :
create_notification ( job = job , job_row_number = i )
check_for_missing_rows_in_completed_jobs ( )
save_email_task . assert_called_once_with (
(
str ( job . service_id ) ,
" uuid " ,
" something_encrypted " ,
) ,
{ } ,
queue = " database-tasks "
)
2019-11-15 15:32:16 +00:00
def test_check_for_missing_rows_in_completed_jobs_uses_sender_id ( mocker , sample_email_template , fake_uuid ) :
mocker . patch ( ' app.celery.tasks.s3.get_job_and_metadata_from_s3 ' ,
return_value = ( load_example_csv ( ' multiple_email ' ) , { ' sender_id ' : fake_uuid } ) )
mock_process_row = mocker . patch ( ' app.celery.scheduled_tasks.process_row ' )
job = create_job ( template = sample_email_template ,
notification_count = 5 ,
job_status = JOB_STATUS_FINISHED ,
2020-09-26 11:06:44 +01:00
processing_finished = datetime . utcnow ( ) - timedelta ( minutes = 20 ) )
2019-11-15 15:32:16 +00:00
for i in range ( 0 , 4 ) :
create_notification ( job = job , job_row_number = i )
check_for_missing_rows_in_completed_jobs ( )
mock_process_row . assert_called_once_with (
mock . ANY , mock . ANY , job , job . service , sender_id = fake_uuid
)
2019-11-29 21:24:17 +00:00
2019-12-03 10:26:59 +00:00
MockServicesSendingToTVNumbers = namedtuple (
' ServicesSendingToTVNumbers ' ,
[
' service_id ' ,
' notification_count ' ,
]
)
2019-12-05 16:07:06 +00:00
MockServicesWithHighFailureRate = namedtuple (
' ServicesWithHighFailureRate ' ,
[
' service_id ' ,
' permanent_failure_rate ' ,
]
)
2019-12-03 10:26:59 +00:00
@pytest.mark.parametrize ( " failure_rates, sms_to_tv_numbers, expected_message " , [
2019-11-29 21:24:17 +00:00
[
2019-12-05 16:07:06 +00:00
[ MockServicesWithHighFailureRate ( " 123 " , 0.3 ) ] ,
2019-12-03 10:26:59 +00:00
[ ] ,
2019-11-29 21:24:17 +00:00
" 1 service(s) have had high permanent-failure rates for sms messages in last "
2019-12-11 10:44:40 +00:00
" 24 hours: \n service: {} /services/ {} failure rate: 0.3, \n " . format (
Config . ADMIN_BASE_URL , " 123 "
2019-12-05 11:46:10 +00:00
)
2019-12-03 10:26:59 +00:00
] ,
[
[ ] ,
2019-12-04 14:40:24 +00:00
[ MockServicesSendingToTVNumbers ( " 123 " , 300 ) ] ,
2020-01-17 11:30:19 +00:00
" 1 service(s) have sent over 500 sms messages to tv numbers in last 24 hours: \n "
2019-12-11 10:44:40 +00:00
" service: {} /services/ {} count of sms to tv numbers: 300, \n " . format (
Config . ADMIN_BASE_URL , " 123 "
2019-12-05 11:46:10 +00:00
)
2019-11-29 21:24:17 +00:00
]
] )
def test_check_for_services_with_high_failure_rates_or_sending_to_tv_numbers (
2019-12-03 10:26:59 +00:00
mocker , notify_db_session , failure_rates , sms_to_tv_numbers , expected_message
2019-11-29 21:24:17 +00:00
) :
2019-12-17 11:47:23 +00:00
mock_logger = mocker . patch ( ' app.celery.tasks.current_app.logger.warning ' )
2019-11-29 21:24:17 +00:00
mock_create_ticket = mocker . patch ( ' app.celery.scheduled_tasks.zendesk_client.create_ticket ' )
mock_failure_rates = mocker . patch (
2019-12-05 16:07:06 +00:00
' app.celery.scheduled_tasks.dao_find_services_with_high_failure_rates ' , return_value = failure_rates
2019-11-29 21:24:17 +00:00
)
2019-12-03 10:26:59 +00:00
mock_sms_to_tv_numbers = mocker . patch (
' app.celery.scheduled_tasks.dao_find_services_sending_to_tv_numbers ' , return_value = sms_to_tv_numbers
)
2019-11-29 21:24:17 +00:00
2019-12-17 11:47:23 +00:00
zendesk_actions = " \n You can find instructions for this ticket in our manual: \n https://github.com/alphagov/notifications-manuals/wiki/Support-Runbook#Deal-with-services-with-high-failure-rates-or-sending-sms-to-tv-numbers " # noqa
2019-12-03 16:18:07 +00:00
2019-11-29 21:24:17 +00:00
check_for_services_with_high_failure_rates_or_sending_to_tv_numbers ( )
assert mock_failure_rates . called
2019-12-03 10:26:59 +00:00
assert mock_sms_to_tv_numbers . called
2019-11-29 21:24:17 +00:00
mock_logger . assert_called_once_with ( expected_message )
mock_create_ticket . assert_called_with (
2019-12-03 16:18:07 +00:00
message = expected_message + zendesk_actions ,
2019-11-29 21:24:17 +00:00
subject = " [test] High failure rates for sms spotted for services " ,
ticket_type = ' incident '
)
2020-10-26 17:34:59 +00:00
def test_send_canary_to_cbc_proxy_invokes_cbc_proxy_client (
mocker ,
2020-11-17 12:35:22 +00:00
notify_api
2020-10-26 17:34:59 +00:00
) :
mock_send_canary = mocker . patch (
2020-11-17 12:35:22 +00:00
' app.clients.cbc_proxy.CBCProxyCanary.send_canary ' ,
2020-10-26 17:34:59 +00:00
)
scheduled_tasks . send_canary_to_cbc_proxy ( )
2020-11-25 17:39:32 +00:00
assert mock_send_canary . called is True
2020-10-26 17:34:59 +00:00
# the 0th argument of the call to send_canary
identifier = mock_send_canary . mock_calls [ 0 ] [ 1 ] [ 0 ]
try :
uuid . UUID ( identifier )
except BaseException :
pytest . fail ( f " { identifier } is not a valid uuid " )
2020-10-27 15:39:28 +00:00
2020-11-25 17:39:32 +00:00
def test_send_canary_to_cbc_proxy_does_nothing_if_cbc_proxy_disabled (
mocker , notify_api
) :
mock_send_canary = mocker . patch (
' app.clients.cbc_proxy.CBCProxyCanary.send_canary ' ,
)
with set_config ( notify_api , ' CBC_PROXY_ENABLED ' , False ) :
scheduled_tasks . send_canary_to_cbc_proxy ( )
assert mock_send_canary . called is False
2020-11-16 12:47:38 +00:00
def test_trigger_link_tests_calls_for_all_providers (
mocker , notify_api
2020-10-27 15:39:28 +00:00
) :
2020-11-16 12:47:38 +00:00
mock_trigger_link_test = mocker . patch (
' app.celery.scheduled_tasks.trigger_link_test ' ,
2020-10-27 15:39:28 +00:00
)
2020-11-16 12:47:38 +00:00
with set_config ( notify_api , ' ENABLED_CBCS ' , [ ' ee ' , ' vodafone ' ] ) :
trigger_link_tests ( )
2020-10-27 15:39:28 +00:00
2020-11-16 12:47:38 +00:00
assert mock_trigger_link_test . apply_async . call_args_list == [
2021-01-13 17:21:40 +00:00
call ( kwargs = { ' provider ' : ' ee ' } , queue = ' broadcast-tasks ' ) ,
call ( kwargs = { ' provider ' : ' vodafone ' } , queue = ' broadcast-tasks ' )
2020-11-16 12:47:38 +00:00
]
2020-11-25 17:39:32 +00:00
def test_trigger_link_does_nothing_if_cbc_proxy_disabled (
mocker , notify_api
) :
mock_trigger_link_test = mocker . patch (
' app.celery.scheduled_tasks.trigger_link_test ' ,
)
with set_config ( notify_api , ' ENABLED_CBCS ' , [ ' ee ' , ' vodafone ' ] ) , set_config ( notify_api , ' CBC_PROXY_ENABLED ' , False ) :
trigger_link_tests ( )
assert mock_trigger_link_test . called is False