diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 921f12fdc..1afe1aed9 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -1,4 +1,7 @@ -from datetime import datetime +from datetime import ( + datetime, + timedelta +) from flask import current_app from sqlalchemy.exc import SQLAlchemyError @@ -10,7 +13,12 @@ from app.dao.invited_user_dao import delete_invitations_created_more_than_two_da from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than from app.dao.notifications_dao import ( delete_notifications_created_more_than_a_week_ago, - dao_timeout_notifications + dao_timeout_notifications, + get_count_of_slow_delivery_sms_notifications_for_provider +) +from app.dao.provider_details_dao import ( + get_current_provider, + dao_toggle_sms_provider ) from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago from app.statsd_decorators import statsd @@ -141,3 +149,36 @@ def send_daily_performance_platform_stats(): email_sent_count, 'day' ) + + +@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery') +@statsd(namespace="tasks") +def switch_current_sms_provider_on_slow_delivery(): + """ + Switch providers if there are at least two slow delivery notifications (more than four minutes) + in the last ten minutes. Search from the time we last switched to the current provider. + """ + functional_test_provider_service_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SERVICE_ID') + functional_test_provider_template_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID') + + if functional_test_provider_service_id and functional_test_provider_template_id: + current_provider = get_current_provider('sms') + slow_delivery_notifications = get_count_of_slow_delivery_sms_notifications_for_provider( + provider=current_provider.identifier, + threshold=2, + created_at=current_provider.updated_at, + sent_at=datetime.utcnow() - timedelta(minutes=10), + delivery_time=timedelta(minutes=4), + service_id=functional_test_provider_service_id, + template_id=functional_test_provider_template_id + ) + + if slow_delivery_notifications: + current_app.logger.warning( + '{} slow delivery notifications detected for provider {}'.format( + slow_delivery_notifications.total, + slow_delivery_notifications.sent_by + ) + ) + + dao_toggle_sms_provider(current_provider.identifier) diff --git a/app/config.py b/app/config.py index a5e7c8240..303e18943 100644 --- a/app/config.py +++ b/app/config.py @@ -131,6 +131,11 @@ class Config(object): 'schedule': crontab(minute=30, hour=0), # 00:30 'options': {'queue': 'periodic'} }, + 'switch-current-sms-provider-on-slow-delivery': { + 'task': 'switch-current-sms-provider-on-slow-delivery', + 'schedule': crontab(), # Every minute + 'options': {'queue': 'periodic'} + }, 'timeout-sending-notifications': { 'task': 'timeout-sending-notifications', 'schedule': crontab(minute=0, hour='0,1,2'),