Merge pull request #824 from alphagov/feat-switch-providers-on-slow-delivery

Auto-switch providers on slow delivery of notifications

app/celery/scheduled_tasks.py

@@ -1,4 +1,7 @@
-from datetime import datetime
+from datetime import (
+    datetime,
+    timedelta
+)
 
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
@@ -10,7 +13,12 @@ from app.dao.invited_user_dao import delete_invitations_created_more_than_two_da
 from app.dao.jobs_dao import dao_set_scheduled_jobs_to_pending, dao_get_jobs_older_than
 from app.dao.notifications_dao import (
     delete_notifications_created_more_than_a_week_ago,
-    dao_timeout_notifications
+    dao_timeout_notifications,
+    is_delivery_slow_for_provider
+)
+from app.dao.provider_details_dao import (
+    get_current_provider,
+    dao_toggle_sms_provider
 )
 from app.dao.users_dao import delete_codes_older_created_more_than_a_day_ago
 from app.statsd_decorators import statsd
@@ -141,3 +149,34 @@ def send_daily_performance_platform_stats():
             email_sent_count,
             'day'
         )
+
+
+@notify_celery.task(name='switch-current-sms-provider-on-slow-delivery')
+@statsd(namespace="tasks")
+def switch_current_sms_provider_on_slow_delivery():
+    """
+    Switch providers if there are at least two slow delivery notifications (more than four minutes)
+    in the last ten minutes. Search from the time we last switched to the current provider.
+    """
+    functional_test_provider_service_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SERVICE_ID')
+    functional_test_provider_template_id = current_app.config.get('FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID')
+
+    if functional_test_provider_service_id and functional_test_provider_template_id:
+        current_provider = get_current_provider('sms')
+        slow_delivery_notifications = is_delivery_slow_for_provider(
+            provider=current_provider.identifier,
+            threshold=2,
+            sent_at=max(datetime.utcnow() - timedelta(minutes=10), current_provider.updated_at),
+            delivery_time=timedelta(minutes=4),
+            service_id=functional_test_provider_service_id,
+            template_id=functional_test_provider_template_id
+        )
+
+        if slow_delivery_notifications:
+            current_app.logger.warning(
+                'Slow delivery notifications detected for provider {}'.format(
+                    current_provider.identifier
+                )
+            )
+
+            dao_toggle_sms_provider(current_provider.identifier)

app/config.py

@@ -131,6 +131,11 @@ class Config(object):
             'schedule': crontab(minute=30, hour=0),  # 00:30
             'options': {'queue': 'periodic'}
         },
+        'switch-current-sms-provider-on-slow-delivery': {
+            'task': 'switch-current-sms-provider-on-slow-delivery',
+            'schedule': crontab(),  # Every minute
+            'options': {'queue': 'periodic'}
+        },
         'timeout-sending-notifications': {
             'task': 'timeout-sending-notifications',
             'schedule': crontab(minute=0, hour='0,1,2'),
@@ -166,6 +171,9 @@ class Config(object):
 
     SIMULATED_SMS_NUMBERS = ('+447700900000', '+447700900111', '+447700900222')
 
+    FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = None
+    FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = None
+
 
 ######################
 # Config overrides ###
@@ -231,6 +239,8 @@ class Live(Config):
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     STATSD_ENABLED = True
     FROM_NUMBER = '40604'
+    FUNCTIONAL_TEST_PROVIDER_SERVICE_ID = '6c1d81bb-dae2-4ee9-80b0-89a4aae9f649'
+    FUNCTIONAL_TEST_PROVIDER_SMS_TEMPLATE_ID = 'ba9e1789-a804-40b8-871f-cc60d4c1286f'
 
 
 class CloudFoundryConfig(Config):

app/dao/notifications_dao.py

@@ -18,6 +18,7 @@ from app.models import (
     NotificationStatistics,
     Template,
     NOTIFICATION_CREATED,
+    NOTIFICATION_DELIVERED,
     NOTIFICATION_SENDING,
     NOTIFICATION_PENDING,
     NOTIFICATION_TECHNICAL_FAILURE,
@@ -421,3 +422,22 @@ def get_total_sent_notifications_in_date_range(start_date, end_date, notificatio
     ).scalar()
 
     return result or 0
+
+
+def is_delivery_slow_for_provider(
+    sent_at,
+    provider,
+    threshold,
+    delivery_time,
+    service_id,
+    template_id
+):
+    count = db.session.query(Notification).filter(
+        Notification.service_id == service_id,
+        Notification.template_id == template_id,
+        Notification.sent_at >= sent_at,
+        Notification.status == NOTIFICATION_DELIVERED,
+        Notification.sent_by == provider,
+        (Notification.updated_at - Notification.sent_at) >= delivery_time,
+    ).count()
+    return count >= threshold

app/dao/services_dao.py

@@ -110,11 +110,11 @@ def dao_fetch_service_by_id_and_user(service_id, user_id):
 
 @transactional
 @version_class(Service)
-def dao_create_service(service, user):
+def dao_create_service(service, user, service_id=None):
     from app.dao.permissions_dao import permission_dao
     service.users.append(user)
     permission_dao.add_default_service_permissions_for_user(user, service)
-    service.id = uuid.uuid4()  # must be set now so version history model can use same id
+    service.id = service_id or uuid.uuid4()  # must be set now so version history model can use same id
     service.active = True
     service.research_mode = False
     db.session.add(service)

app/dao/templates_dao.py

@@ -13,8 +13,8 @@ from app.dao.dao_utils import (
 
 @transactional
 @version_class(Template, TemplateHistory)
-def dao_create_template(template):
-    template.id = uuid.uuid4()  # must be set now so version history model can use same id
+def dao_create_template(template, template_id=None):
+    template.id = template_id or uuid.uuid4()  # must be set now so version history model can use same id
     template.archived = False
     db.session.add(template)