Merge pull request #795 from alphagov/switch-providers-on-error

Auto-switch providers on error

app/dao/provider_details_dao.py

@@ -1,7 +1,12 @@
 from datetime import datetime
 
 from sqlalchemy import asc
+
 from app.dao.dao_utils import transactional
+from app.provider_details.switch_providers import (
+    provider_is_already_primary_or_inactive,
+    update_provider_priorities
+)
 from app.models import ProviderDetails, ProviderDetailsHistory
 from app import db
 
@@ -14,6 +19,49 @@ def get_provider_details_by_id(provider_details_id):
     return ProviderDetails.query.get(provider_details_id)
 
 
+def get_provider_details_by_identifier(identifier):
+    return ProviderDetails.query.filter_by(identifier=identifier).one()
+
+
+def get_alternative_sms_provider(identifier):
+    alternate_provider = None
+    if identifier == 'firetext':
+        alternate_provider = 'mmg'
+    elif identifier == 'mmg':
+        alternate_provider = 'firetext'
+
+    return ProviderDetails.query.filter_by(identifier=alternate_provider).one()
+
+
+def get_current_provider(notification_type):
+    return ProviderDetails.query.filter_by(
+        notification_type=notification_type
+    ).order_by(
+        asc(ProviderDetails.priority)
+    ).first()
+
+
+@transactional
+def dao_toggle_sms_provider(identifier):
+    alternate_provider = get_alternative_sms_provider(identifier)
+    dao_switch_sms_provider_to_provider_with_identifier(alternate_provider.identifier)
+
+
+@transactional
+def dao_switch_sms_provider_to_provider_with_identifier(identifier):
+    current_provider = get_current_provider('sms')
+    new_provider = get_provider_details_by_identifier(identifier)
+
+    if current_provider.priority == new_provider.priority:
+        # Since both priorities are equal, set the current provider
+        # to the one that we want to switch from
+        current_provider = get_alternative_sms_provider(identifier)
+
+    if not provider_is_already_primary_or_inactive(current_provider, new_provider, identifier):
+        update_provider_priorities(current_provider, new_provider)
+        db.session.add_all([current_provider, new_provider])
+
+
 def get_provider_details_by_notification_type(notification_type):
     return ProviderDetails.query.filter_by(
         notification_type=notification_type

app/delivery/send_to_providers.py

@@ -9,11 +9,13 @@ from notifications_utils.template import HTMLEmailTemplate, PlainTextEmailTempla
 
 from app import clients, statsd_client, create_uuid
 from app.dao.notifications_dao import dao_update_notification
-from app.dao.provider_details_dao import get_provider_details_by_notification_type
+from app.dao.provider_details_dao import (
+    get_provider_details_by_notification_type,
+    dao_toggle_sms_provider
+)
 from app.dao.services_dao import dao_fetch_service_by_id
 from app.celery.research_mode_tasks import send_sms_response, send_email_response
 from app.dao.templates_dao import dao_get_template_by_id
-
 from app.models import SMS_TYPE, KEY_TYPE_TEST, BRANDING_ORG, EMAIL_TYPE
 
 
@@ -37,13 +39,18 @@ def send_sms_to_provider(notification):
             )
             notification.billable_units = 0
         else:
-            provider.send_sms(
-                to=validate_and_format_phone_number(notification.to),
-                content=str(template),
-                reference=str(notification.id),
-                sender=service.sms_sender
-            )
-            notification.billable_units = template.fragment_count
+            try:
+                provider.send_sms(
+                    to=validate_and_format_phone_number(notification.to),
+                    content=str(template),
+                    reference=str(notification.id),
+                    sender=service.sms_sender
+                )
+            except Exception as e:
+                dao_toggle_sms_provider(provider.name)
+                raise e
+            else:
+                notification.billable_units = template.fragment_count
 
         notification.sent_at = datetime.utcnow()
         notification.sent_by = provider.get_name()

app/provider_details/switch_providers.py

@@ -0,0 +1,46 @@
+from flask import current_app
+
+
+def provider_is_already_primary_or_inactive(current_provider, new_provider, identifier):
+    if current_provider.identifier == identifier:
+        current_app.logger.warning('Provider {} is already activated'.format(current_provider.display_name))
+        return True
+
+    elif not new_provider.active:
+        current_app.logger.warning('Cancelling switch from {} to {} as {} is inactive'.format(
+            current_provider.identifier,
+            new_provider.identifier,
+            new_provider.identifier
+        ))
+        return True
+
+    return False
+
+
+def update_provider_priorities(current_provider, new_provider):
+    # Swap priority to change primary provider
+    if new_provider.priority > current_provider.priority:
+        new_provider.priority, current_provider.priority = current_provider.priority, new_provider.priority
+
+    # Incease other provider priority if equal
+    elif new_provider.priority == current_provider.priority:
+        current_provider.priority += 10
+
+    _print_provider_switch_logs(current_provider, new_provider)
+
+
+def _print_provider_switch_logs(current_provider, new_provider):
+    current_app.logger.warning('Switching provider from {} to {}'.format(
+        current_provider.identifier,
+        new_provider.identifier
+    ))
+
+    current_app.logger.warning('Provider {} now updated with priority of {}'.format(
+        current_provider.identifier,
+        current_provider.priority
+    ))
+
+    current_app.logger.warning('Provider {} now updated with priority of {}'.format(
+        new_provider.identifier,
+        new_provider.priority
+    ))