Merge branch 'master' into add-template-version

Conflicts:
	tests/app/dao/test_notification_dao.py

app/__init__.py

@@ -1,6 +1,5 @@
 import uuid
 import os
-import re
 from flask import request, url_for
 from flask import Flask, _request_ctx_stack
 from flask.ext.sqlalchemy import SQLAlchemy
@@ -8,6 +7,7 @@ from flask_marshmallow import Marshmallow
 from werkzeug.local import LocalProxy
 from notifications_utils import logging
 from app.celery.celery import NotifyCelery
+from app.clients import Clients
 from app.clients.sms.mmg import MMGClient
 from app.clients.sms.twilio import TwilioClient
 from app.clients.sms.firetext import FiretextClient
@@ -26,6 +26,8 @@ mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 encryption = Encryption()
 
+clients = Clients()
+
 api_user = LocalProxy(lambda: _request_ctx_stack.top.api_user)
 
 
@@ -48,6 +50,7 @@ def create_app(app_name=None):
     aws_ses_client.init_app(application.config['AWS_REGION'])
     notify_celery.init_app(application)
     encryption.init_app(application)
+    clients.init_app(sms_clients=[firetext_client, mmg_client], email_clients=[aws_ses_client])
 
     from app.service.rest import service as service_blueprint
     from app.user.rest import user as user_blueprint
@@ -61,6 +64,7 @@ def create_app(app_name=None):
     from app.notifications_statistics.rest import notifications_statistics as notifications_statistics_blueprint
     from app.template_statistics.rest import template_statistics as template_statistics_blueprint
     from app.events.rest import events as events_blueprint
+    from app.provider_details.rest import provider_details as provider_details_blueprint
 
     application.register_blueprint(service_blueprint, url_prefix='/service')
     application.register_blueprint(user_blueprint, url_prefix='/user')
@@ -74,6 +78,7 @@ def create_app(app_name=None):
     application.register_blueprint(notifications_statistics_blueprint)
     application.register_blueprint(template_statistics_blueprint)
     application.register_blueprint(events_blueprint)
+    application.register_blueprint(provider_details_blueprint, url_prefix='/provider-details')
 
     return application
 
@@ -86,7 +91,7 @@ def init_app(app):
             url_for('notifications.process_ses_response'),
             url_for('notifications.process_firetext_response'),
             url_for('notifications.process_mmg_response'),
-            url_for('status.show_delivery_status'),
+            url_for('status.show_delivery_status')
         ]
         if request.path not in no_auth_req:
             from app.authentication import auth

app/celery/tasks.py

@@ -3,12 +3,12 @@ from datetime import datetime
 
 from flask import current_app
 from sqlalchemy.exc import SQLAlchemyError
-
-from app.clients.email.aws_ses import AwsSesClientException
-from app.clients.sms.firetext import FiretextClientException
-from app.clients.sms.mmg import MMGClientException
+from app import clients
+from app.clients.email import EmailClientException
+from app.clients.sms import SmsClientException
 from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
+from app.dao.provider_details_dao import get_provider_details_by_notification_type
 
 from notifications_utils.template import Template, unlink_govuk_escaped
 
@@ -23,10 +23,7 @@ from app import (
     DATETIME_FORMAT,
     DATE_FORMAT,
     notify_celery,
-    encryption,
-    firetext_client,
-    aws_ses_client,
-    mmg_client
+    encryption
 )
 
 from app.aws import s3
@@ -160,7 +157,7 @@ def process_job(job_id):
             'personalisation': {
                 key: personalisation.get(key)
                 for key in template.placeholders
-            }
+                }
         })
 
         if template.template_type == 'sms':
@@ -207,7 +204,8 @@ def remove_job(job_id):
 def send_sms(service_id, notification_id, encrypted_notification, created_at):
     notification = encryption.decrypt(encrypted_notification)
     service = dao_fetch_service_by_id(service_id)
-    client = mmg_client
+
+    provider = provider_to_use('sms', notification_id)
 
     restricted = False
 
@@ -236,23 +234,23 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
             status='failed' if restricted else 'sending',
             created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
-            sent_by=client.get_name(),
+            sent_by=provider.get_name(),
             content_char_count=template.replaced_content_count
         )
 
-        dao_create_notification(notification_db_object, TEMPLATE_TYPE_SMS, client.get_name())
+        dao_create_notification(notification_db_object, TEMPLATE_TYPE_SMS, provider.get_name())
 
         if restricted:
             return
 
         try:
-            client.send_sms(
+            provider.send_sms(
                 to=validate_and_format_phone_number(notification['to']),
                 content=template.replaced,
                 reference=str(notification_id)
             )
 
-        except MMGClientException as e:
+        except SmsClientException as e:
             current_app.logger.error(
                 "SMS notification {} failed".format(notification_id)
             )
@@ -270,9 +268,10 @@ def send_sms(service_id, notification_id, encrypted_notification, created_at):
 @notify_celery.task(name="send-email")
 def send_email(service_id, notification_id, from_address, encrypted_notification, created_at):
     notification = encryption.decrypt(encrypted_notification)
-    client = aws_ses_client
     service = dao_fetch_service_by_id(service_id)
 
+    provider = provider_to_use('email', notification_id)
+
     restricted = False
 
     if not service_allowed_to_send_to(notification['to'], service):
@@ -292,10 +291,10 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
             status='failed' if restricted else 'sending',
             created_at=datetime.strptime(created_at, DATETIME_FORMAT),
             sent_at=sent_at,
-            sent_by=client.get_name()
+            sent_by=provider.get_name()
         )
 
-        dao_create_notification(notification_db_object, TEMPLATE_TYPE_EMAIL, client.get_name())
+        dao_create_notification(notification_db_object, TEMPLATE_TYPE_EMAIL, provider.get_name())
 
         if restricted:
             return
@@ -305,7 +304,7 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
                 dao_get_template_by_id(notification['template']).__dict__,
                 values=notification.get('personalisation', {})
             )
-            reference = client.send_email(
+            reference = provider.send_email(
                 from_address,
                 notification['to'],
                 template.replaced_subject,
@@ -313,7 +312,7 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
                 html_body=template.as_HTML_email,
             )
             update_notification_reference_by_id(notification_id, reference)
-        except AwsSesClientException as e:
+        except EmailClientException as e:
             current_app.logger.exception(e)
             notification_db_object.status = 'failed'
 
@@ -327,20 +326,15 @@ def send_email(service_id, notification_id, from_address, encrypted_notification
 
 @notify_celery.task(name='send-sms-code')
 def send_sms_code(encrypted_verification):
+    provider = provider_to_use('sms', 'send-sms-code')
+
     verification_message = encryption.decrypt(encrypted_verification)
     try:
-        mmg_client.send_sms(validate_and_format_phone_number(verification_message['to']),
-                            "{} is your Notify authentication code".format(
-                                verification_message['secret_code']),
-                            'send-sms-code')
-    except MMGClientException as e:
-        current_app.logger.exception(e)
-
-
-def send_sms_via_firetext(to, content, reference):
-    try:
-        firetext_client.send_sms(to=to, content=content, reference=reference)
-    except FiretextClientException as e:
+        provider.send_sms(validate_and_format_phone_number(verification_message['to']),
+                          "{} is your Notify authentication code".format(
+                              verification_message['secret_code']),
+                          'send-sms-code')
+    except SmsClientException as e:
         current_app.logger.exception(e)
 
 
@@ -371,6 +365,8 @@ def invited_user_url(base_url, token):
 
 @notify_celery.task(name='email-invited-user')
 def email_invited_user(encrypted_invitation):
+    provider = provider_to_use('email', 'email-invited-user')
+
     invitation = encryption.decrypt(encrypted_invitation)
     url = invited_user_url(current_app.config['ADMIN_BASE_URL'],
                            invitation['token'])
@@ -384,11 +380,11 @@ def email_invited_user(encrypted_invitation):
             current_app.config['NOTIFY_EMAIL_DOMAIN']
         )
         subject_line = invitation_subject_line(invitation['user_name'], invitation['service_name'])
-        aws_ses_client.send_email(email_from,
-                                  invitation['to'],
-                                  subject_line,
-                                  invitation_content)
-    except AwsSesClientException as e:
+        provider.send_email(email_from,
+                            invitation['to'],
+                            subject_line,
+                            invitation_content)
+    except EmailClientException as e:
         current_app.logger.exception(e)
 
 
@@ -406,17 +402,23 @@ def password_reset_message(name, url):
 
 @notify_celery.task(name='email-reset-password')
 def email_reset_password(encrypted_reset_password_message):
+    provider = provider_to_use('email', 'email-reset-password')
+
     reset_password_message = encryption.decrypt(encrypted_reset_password_message)
     try:
         email_from = '"GOV.UK Notify" <{}>'.format(
             current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS']
         )
-        aws_ses_client.send_email(email_from,
-                                  reset_password_message['to'],
-                                  "Reset your GOV.UK Notify password",
-                                  password_reset_message(name=reset_password_message['name'],
-                                                         url=reset_password_message['reset_password_url']))
-    except AwsSesClientException as e:
+        provider.send_email(
+            email_from,
+            reset_password_message['to'],
+            "Reset your GOV.UK Notify password",
+            password_reset_message(
+                name=reset_password_message['name'],
+                url=reset_password_message['reset_password_url']
+            )
+        )
+    except EmailClientException as e:
         current_app.logger.exception(e)
 
 
@@ -431,22 +433,26 @@ def registration_verification_template(name, url):
 
 @notify_celery.task(name='email-registration-verification')
 def email_registration_verification(encrypted_verification_message):
+    provider = provider_to_use('email', 'email-reset-password')
+
     verification_message = encryption.decrypt(encrypted_verification_message)
     try:
         email_from = '"GOV.UK Notify" <{}>'.format(
             current_app.config['VERIFY_CODE_FROM_EMAIL_ADDRESS']
         )
-        aws_ses_client.send_email(email_from,
-                                  verification_message['to'],
-                                  "Confirm GOV.UK Notify registration",
-                                  registration_verification_template(name=verification_message['name'],
-                                                                     url=verification_message['url']))
-    except AwsSesClientException as e:
+        provider.send_email(
+            email_from,
+            verification_message['to'],
+            "Confirm GOV.UK Notify registration",
+            registration_verification_template(
+                name=verification_message['name'],
+                url=verification_message['url'])
+        )
+    except EmailClientException as e:
         current_app.logger.exception(e)
 
 
 def service_allowed_to_send_to(recipient, service):
-
     if not service.restricted:
         return True
 
@@ -456,3 +462,17 @@ def service_allowed_to_send_to(recipient, service):
             [user.mobile_number, user.email_address] for user in service.users
         )
     )
+
+
+def provider_to_use(notification_type, notification_id):
+    active_providers_in_order = [
+        provider for provider in get_provider_details_by_notification_type(notification_type) if provider.active
+    ]
+
+    if not active_providers_in_order:
+        current_app.logger.error(
+            "{} {} failed as no active providers".format(notification_type, notification_id)
+        )
+        raise Exception("No active {} providers".format(notification_type))
+
+    return clients.get_client_by_name_and_type(active_providers_in_order[0].identifier, notification_type)

app/clients/__init__.py

@@ -1,4 +1,3 @@
-
 class ClientException(Exception):
     '''
     Base Exceptions for sending notifications that fail
@@ -16,3 +15,30 @@ class Client(object):
 STATISTICS_REQUESTED = 'requested'
 STATISTICS_DELIVERED = 'delivered'
 STATISTICS_FAILURE = 'failure'
+
+
+class Clients(object):
+    sms_clients = {}
+    email_clients = {}
+
+    def init_app(self, sms_clients, email_clients):
+        for client in sms_clients:
+            self.sms_clients[client.name] = client
+
+        for client in email_clients:
+            self.email_clients[client.name] = client
+
+    def get_sms_client(self, name):
+        return self.sms_clients.get(name)
+
+    def get_email_client(self, name):
+        return self.email_clients.get(name)
+
+    def get_client_by_name_and_type(self, name, notification_type):
+        assert notification_type in ['email', 'sms']
+
+        if notification_type == 'email':
+            return self.get_email_client(name)
+
+        if notification_type == 'sms':
+            return self.get_sms_client(name)

app/dao/notifications_dao.py

@@ -1,4 +1,3 @@
-import math
 from sqlalchemy import (desc, func, Integer)
 from sqlalchemy.sql.expression import cast
 
@@ -20,8 +19,8 @@ from app.models import (
     TEMPLATE_TYPE_SMS,
     TEMPLATE_TYPE_EMAIL,
     Template,
-    ProviderStatistics
-)
+    ProviderStatistics,
+    ProviderDetails)
 
 from notifications_utils.template import get_sms_fragment_count
 
@@ -88,7 +87,9 @@ def dao_get_template_statistics_for_service(service_id, limit_days=None):
 
 
 @transactional
-def dao_create_notification(notification, notification_type, provider):
+def dao_create_notification(notification, notification_type, provider_identifier):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
     if notification.job_id:
         db.session.query(Job).filter_by(
             id=notification.job_id
@@ -125,7 +126,7 @@ def dao_create_notification(notification, notification_type, provider):
     update_count = db.session.query(ProviderStatistics).filter_by(
         day=date.today(),
         service_id=notification.service_id,
-        provider=provider
+        provider_id=provider.id
     ).update({'unit_count': ProviderStatistics.unit_count + (
         1 if notification_type == TEMPLATE_TYPE_EMAIL else get_sms_fragment_count(notification.content_char_count))})
 
@@ -133,7 +134,7 @@ def dao_create_notification(notification, notification_type, provider):
         provider_stats = ProviderStatistics(
             day=notification.created_at.date(),
             service_id=notification.service_id,
-            provider=provider,
+            provider_id=provider.id,
             unit_count=1 if notification_type == TEMPLATE_TYPE_EMAIL else get_sms_fragment_count(
                 notification.content_char_count))
         db.session.add(provider_stats)

app/dao/provider_details_dao.py

@@ -0,0 +1,23 @@
+from sqlalchemy import asc
+from app.dao.dao_utils import transactional
+from app.models import ProviderDetails
+from app import db
+
+
+def get_provider_details():
+    return ProviderDetails.query.order_by(asc(ProviderDetails.priority), asc(ProviderDetails.notification_type)).all()
+
+
+def get_provider_details_by_id(provider_details_id):
+    return ProviderDetails.query.get(provider_details_id)
+
+
+def get_provider_details_by_notification_type(notification_type):
+    return ProviderDetails.query.filter_by(
+        notification_type=notification_type
+    ).order_by(asc(ProviderDetails.priority)).all()
+
+
+@transactional
+def dao_update_provider_details(provider_details):
+    db.session.add(provider_details)

app/dao/provider_rates_dao.py

@@ -1,9 +1,11 @@
-from app.models import ProviderRates
+from app.models import ProviderRates, ProviderDetails
 from app import db
 from app.dao.dao_utils import transactional
 
 
 @transactional
-def create_provider_rates(provider, valid_from, rate):
-    provider_rates = ProviderRates(provider=provider, valid_from=valid_from, rate=rate)
+def create_provider_rates(provider_identifier, valid_from, rate):
+    provider = ProviderDetails.query.filter_by(identifier=provider_identifier).one()
+
+    provider_rates = ProviderRates(provider_id=provider.id, valid_from=valid_from, rate=rate)
     db.session.add(provider_rates)

app/dao/provider_statistics_dao.py

@@ -1,6 +1,5 @@
 from sqlalchemy import func
-from app import db
-from app.models import (ProviderStatistics, SMS_PROVIDERS, EMAIL_PROVIDERS)
+from app.models import (ProviderStatistics, SMS_PROVIDERS, EMAIL_PROVIDERS, ProviderDetails)
 
 
 def get_provider_statistics(service, **kwargs):
@@ -33,7 +32,9 @@ def get_fragment_count(service, date_from, date_to):
 def filter_query(query, service, **kwargs):
     query = query.filter_by(service=service)
     if 'providers' in kwargs:
-        query = query.filter(ProviderStatistics.provider.in_(kwargs['providers']))
+        providers = ProviderDetails.query.filter(ProviderDetails.identifier.in_(kwargs['providers'])).all()
+        provider_ids = [provider.id for provider in providers]
+        query = query.filter(ProviderStatistics.provider_id.in_(provider_ids))
     if 'date_from' in kwargs:
         query.filter(ProviderStatistics.day >= kwargs['date_from'])
     if 'date_to' in kwargs:

app/models.py

@@ -191,13 +191,18 @@ SMS_PROVIDERS = [MMG_PROVIDER, TWILIO_PROVIDER, FIRETEXT_PROVIDER]
 EMAIL_PROVIDERS = [SES_PROVIDER]
 PROVIDERS = SMS_PROVIDERS + EMAIL_PROVIDERS
 
+NOTIFICATION_TYPE = ['email', 'sms', 'letter']
+
 
 class ProviderStatistics(db.Model):
     __tablename__ = 'provider_statistics'
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     day = db.Column(db.Date, nullable=False)
-    provider = db.Column(db.Enum(*PROVIDERS, name='providers'), nullable=False)
+    provider_id = db.Column(UUID(as_uuid=True), db.ForeignKey('provider_details.id'), index=True, nullable=False)
+    provider = db.relationship(
+        'ProviderDetails', backref=db.backref('provider_stats', lazy='dynamic')
+    )
     service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
     service = db.relationship('Service', backref=db.backref('service_provider_stats', lazy='dynamic'))
     unit_count = db.Column(db.BigInteger, nullable=False)
@@ -208,8 +213,20 @@ class ProviderRates(db.Model):
 
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     valid_from = db.Column(db.DateTime, nullable=False)
-    provider = db.Column(db.Enum(*PROVIDERS, name='providers'), nullable=False)
     rate = db.Column(db.Numeric(), nullable=False)
+    provider_id = db.Column(UUID(as_uuid=True), db.ForeignKey('provider_details.id'), index=True, nullable=False)
+    provider = db.relationship('ProviderDetails', backref=db.backref('provider_rates', lazy='dynamic'))
+
+
+class ProviderDetails(db.Model):
+    __tablename__ = 'provider_details'
+
+    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    display_name = db.Column(db.String, nullable=False)
+    identifier = db.Column(db.String, nullable=False)
+    priority = db.Column(db.Integer, nullable=False)
+    notification_type = db.Column(db.Enum(*NOTIFICATION_TYPE, name='notification_type'), nullable=False)
+    active = db.Column(db.Boolean, default=False)
 
 
 JOB_STATUS_TYPES = ['pending', 'in progress', 'finished', 'sending limits exceeded']

app/provider_details/rest.py

@@ -0,0 +1,42 @@
+from flask import Blueprint, jsonify, request
+
+from app.schemas import provider_details_schema
+from app.dao.provider_details_dao import (
+    get_provider_details,
+    get_provider_details_by_id,
+    get_provider_details_by_id,
+    dao_update_provider_details
+)
+
+provider_details = Blueprint('provider_details', __name__)
+
+
+@provider_details.route('', methods=['GET'])
+def get_providers():
+    data, errors = provider_details_schema.dump(get_provider_details(), many=True)
+    return jsonify(provider_details=data)
+
+
+@provider_details.route('/<uuid:provider_details_id>', methods=['GET'])
+def get_provider_by_id(provider_details_id):
+    data, errors = provider_details_schema.dump(get_provider_details_by_id(provider_details_id))
+    return jsonify(provider_details=data)
+
+
+@provider_details.route('/<uuid:provider_details_id>', methods=['POST'])
+def update_provider_details(provider_details_id):
+    fetched_provider_details = get_provider_details_by_id(provider_details_id)
+
+    current_data = dict(provider_details_schema.dump(fetched_provider_details).data.items())
+    current_data.update(request.get_json())
+    update_dict, errors = provider_details_schema.load(current_data)
+    if errors:
+        return jsonify(result="error", message=errors), 400
+
+    if "identifier" in request.get_json().keys():
+        return jsonify(message={
+            "identifier": ["Not permitted to be updated"]
+        }, result='error'), 400
+
+    dao_update_provider_details(update_dict)
+    return jsonify(provider_details=provider_details_schema.dump(fetched_provider_details).data), 200

app/schemas.py

@@ -84,6 +84,12 @@ class UserSchema(BaseSchema):
             "_password", "verify_codes")
 
 
+class ProviderDetailsSchema(BaseSchema):
+    class Meta:
+        model = models.ProviderDetails
+        exclude = ("provider_rates", "provider_stats")
+
+
 class ServiceSchema(BaseSchema):
 
     created_by = field_for(models.Service, 'created_by', required=True)
@@ -402,4 +408,5 @@ api_key_history_schema = ApiKeyHistorySchema()
 template_history_schema = TemplateHistorySchema()
 event_schema = EventSchema()
 from_to_date_schema = FromToDateSchema()
+provider_details_schema = ProviderDetailsSchema()
 week_aggregate_notification_statistics_schema = WeekAggregateNotificationStatisticsSchema()