Merge branch 'main' into jim/062522/configupdates

initial sms provider cleanup

.gitignore

@@ -71,10 +71,12 @@ environment.sh
 .envrc
 .env
 .env*
+varsfile
 
 celerybeat-schedule
 
 # CloudFoundry
 .cf
+varsfile*
 
 /scripts/run_my_tests.sh

Procfile

@@ -1 +1,2 @@
-web: ./scripts/paas_app_wrapper.sh
+web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
+worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null

app/__init__.py

@@ -34,8 +34,7 @@ from app.clients.cbc_proxy import CBCProxyClient
 from app.clients.document_download import DocumentDownloadClient
 from app.clients.email.aws_ses import AwsSesClient
 from app.clients.email.aws_ses_stub import AwsSesStubClient
-from app.clients.sms.firetext import FiretextClient
-from app.clients.sms.mmg import MMGClient
+from app.clients.sms.aws_sns import AwsSnsClient
 
 
 class SQLAlchemy(_SQLAlchemy):
@@ -54,10 +53,9 @@ db = SQLAlchemy()
 migrate = Migrate()
 ma = Marshmallow()
 notify_celery = NotifyCelery()
-firetext_client = FiretextClient()
-mmg_client = MMGClient()
 aws_ses_client = AwsSesClient()
 aws_ses_stub_client = AwsSesStubClient()
+aws_sns_client = AwsSnsClient()
 encryption = Encryption()
 zendesk_client = ZendeskClient()
 statsd_client = StatsdClient()
@@ -96,8 +94,7 @@ def create_app(application):
     zendesk_client.init_app(application)
     statsd_client.init_app(application)
     logging.init_app(application, statsd_client)
-    firetext_client.init_app(application, statsd_client=statsd_client)
-    mmg_client.init_app(application, statsd_client=statsd_client)
+    aws_sns_client.init_app(application, statsd_client=statsd_client)
 
     aws_ses_client.init_app(application.config['AWS_REGION'], statsd_client=statsd_client)
     aws_ses_stub_client.init_app(
@@ -108,7 +105,7 @@ def create_app(application):
     # If a stub url is provided for SES, then use the stub client rather than the real SES boto client
     email_clients = [aws_ses_stub_client] if application.config['SES_STUB_URL'] else [aws_ses_client]
     notification_provider_clients.init_app(
-        sms_clients=[firetext_client, mmg_client],
+        sms_clients=[aws_sns_client],
         email_clients=email_clients
     )
 

app/authentication/auth.py

@@ -77,6 +77,8 @@ def requires_internal_auth(expected_client_id):
     client_id = _get_token_issuer(auth_token)
 
     if client_id != expected_client_id:
+        current_app.logger.info('client_id: %s', client_id)
+        current_app.logger.info('expected_client_id: %s', expected_client_id)
         raise AuthError("Unauthorized: not allowed to perform this action", 401)
 
     api_keys = [

app/celery/process_sms_client_response_tasks.py

@@ -6,8 +6,6 @@ from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
 from app.clients import ClientException
-from app.clients.sms.firetext import get_firetext_responses
-from app.clients.sms.mmg import get_mmg_responses
 from app.dao import notifications_dao
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
@@ -15,44 +13,44 @@ from app.notifications.notifications_ses_callback import (
     check_and_queue_callback_task,
 )
 
-sms_response_mapper = {
-    'MMG': get_mmg_responses,
-    'Firetext': get_firetext_responses,
-}
+# sms_response_mapper = {
+#     'MMG': get_mmg_responses,
+#     'Firetext': get_firetext_responses,
+# }
 
 
-@notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
-def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
-    # validate reference
-    try:
-        uuid.UUID(provider_reference, version=4)
-    except ValueError as e:
-        current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
-        raise e
+# @notify_celery.task(bind=True, name="process-sms-client-response", max_retries=5, default_retry_delay=300)
+# def process_sms_client_response(self, status, provider_reference, client_name, detailed_status_code=None):
+#     # validate reference
+#     try:
+#         uuid.UUID(provider_reference, version=4)
+#     except ValueError as e:
+#         current_app.logger.exception(f'{client_name} callback with invalid reference {provider_reference}')
+#         raise e
 
-    response_parser = sms_response_mapper[client_name]
+#     response_parser = sms_response_mapper[client_name]
 
-    # validate status
-    try:
-        notification_status, detailed_status = response_parser(status, detailed_status_code)
-        current_app.logger.info(
-            f'{client_name} callback returned status of {notification_status}'
-            f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
-        )
-    except KeyError:
-        _process_for_status(
-            notification_status='technical-failure',
-            client_name=client_name,
-            provider_reference=provider_reference
-        )
-        raise ClientException(f'{client_name} callback failed: status {status} not found.')
+#     # validate status
+#     try:
+#         notification_status, detailed_status = response_parser(status, detailed_status_code)
+#         current_app.logger.info(
+#             f'{client_name} callback returned status of {notification_status}'
+#             f'({status}): {detailed_status}({detailed_status_code}) for reference: {provider_reference}'
+#         )
+#     except KeyError:
+#         _process_for_status(
+#             notification_status='technical-failure',
+#             client_name=client_name,
+#             provider_reference=provider_reference
+#         )
+#         raise ClientException(f'{client_name} callback failed: status {status} not found.')
 
-    _process_for_status(
-        notification_status=notification_status,
-        client_name=client_name,
-        provider_reference=provider_reference,
-        detailed_status_code=detailed_status_code
-    )
+#     _process_for_status(
+#         notification_status=notification_status,
+#         client_name=client_name,
+#         provider_reference=provider_reference,
+#         detailed_status_code=detailed_status_code
+#     )
 
 
 def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None):

app/clients/__init__.py

@@ -1,3 +1,5 @@
+from celery import current_app
+
 class ClientException(Exception):
     '''
     Base Exceptions for sending notifications that fail
@@ -36,7 +38,7 @@ class NotificationProviderClients(object):
 
     def get_client_by_name_and_type(self, name, notification_type):
         assert notification_type in ['email', 'sms']
-
+        
         if notification_type == 'email':
             return self.get_email_client(name)
 

app/clients/sms/__init__.py

@@ -1,61 +1,25 @@
-from time import monotonic
-
 from app.clients import Client, ClientException
 
 
 class SmsClientResponseException(ClientException):
-    '''
+    """
     Base Exception for SmsClientsResponses
-    '''
+    """
 
     def __init__(self, message):
         self.message = message
 
     def __str__(self):
-        return f"SMS client error ({self.message})"
+        return "Message {}".format(self.message)
 
 
 class SmsClient(Client):
-    '''
+    """
     Base Sms client for sending smss.
-    '''
+    """
 
-    def init_app(self, current_app, statsd_client):
-        self.current_app = current_app
-        self.statsd_client = statsd_client
+    def send_sms(self, *args, **kwargs):
+        raise NotImplementedError("TODO Need to implement.")
 
-    def record_outcome(self, success):
-        log_message = "Provider request for {} {}".format(
-            self.name,
-            "succeeded" if success else "failed",
-        )
-
-        if success:
-            self.current_app.logger.info(log_message)
-            self.statsd_client.incr(f"clients.{self.name}.success")
-        else:
-            self.statsd_client.incr(f"clients.{self.name}.error")
-            self.current_app.logger.warning(log_message)
-
-    def send_sms(self, to, content, reference, international, sender):
-        start_time = monotonic()
-
-        try:
-            response = self.try_send_sms(to, content, reference, international, sender)
-            self.record_outcome(True)
-        except SmsClientResponseException as e:
-            self.record_outcome(False)
-            raise e
-        finally:
-            elapsed_time = monotonic() - start_time
-            self.statsd_client.timing(f"clients.{self.name}.request-time", elapsed_time)
-            self.current_app.logger.info(f"{self.name} request for {reference} finished in {elapsed_time}")
-
-        return response
-
-    def try_send_sms(self, *args, **kwargs):
-        raise NotImplementedError('TODO Need to implement.')
-
-    @property
-    def name(self):
-        raise NotImplementedError('TODO Need to implement.')
+    def get_name(self):
+        raise NotImplementedError("TODO Need to implement.")

app/clients/sms/aws_sns.py

@@ -0,0 +1,89 @@
+import re
+from time import monotonic
+
+import boto3
+import botocore
+import phonenumbers
+
+from app.clients.sms import SmsClient
+
+
+class AwsSnsClient(SmsClient):
+    """
+    AwsSns sms client
+    """
+
+    def init_app(self, current_app, statsd_client, *args, **kwargs):
+        self._client = boto3.client("sns", region_name=current_app.config["AWS_REGION"])
+        self._long_codes_client = boto3.client("sns", region_name=current_app.config["AWS_PINPOINT_REGION"])
+        super(SmsClient, self).__init__(*args, **kwargs)
+        self.current_app = current_app
+        self.statsd_client = statsd_client
+        self.long_code_regex = re.compile(r"^\+1\d{10}$")
+        
+    @property
+    def name(self):
+        return 'sns'
+
+    def get_name(self):
+        return 'sns'
+
+    def send_sms(self, to, content, reference, sender=None, international=False):
+        matched = False
+
+        for match in phonenumbers.PhoneNumberMatcher(to, "US"):
+            matched = True
+            to = phonenumbers.format_number(match.number, phonenumbers.PhoneNumberFormat.E164)
+
+            client = self._client
+            # See documentation
+            # https://docs.aws.amazon.com/sns/latest/dg/sms_publish-to-phone.html#sms_publish_sdk
+            attributes = {
+                "AWS.SNS.SMS.SMSType": {
+                    "DataType": "String",
+                    "StringValue": "Transactional",
+                }
+            }
+
+            # If sending with a long code number, we need to use another AWS region
+            # and specify the phone number we want to use as the origination number
+            send_with_dedicated_phone_number = self._send_with_dedicated_phone_number(sender)
+            if send_with_dedicated_phone_number:
+                client = self._long_codes_client
+                attributes["AWS.MM.SMS.OriginationNumber"] = {
+                    "DataType": "String",
+                    "StringValue": sender,
+                }
+
+            # If the number is US based, we must use a US Toll Free number to send the message
+            country = phonenumbers.region_code_for_number(match.number)
+            if country == "US":
+                client = self._long_codes_client
+                attributes["AWS.MM.SMS.OriginationNumber"] = {
+                    "DataType": "String",
+                    "StringValue": self.current_app.config["AWS_US_TOLL_FREE_NUMBER"],
+                }
+
+            try:
+                start_time = monotonic()
+                response = client.publish(PhoneNumber=to, Message=content, MessageAttributes=attributes)
+            except botocore.exceptions.ClientError as e:
+                self.statsd_client.incr("clients.sns.error")
+                raise str(e)
+            except Exception as e:
+                self.statsd_client.incr("clients.sns.error")
+                raise str(e)
+            finally:
+                elapsed_time = monotonic() - start_time
+                self.current_app.logger.info("AWS SNS request finished in {}".format(elapsed_time))
+                self.statsd_client.timing("clients.sns.request-time", elapsed_time)
+                self.statsd_client.incr("clients.sns.success")
+            return response["MessageId"]
+
+        if not matched:
+            self.statsd_client.incr("clients.sns.error")
+            self.current_app.logger.error("No valid numbers found in {}".format(to))
+            raise ValueError("No valid numbers found for SMS delivery")
+
+    def _send_with_dedicated_phone_number(self, sender):
+        return sender and re.match(self.long_code_regex, sender)

app/clients/sms/firetext.py

@@ -1,88 +0,0 @@
-import json
-import logging
-
-from requests import RequestException, request
-
-from app.clients.sms import SmsClient, SmsClientResponseException
-
-logger = logging.getLogger(__name__)
-
-# Firetext will send a delivery receipt with three different status codes.
-# The `firetext_response` maps these codes to the notification statistics status and notification status.
-# If we get a pending (status = 2) delivery receipt followed by a declined (status = 1) delivery receipt we will set
-# the notification status to temporary-failure rather than permanent failure.
-#  See the code in the notification_dao.update_notifications_status_by_id
-firetext_responses = {
-    '0': 'delivered',
-    '1': 'permanent-failure',
-    '2': 'pending'
-}
-
-firetext_codes = {
-    # code '000' means 'No errors reported'
-    '000': {'status': 'temporary-failure', 'reason': 'No error reported'},
-    '101': {'status': 'permanent-failure', 'reason': 'Unknown Subscriber'},
-    '102': {'status': 'temporary-failure', 'reason': 'Absent Subscriber'},
-    '103': {'status': 'temporary-failure', 'reason': 'Subscriber Busy'},
-    '104': {'status': 'temporary-failure', 'reason': 'No Subscriber Memory'},
-    '201': {'status': 'permanent-failure', 'reason': 'Invalid Number'},
-    '301': {'status': 'permanent-failure', 'reason': 'SMS Not Supported'},
-    '302': {'status': 'temporary-failure', 'reason': 'SMS Not Supported'},
-    '401': {'status': 'permanent-failure', 'reason': 'Message Rejected'},
-    '900': {'status': 'temporary-failure', 'reason': 'Routing Error'},
-}
-
-
-def get_firetext_responses(status, detailed_status_code=None):
-    detailed_status = firetext_codes[detailed_status_code]['reason'] if firetext_codes.get(
-        detailed_status_code, None
-    ) else None
-    return (firetext_responses[status], detailed_status)
-
-
-def get_message_status_and_reason_from_firetext_code(detailed_status_code):
-    return firetext_codes[detailed_status_code]['status'], firetext_codes[detailed_status_code]['reason']
-
-
-class FiretextClient(SmsClient):
-    '''
-    FireText sms client.
-    '''
-
-    def init_app(self, *args, **kwargs):
-        super().init_app(*args, **kwargs)
-        self.api_key = self.current_app.config.get('FIRETEXT_API_KEY')
-        self.international_api_key = self.current_app.config.get('FIRETEXT_INTERNATIONAL_API_KEY')
-        self.url = self.current_app.config.get('FIRETEXT_URL')
-
-    @property
-    def name(self):
-        return 'firetext'
-
-    def try_send_sms(self, to, content, reference, international, sender):
-        data = {
-            "apiKey": self.international_api_key if international else self.api_key,
-            "from": sender,
-            "to": to.replace('+', ''),
-            "message": content,
-            "reference": reference
-        }
-
-        try:
-            response = request(
-                "POST",
-                self.url,
-                data=data,
-                timeout=60
-            )
-            response.raise_for_status()
-            try:
-                json.loads(response.text)
-                if response.json()['code'] != 0:
-                    raise ValueError("Expected 'code' to be '0'")
-            except (ValueError, AttributeError):
-                raise SmsClientResponseException("Invalid response JSON")
-        except RequestException:
-            raise SmsClientResponseException("Request failed")
-
-        return response

app/clients/sms/mmg.py

@@ -1,110 +0,0 @@
-import json
-
-from requests import RequestException, request
-
-from app.clients.sms import SmsClient, SmsClientResponseException
-
-mmg_response_map = {
-    '2': {'status': 'permanent-failure', 'substatus': {
-        "1": "Number does not exist",
-        "4": "Rejected by operator",
-        "5": "Unidentified Subscriber",
-        "9": "Undelivered",
-        "11": "Service for Subscriber suspended",
-        "12": "Illegal equipment",
-        "2049": "Subscriber IMSI blacklisted",
-        "2050": "Number blacklisted in do-not-disturb blacklist",
-        "2052": "Destination number blacklisted",
-        "2053": "Source address blacklisted"
-    }},
-    '3': {'status': 'delivered', 'substatus': {"2": "Delivered to operator", "5": "Delivered to handset"}},
-    '4': {'status': 'temporary-failure', 'substatus': {
-        "6": "Absent Subscriber",
-        "8": "Roaming not allowed",
-        "13": "SMS Not Supported",
-        "15": "Expired",
-        "27": "Absent Subscriber",
-        "29": "Invalid delivery report",
-        "32": "Delivery Failure",
-    }},
-    '5': {'status': 'permanent-failure', 'substatus': {
-        "6": "Network out of coverage",
-        "8": "Incorrect number prefix",
-        "10": "Number on do-not-disturb service",
-        "11": "Sender id not registered",
-        "13": "Sender id blacklisted",
-        "14": "Destination number blacklisted",
-        "19": "Routing unavailable",
-        "20": "Rejected by anti-flooding mechanism",
-        "21": "System error",  # it says to retry those messages or contact support
-        "23": "Duplicate message id",
-        "24": "Message formatted incorrectly",
-        "25": "Message too long",
-        "51": "Missing recipient value",
-        "52": "Invalid destination",
-    }},
-}
-
-
-def get_mmg_responses(status, detailed_status_code=None):
-    return (mmg_response_map[status]["status"], mmg_response_map[status]["substatus"].get(detailed_status_code, None))
-
-
-class MMGClientResponseException(SmsClientResponseException):
-    def __init__(self, response, exception):
-        status_code = response.status_code if response is not None else 504
-        text = response.text if response is not None else "Gateway Time-out"
-
-        self.status_code = status_code
-        self.text = text
-        self.exception = exception
-
-    def __str__(self):
-        return "Code {} text {} exception {}".format(self.status_code, self.text, str(self.exception))
-
-
-class MMGClient(SmsClient):
-    '''
-    MMG sms client
-    '''
-
-    def init_app(self, *args, **kwargs):
-        super().init_app(*args, **kwargs)
-        self.api_key = self.current_app.config.get('MMG_API_KEY')
-        self.mmg_url = self.current_app.config.get('MMG_URL')
-
-    @property
-    def name(self):
-        return 'mmg'
-
-    def try_send_sms(self, to, content, reference, international, sender):
-        data = {
-            "reqType": "BULK",
-            "MSISDN": to,
-            "msg": content,
-            "sender": sender,
-            "cid": reference,
-            "multi": True
-        }
-
-        try:
-            response = request(
-                "POST",
-                self.mmg_url,
-                data=json.dumps(data),
-                headers={
-                    'Content-Type': 'application/json',
-                    'Authorization': 'Basic {}'.format(self.api_key)
-                },
-                timeout=60
-            )
-
-            response.raise_for_status()
-            try:
-                json.loads(response.text)
-            except (ValueError, AttributeError):
-                raise SmsClientResponseException("Invalid response JSON")
-        except RequestException:
-            raise SmsClientResponseException("Request failed")
-
-        return response

app/cloudfoundry_config.py

@@ -6,7 +6,7 @@ def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
 
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'].replace('postgres',
-                                                                                                       'postgresql')
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
+                                                                                                      'postgresql')
     # Redis config
-    os.environ['REDIS_URL'] = vcap_services['redis'][0]['credentials']['uri']
+    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri']

app/config.py

@@ -82,7 +82,7 @@ class TaskNames(object):
 
 class Config(object):
     # URL of admin app
-    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL', 'http://localhost:6012')
+    ADMIN_BASE_URL = os.getenv('ADMIN_BASE_URL')
 
     # URL of api app (on AWS this is the internal api endpoint)
     API_HOST_NAME = os.getenv('API_HOST_NAME')
@@ -92,15 +92,23 @@ class Config(object):
     GOVUK_ALERTS_CLIENT_ID = 'govuk-alerts'
 
     INTERNAL_CLIENT_API_KEYS = json.loads(
-        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{}')
+        os.environ.get('INTERNAL_CLIENT_API_KEYS', '{"notify-admin":["dev-notify-secret-key"]}')
     )
 
     # encyption secret/salt
-    SECRET_KEY = os.getenv('SECRET_KEY')
-    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT')
+    ADMIN_CLIENT_SECRET = os.getenv('ADMIN_CLIENT_SECRET', 'dev-notify-secret-key')
+    SECRET_KEY = os.getenv('SECRET_KEY', 'dev-notify-secret-key')
+    DANGEROUS_SALT = os.getenv('DANGEROUS_SALT', 'dev-notify-salt ')
 
     # DB conection string
     SQLALCHEMY_DATABASE_URI = os.getenv('SQLALCHEMY_DATABASE_URI')
+    
+    # Redis conection string
+    REDIS_URL = os.getenv('REDIS_URL')
+    
+    # AWS SMS
+    AWS_PINPOINT_REGION = os.getenv("AWS_PINPOINT_REGION", "us-west-2")
+    AWS_US_TOLL_FREE_NUMBER = os.getenv("AWS_US_TOLL_FREE_NUMBER", "+18446120782")
 
     # MMG API Key
     MMG_API_KEY = os.getenv('MMG_API_KEY')
@@ -369,8 +377,8 @@ class Config(object):
     FIRETEXT_INBOUND_SMS_AUTH = json.loads(os.environ.get('FIRETEXT_INBOUND_SMS_AUTH', '[]'))
     MMG_INBOUND_SMS_AUTH = json.loads(os.environ.get('MMG_INBOUND_SMS_AUTH', '[]'))
     MMG_INBOUND_SMS_USERNAME = json.loads(os.environ.get('MMG_INBOUND_SMS_USERNAME', '[]'))
-    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', '')
-    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', '')
+    ROUTE_SECRET_KEY_1 = os.environ.get('ROUTE_SECRET_KEY_1', 'dev-route-secret-key-1')
+    ROUTE_SECRET_KEY_2 = os.environ.get('ROUTE_SECRET_KEY_2', 'dev-route-secret-key-2')
 
     HIGH_VOLUME_SERVICE = json.loads(os.environ.get('HIGH_VOLUME_SERVICE', '[]'))
 
@@ -406,8 +414,8 @@ class Development(Config):
 
     REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
 
-    CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
-    CONTACT_LIST_BUCKET_NAME = 'development-contact-list'
+    CSV_UPLOAD_BUCKET_NAME = 'local-notifications-csv-upload'
+    CONTACT_LIST_BUCKET_NAME = 'local-contact-list'
     TEST_LETTERS_BUCKET_NAME = 'development-test-letters'
     DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
     LETTERS_PDF_BUCKET_NAME = 'development-letters-pdf'
@@ -416,10 +424,10 @@ class Development(Config):
     TRANSIENT_UPLOADED_LETTERS = 'development-transient-uploaded-letters'
     LETTER_SANITISE_BUCKET_NAME = 'development-letters-sanitise'
 
-    INTERNAL_CLIENT_API_KEYS = {
-        Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
-        Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
-    }
+    # INTERNAL_CLIENT_API_KEYS = {
+    #     Config.ADMIN_CLIENT_ID: ['dev-notify-secret-key'],
+    #     Config.GOVUK_ALERTS_CLIENT_ID: ['govuk-alerts-secret-key']
+    # }
 
     SECRET_KEY = 'dev-notify-secret-key'
     DANGEROUS_SALT = 'dev-notify-salt'
@@ -525,7 +533,7 @@ class Staging(Config):
 
 
 class Live(Config):
-    NOTIFY_EMAIL_DOMAIN = 'notifications.service.gov.uk'
+    NOTIFY_EMAIL_DOMAIN = os.environ.get('NOTIFY_EMAIL_DOMAIN')
     NOTIFY_ENVIRONMENT = 'live'
     CSV_UPLOAD_BUCKET_NAME = 'live-notifications-csv-upload'
     CONTACT_LIST_BUCKET_NAME = 'production-contact-list'

app/dao/notifications_dao.py

@@ -21,9 +21,6 @@ from sqlalchemy.sql.expression import case
 from werkzeug.datastructures import MultiDict
 
 from app import create_uuid, db, statsd_client
-from app.clients.sms.firetext import (
-    get_message_status_and_reason_from_firetext_code,
-)
 from app.dao.dao_utils import autocommit
 from app.letters.utils import LetterPDFNotFound, find_letter_pdf_in_s3
 from app.models import (
@@ -86,25 +83,6 @@ def dao_create_notification(notification):
     db.session.add(notification)
 
 
-def _decide_permanent_temporary_failure(status, notification, detailed_status_code=None):
-    # Firetext will send us a pending status, followed by a success or failure status.
-    # When we get a failure status we need to look at the detailed_status_code to determine if the failure type
-    # is a permanent-failure or temporary-failure.
-    if notification.sent_by == 'firetext':
-        if status == NOTIFICATION_PERMANENT_FAILURE and detailed_status_code:
-            try:
-                status, reason = get_message_status_and_reason_from_firetext_code(detailed_status_code)
-                current_app.logger.info(
-                    f'Updating notification id {notification.id} to status {status}, reason: {reason}')
-                return status
-            except KeyError:
-                current_app.logger.warning(f'Failure code {detailed_status_code} from Firetext not recognised')
-        # fallback option:
-        if status == NOTIFICATION_PERMANENT_FAILURE and notification.status == NOTIFICATION_PENDING:
-            status = NOTIFICATION_TEMPORARY_FAILURE
-    return status
-
-
 def country_records_delivery(phone_prefix):
     dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
     return dlr and dlr.lower() == 'yes'

app/delivery/send_to_providers.py

@@ -45,6 +45,9 @@ def send_sms_to_provider(notification):
 
     if notification.status == 'created':
         provider = provider_to_use(SMS_TYPE, notification.international)
+        if not provider:
+            technical_failure(notification=notification)
+            return
 
         template_model = SerialisedTemplate.from_id_and_service_id(
             template_id=notification.template_id, service_id=service.id, version=notification.template_version
@@ -168,9 +171,11 @@ provider_cache = TTLCache(maxsize=8, ttl=10)
 
 
 @cached(cache=provider_cache)
-def provider_to_use(notification_type, international=False):
+def provider_to_use(notification_type, international=True):
+    international = False # TODO: remove or resolve the functionality of this flag
+    # TODO rip firetext and mmg out of early migrations and clean up the expression below
     active_providers = [
-        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active
+        p for p in get_provider_details_by_notification_type(notification_type, international) if p.active and p.identifier not in ['firetext','mmg']
     ]
 
     if not active_providers:
@@ -180,11 +185,10 @@ def provider_to_use(notification_type, international=False):
         raise Exception("No active {} providers".format(notification_type))
 
     if len(active_providers) == 1:
-        weights = [100]
+        chosen_provider = active_providers[0]
     else:
         weights = [p.priority for p in active_providers]
-
-    chosen_provider = random.choices(active_providers, weights=weights)[0]
+        chosen_provider = random.choices(active_providers, weights=weights)[0]
 
     return notification_provider_clients.get_client_by_name_and_type(chosen_provider.identifier, notification_type)
 

app/models.py

@@ -1136,9 +1136,10 @@ class TemplateHistory(TemplateBase):
 
 MMG_PROVIDER = "mmg"
 FIRETEXT_PROVIDER = "firetext"
+SNS_PROVIDER = 'sns'
 SES_PROVIDER = 'ses'
 
-SMS_PROVIDERS = [MMG_PROVIDER, FIRETEXT_PROVIDER]
+SMS_PROVIDERS = [MMG_PROVIDER, FIRETEXT_PROVIDER, SNS_PROVIDER]
 EMAIL_PROVIDERS = [SES_PROVIDER]
 PROVIDERS = SMS_PROVIDERS + EMAIL_PROVIDERS
 

app/notifications/notifications_sms_callback.py

@@ -1,8 +1,8 @@
 from flask import Blueprint, json, jsonify, request
 
-from app.celery.process_sms_client_response_tasks import (
-    process_sms_client_response,
-)
+# from app.celery.process_sms_client_response_tasks import (
+#     process_sms_client_response,
+# )
 from app.config import QueueNames
 from app.errors import InvalidRequest, register_errors
 
@@ -10,48 +10,48 @@ sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifi
 register_errors(sms_callback_blueprint)
 
 
-@sms_callback_blueprint.route('/mmg', methods=['POST'])
-def process_mmg_response():
-    client_name = 'MMG'
-    data = json.loads(request.data)
-    errors = validate_callback_data(data=data,
-                                    fields=['status', 'CID'],
-                                    client_name=client_name)
-    if errors:
-        raise InvalidRequest(errors, status_code=400)
+# @sms_callback_blueprint.route('/mmg', methods=['POST'])
+# def process_mmg_response():
+#     client_name = 'MMG'
+#     data = json.loads(request.data)
+#     errors = validate_callback_data(data=data,
+#                                     fields=['status', 'CID'],
+#                                     client_name=client_name)
+#     if errors:
+#         raise InvalidRequest(errors, status_code=400)
 
-    status = str(data.get('status'))
-    detailed_status_code = str(data.get('substatus'))
+#     status = str(data.get('status'))
+#     detailed_status_code = str(data.get('substatus'))
 
-    provider_reference = data.get('CID')
+#     provider_reference = data.get('CID')
 
-    process_sms_client_response.apply_async(
-        [status, provider_reference, client_name, detailed_status_code],
-        queue=QueueNames.SMS_CALLBACKS,
-    )
+#     process_sms_client_response.apply_async(
+#         [status, provider_reference, client_name, detailed_status_code],
+#         queue=QueueNames.SMS_CALLBACKS,
+#     )
 
-    return jsonify(result='success'), 200
+#     return jsonify(result='success'), 200
 
 
-@sms_callback_blueprint.route('/firetext', methods=['POST'])
-def process_firetext_response():
-    client_name = 'Firetext'
-    errors = validate_callback_data(data=request.form,
-                                    fields=['status', 'reference'],
-                                    client_name=client_name)
-    if errors:
-        raise InvalidRequest(errors, status_code=400)
+# @sms_callback_blueprint.route('/firetext', methods=['POST'])
+# def process_firetext_response():
+#     client_name = 'Firetext'
+#     errors = validate_callback_data(data=request.form,
+#                                     fields=['status', 'reference'],
+#                                     client_name=client_name)
+#     if errors:
+#         raise InvalidRequest(errors, status_code=400)
 
-    status = request.form.get('status')
-    detailed_status_code = request.form.get('code')
-    provider_reference = request.form.get('reference')
+#     status = request.form.get('status')
+#     detailed_status_code = request.form.get('code')
+#     provider_reference = request.form.get('reference')
 
-    process_sms_client_response.apply_async(
-        [status, provider_reference, client_name, detailed_status_code],
-        queue=QueueNames.SMS_CALLBACKS,
-    )
+#     process_sms_client_response.apply_async(
+#         [status, provider_reference, client_name, detailed_status_code],
+#         queue=QueueNames.SMS_CALLBACKS,
+#     )
 
-    return jsonify(result='success'), 200
+#     return jsonify(result='success'), 200
 
 
 def validate_callback_data(data, fields, client_name):

app/notifications/process_notifications.py

@@ -105,9 +105,14 @@ def persist_notification(
     document_download_count=None,
     updated_at=None
 ):
+    current_app.logger.info('Presisting notification')
+    
     notification_created_at = created_at or datetime.utcnow()
     if not notification_id:
         notification_id = uuid.uuid4()
+    
+    current_app.logger.info('Presisting notification with id {}'.format(notification_id))
+    
     notification = Notification(
         id=notification_id,
         template_id=template_id,
@@ -130,6 +135,8 @@ def persist_notification(
         document_download_count=document_download_count,
         updated_at=updated_at
     )
+    
+    current_app.logger.info('Presisting notification with to address: {}'.format(notification.to))
 
     if notification_type == SMS_TYPE:
         formatted_recipient = validate_and_format_phone_number(recipient, international=True)
@@ -139,7 +146,9 @@ def persist_notification(
         notification.phone_prefix = recipient_info.country_prefix
         notification.rate_multiplier = recipient_info.billable_units
     elif notification_type == EMAIL_TYPE:
+        current_app.logger.info('Presisting notification with type: {}'.format(EMAIL_TYPE))
         notification.normalised_to = format_email_address(notification.to)
+        current_app.logger.info('Presisting notification to formatted email: {}'.format(notification.normalised_to))
     elif notification_type == LETTER_TYPE:
         notification.postage = postage
         notification.international = postage in INTERNATIONAL_POSTAGE_TYPES
@@ -147,17 +156,24 @@ def persist_notification(
 
     # if simulated create a Notification model to return but do not persist the Notification to the dB
     if not simulated:
+        current_app.logger.info('Firing dao_create_notification')
         dao_create_notification(notification)
         if key_type != KEY_TYPE_TEST and current_app.config['REDIS_ENABLED']:
+            current_app.logger.info('Redis enabled, querying cache key for service id: {}'.format(service.id))
             cache_key = redis.daily_limit_cache_key(service.id)
+            current_app.logger.info('Redis daily limit cache key: {}'.format(cache_key))
             if redis_store.get(cache_key) is None:
+                current_app.logger.info('Redis daily limit cache key does not exist')
                 # if cache does not exist set the cache to 1 with an expiry of 24 hours,
                 # The cache should be set by the time we create the notification
                 # but in case it is this will make sure the expiry is set to 24 hours,
                 # where if we let the incr method create the cache it will be set a ttl.
                 redis_store.set(cache_key, 1, ex=86400)
+                current_app.logger.info('Set redis daily limit cache key to 1')
             else:
+                current_app.logger.info('Redis daily limit cache key does exist')
                 redis_store.incr(cache_key)
+                current_app.logger.info('Redis daily limit cache key has been incremented')
         current_app.logger.info(
             "{} {} created at {}".format(notification_type, notification_id, notification_created_at)
         )

app/user/rest.py

@@ -370,13 +370,19 @@ def send_user_confirm_new_email(user_id):
 
 @user_blueprint.route('/<uuid:user_id>/email-verification', methods=['POST'])
 def send_new_user_email_verification(user_id):
+    current_app.logger.info('Sending email verification for user {}'.format(user_id))
     request_json = request.get_json()
 
     # when registering, we verify all users' email addresses using this function
     user_to_send_to = get_user_by_id(user_id=user_id)
+    current_app.logger.info('user_to_send_to is {}'.format(user_to_send_to))
+    current_app.logger.info('user_to_send_to.email_address is {}'.format(user_to_send_to.email_address))
 
     template = dao_get_template_by_id(current_app.config['NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID'])
     service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
+    
+    current_app.logger.info('template.id is {}'.format(template.id))
+    current_app.logger.info('service.id is {}'.format(service.id))
 
     saved_notification = persist_notification(
         template_id=template.id,
@@ -395,18 +401,27 @@ def send_new_user_email_verification(user_id):
         key_type=KEY_TYPE_NORMAL,
         reply_to_text=service.get_default_reply_to_email_address()
     )
+    current_app.logger.info('Sending notification to queue')
 
     send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+    
+    current_app.logger.info('Sent notification to queue')
 
     return jsonify({}), 204
 
 
 @user_blueprint.route('/<uuid:user_id>/email-already-registered', methods=['POST'])
 def send_already_registered_email(user_id):
+    current_app.logger.info('Email already registered for user {}'.format(user_id))
     to = email_data_request_schema.load(request.get_json())
+    
+    current_app.logger.info('To email is {}'.format(to['email']))
 
     template = dao_get_template_by_id(current_app.config['ALREADY_REGISTERED_EMAIL_TEMPLATE_ID'])
     service = Service.query.get(current_app.config['NOTIFY_SERVICE_ID'])
+    
+    current_app.logger.info('template.id is {}'.format(template.id))
+    current_app.logger.info('service.id is {}'.format(service.id))
 
     saved_notification = persist_notification(
         template_id=template.id,
@@ -423,8 +438,12 @@ def send_already_registered_email(user_id):
         key_type=KEY_TYPE_NORMAL,
         reply_to_text=service.get_default_reply_to_email_address()
     )
+    
+    current_app.logger.info('Sending notification to queue')
 
     send_notification_to_queue(saved_notification, False, queue=QueueNames.NOTIFY)
+    
+    current_app.logger.info('Sent notification to queue')
 
     return jsonify({}), 204
 

manifest.yml

@@ -0,0 +1,43 @@
+---
+
+applications:
+  - name: notifications-api
+    buildpack: python_buildpack
+    instances: 1
+    memory: 1G
+    disk_quota: 1G
+    health-check-type: process
+    health-check-invocation-timeout: 1
+    routes:
+    - route: notifications-api.app.cloud.gov
+
+    services:
+      - api-psql
+      - api-redis
+
+    env:
+      NOTIFY_APP_NAME: api
+      NOTIFY_LOG_PATH: /home/vcap/logs/app.log
+      FLASK_APP: application.py
+      FLASK_ENV: production
+
+      NOTIFY_ENVIRONMENT: live
+      API_HOST_NAME: https://notifications-api.app.cloud.gov
+      ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
+      NOTIFICATION_QUEUE_PREFIX: prototype_10x
+      STATSD_HOST: localhost
+
+      INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["dev-notify-secret-key"]}'
+      
+      # Credentials variables
+      DANGEROUS_SALT: ((DANGEROUS_SALT))
+      SECRET_KEY: ((SECRET_KEY))
+      AWS_REGION: us-west-2
+      AWS_PINPOINT_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: ((AWS_ACCESS_KEY_ID))
+      AWS_SECRET_ACCESS_KEY: ((AWS_SECRET_ACCESS_KEY))
+      AWS_US_TOLL_FREE_NUMBER: +18446120782
+
+      DVLA_EMAIL_ADDRESSES: []
+
+      NOTIFY_EMAIL_DOMAIN: dispostable.com

migrations/versions/0005_add_provider_stats.py

@@ -18,14 +18,14 @@ def upgrade():
     op.create_table('provider_rates',
     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
     sa.Column('valid_from', sa.DateTime(), nullable=False),
-    sa.Column('provider', sa.Enum('mmg', 'twilio', 'firetext', 'ses', name='providers'), nullable=False),
+    sa.Column('provider', sa.Enum('mmg', 'twilio', 'firetext', 'ses', 'sns', name='providers'), nullable=False),
     sa.Column('rate', sa.Numeric(), nullable=False),
     sa.PrimaryKeyConstraint('id')
     )
     op.create_table('provider_statistics',
     sa.Column('id', postgresql.UUID(as_uuid=True), nullable=False),
     sa.Column('day', sa.Date(), nullable=False),
-    sa.Column('provider', sa.Enum('mmg', 'twilio', 'firetext', 'ses', name='providers'), nullable=False),
+    sa.Column('provider', sa.Enum('mmg', 'twilio', 'firetext', 'ses', 'sns', name='providers'), nullable=False),
     sa.Column('service_id', postgresql.UUID(as_uuid=True), nullable=False),
     sa.Column('unit_count', sa.BigInteger(), nullable=False),
     sa.ForeignKeyConstraint(['service_id'], ['services.id'], ),

migrations/versions/0011_ad_provider_details.py

@@ -43,6 +43,9 @@ def upgrade():
     op.execute(
         "INSERT INTO provider_details (id, display_name, identifier, priority, notification_type, active) values ('{}', 'AWS SES', 'ses', 10, 'email', true)".format(str(uuid.uuid4()))
     )
+    op.execute(
+        "INSERT INTO provider_details (id, display_name, identifier, priority, notification_type, active) values ('{}', 'AWS SNS', 'sns', 10, 'sms', true)".format(str(uuid.uuid4()))
+    )
     op.execute(
         "UPDATE provider_rates set provider_id = (select id from provider_details where identifier = 'mmg') where provider = 'mmg'"
     )
@@ -52,6 +55,9 @@ def upgrade():
     op.execute(
         "UPDATE provider_rates set provider_id = (select id from provider_details where identifier = 'ses') where provider = 'ses'"
     )
+    op.execute(
+        "UPDATE provider_rates set provider_id = (select id from provider_details where identifier = 'sns') where provider = 'sns'"
+    )
     op.execute(
         "UPDATE provider_statistics set provider_id = (select id from provider_details where identifier = 'mmg') where provider = 'mmg'"
     )
@@ -61,6 +67,9 @@ def upgrade():
     op.execute(
         "UPDATE provider_statistics set provider_id = (select id from provider_details where identifier = 'ses') where provider = 'ses'"
     )
+    op.execute(
+        "UPDATE provider_statistics set provider_id = (select id from provider_details where identifier = 'sns') where provider = 'sns'"
+    )
 
 def downgrade():
 

migrations/versions/0025_notify_service_data.py

@@ -73,7 +73,7 @@ def upgrade():
                                       'email', datetime.utcnow(), invitation_content, service_id,
                                       invitation_subject, user_id))
 
-    sms_code_content = '((verify_code)) is your Notify authentication code'
+    sms_code_content = '((verify_code)) is your US Notify authentication code'
     op.execute(template_history_insert.format('36fb0730-6259-4da1-8a80-c8de22ad4246', 'Notify SMS verify code',
                                               'sms', datetime.utcnow(), sms_code_content, service_id, None, user_id))
 

migrations/versions/0076_add_intl_flag_to_provider.py

@@ -18,8 +18,8 @@ def upgrade():
     op.add_column('provider_details', sa.Column('supports_international', sa.Boolean(), nullable=False, server_default=sa.false()))
     op.add_column('provider_details_history', sa.Column('supports_international', sa.Boolean(), nullable=False, server_default=sa.false()))
 
-    op.execute("UPDATE provider_details SET supports_international=True WHERE identifier='mmg'")
-    op.execute("UPDATE provider_details_history SET supports_international=True WHERE identifier='mmg'")
+    op.execute("UPDATE provider_details SET supports_international=True WHERE identifier='sns'")
+    op.execute("UPDATE provider_details_history SET supports_international=True WHERE identifier='sns'")
 
 
 def downgrade():

migrations/versions/0265_add_confirm_edit_templates.py

@@ -76,7 +76,7 @@ def upgrade():
         )
     )
 
-    mobile_template_content = """Your mobile number was changed by ((servicemanagername)). Next time you sign in, your Notify authentication code will be sent to this phone."""
+    mobile_template_content = """Your mobile number was changed by ((servicemanagername)). Next time you sign in, your US Notify authentication code will be sent to this phone."""
 
     mobile_template_name = "Phone number changed by service manager"
 

sample.env

@@ -37,3 +37,5 @@ SQLALCHEMY_DATABASE_TEST_URI=postgresql://postgres:chummy@db:5432/test_notificat
 AWS_REGION=us-west-2
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
+AWS_PINPOINT_REGION=
+AWS_US_TOLL_FREE_NUMBER=

tests/app/clients/test_aws_sns.py

@@ -0,0 +1,24 @@
+import pytest
+from app import aws_sns_client
+def test_send_sms_successful_returns_aws_sns_response(notify_api, mocker):
+    boto_mock = mocker.patch.object(aws_sns_client, '_client', create=True)
+    mocker.patch.object(aws_sns_client, 'statsd_client', create=True)
+    to = "6135555555"
+    content = reference = 'foo'
+    with notify_api.app_context():
+        aws_sns_client.send_sms(to, content, reference)
+    boto_mock.publish.assert_called_once_with(
+        PhoneNumber="+16135555555",
+        Message=content,
+        MessageAttributes={'AWS.SNS.SMS.SMSType': {'DataType': 'String', 'StringValue': 'Transactional'}}
+    )
+
+
+def test_send_sms_returns_raises_error_if_there_is_no_valid_number_is_found(notify_api, mocker):
+    mocker.patch.object(aws_sns_client, '_client', create=True)
+    mocker.patch.object(aws_sns_client, 'statsd_client', create=True)
+    to = ""
+    content = reference = 'foo'
+    with pytest.raises(ValueError) as excinfo:
+        aws_sns_client.send_sms(to, content, reference)
+    assert 'No valid numbers found for SMS delivery' in str(excinfo.value)

varsfile.sample

@@ -0,0 +1,35 @@
+
+SECRET_KEY: "dev-notify-secret-key"
+DANGEROUS_SALT: "dev-notify-salt"
+AWS_ACCESS_KEY_ID: <replace me>
+AWS_SECRET_ACCESS_KEY: <replace me>
+
+ADMIN_BASE_URL: https://notifications-admin.app.cloud.gov
+ADMIN_CLIENT_ID: notify-admin
+ADMIN_CLIENT_SECRET: dev-notify-secret-key
+API_HOST_NAME: https://notifications-api.app.cloud.gov
+AWS_ACCESS_KEY_ID: placeholder
+AWS_PINPOINT_REGION: us-west-2
+AWS_REGION: us-west-2
+AWS_SECRET_ACCESS_KEY: placeholder
+AWS_US_TOLL_FREE_NUMBER: 18446120782
+DANGEROUS_SALT: dev-notify-salt
+DVLA_EMAIL_ADDRESSES: []
+FIRETEXT_API_KEY: placeholder
+FIRETEXT_INBOUND_SMS_AUTH: {}
+FIRETEXT_INTERNATIONAL_API_KEY: placeholder
+FLASK_APP: application.py
+FLASK_ENV: production
+INTERNAL_CLIENT_API_KEYS: '{"notify-admin":["dev-notify-secret-key"]}'
+MMG_API_KEY: placeholder
+MMG_INBOUND_SMS_AUTH: {}
+MMG_INBOUND_SMS_USERNAME: {}
+NOTIFICATION_QUEUE_PREFIX: prototype_10x
+NOTIFY_APP_NAME: api
+NOTIFY_EMAIL_DOMAIN: dispostable.com
+NOTIFY_ENVIRONMENT: live
+NOTIFY_LOG_PATH: /home/vcap/logs/app.log
+ROUTE_SECRET_KEY_1: dev-route-secret-key-1
+ROUTE_SECRET_KEY_2: dev-route-secret-key-2
+SECRET_KEY: dev-notify-secret-key
+STATSD_HOST: localhost