Merge branch 'main' into stvnrlly-remove-broadcasts

.github/workflows/checks.yml

@@ -72,7 +72,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - uses: trailofbits/gh-action-pip-audit@v1.0.0
         with:
-          inputs: requirements.txt requirements_for_test.txt
+          inputs: requirements.txt
           ignore-vulns: PYSEC-2022-237
 
   static-scan:

.github/workflows/daily_checks.yml

@@ -39,7 +39,7 @@ jobs:
       - uses: ./.github/actions/setup-project
       - uses: trailofbits/gh-action-pip-audit@v1.0.0
         with:
-          inputs: requirements.txt requirements_for_test.txt
+          inputs: requirements.txt
           ignore-vulns: PYSEC-2022-237
 
   static-scan:

.github/workflows/deploy.yml

@@ -70,7 +70,7 @@ jobs:
         cf_org: gsa-10x-prototyping
         cf_space: 10x-notifications
         push_arguments: >-
-          --var env=staging
+          --vars-file deploy-config/staging.yml
           --var DANGEROUS_SALT="$DANGEROUS_SALT"
           --var SECRET_KEY="$SECRET_KEY"
           --var ADMIN_CLIENT_SECRET="$ADMIN_CLIENT_SECRET"

.gitignore

@@ -4,6 +4,7 @@ queues.csv
 __pycache__/
 *.py[cod]
 
+.venv/
 venv/
 venv-freeze/
 

Makefile

@@ -75,7 +75,8 @@ freeze-requirements: ## Pin all requirements including sub dependencies into req
 .PHONY: audit
 audit:
 	pip install --upgrade pip-audit
-	pip-audit -r requirements.txt -r requirements_for_test.txt -l --ignore-vuln PYSEC-2022-237
+	pip-audit -r requirements.txt -l --ignore-vuln PYSEC-2022-237
+	-pip-audit -r requirements_for_test.txt -l
 
 .PHONY: static-scan
 static-scan:

Procfile

@@ -1,2 +0,0 @@
-web: unset GUNICORN_CMD_ARGS; exec ./scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
-worker: exec ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 2> /dev/null

app/__init__.py

@@ -146,6 +146,9 @@ def register_blueprint(application):
     from app.notifications.notifications_letter_callback import (
         letter_callback_blueprint,
     )
+    from app.notifications.notifications_ses_callback import (
+        ses_callback_blueprint,
+    )
     from app.notifications.notifications_sms_callback import (
         sms_callback_blueprint,
     )
@@ -189,6 +192,10 @@ def register_blueprint(application):
 
     status_blueprint.before_request(requires_no_auth)
     application.register_blueprint(status_blueprint)
+    
+    # delivery receipts
+    ses_callback_blueprint.before_request(requires_no_auth)
+    application.register_blueprint(ses_callback_blueprint)
 
     # delivery receipts
     # TODO: make sure research mode can still trigger sms callbacks, then re-enable this

app/celery/nightly_tasks.py

@@ -11,6 +11,7 @@ from sqlalchemy.exc import SQLAlchemyError
 
 from app import notify_celery, statsd_client, zendesk_client
 from app.aws import s3
+from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.config import QueueNames
 from app.cronitor import cronitor
 from app.dao.fact_processing_time_dao import insert_update_processing_time
@@ -37,9 +38,6 @@ from app.models import (
     FactProcessingTime,
     Notification,
 )
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 from app.utils import get_london_midnight_in_utc
 
 

app/celery/process_ses_receipts_tasks.py

@@ -6,75 +6,90 @@ from flask import current_app, json
 from sqlalchemy.orm.exc import NoResultFound
 
 from app import notify_celery, statsd_client
-from app.clients.email.aws_ses import get_aws_responses
+from app.celery.service_callback_tasks import (
+    create_complaint_callback_data,
+    create_delivery_status_callback_data,
+    send_complaint_to_service,
+    send_delivery_status_to_service,
+)
 from app.config import QueueNames
 from app.dao import notifications_dao
-from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING
-from app.notifications.notifications_ses_callback import (
-    _check_and_queue_complaint_callback_task,
-    check_and_queue_callback_task,
-    determine_notification_bounce_type,
-    handle_complaint,
+from app.dao.complaint_dao import save_complaint
+from app.dao.notifications_dao import dao_get_notification_history_by_reference
+from app.dao.service_callback_api_dao import (
+    get_service_complaint_callback_api_for_service,
+    get_service_delivery_status_callback_api_for_service,
 )
+from app.models import NOTIFICATION_PENDING, NOTIFICATION_SENDING, Complaint
 
 
 @notify_celery.task(bind=True, name="process-ses-result", max_retries=5, default_retry_delay=300)
 def process_ses_results(self, response):
     try:
-        ses_message = json.loads(response['Message'])
-        notification_type = ses_message['notificationType']
+        ses_message = json.loads(response["Message"])
+        notification_type = ses_message["notificationType"]
+        # TODO remove after smoke testing on prod is implemented
+        current_app.logger.info(f"Attempting to process SES delivery status message from SNS with type: {notification_type} and body: {ses_message}")
         bounce_message = None
 
         if notification_type == 'Bounce':
-            notification_type, bounce_message = determine_notification_bounce_type(notification_type, ses_message)
+            bounce_message = determine_notification_bounce_type(ses_message)
         elif notification_type == 'Complaint':
             _check_and_queue_complaint_callback_task(*handle_complaint(ses_message))
             return True
 
-        aws_response_dict = get_aws_responses(notification_type)
+        aws_response_dict = get_aws_responses(ses_message)
 
-        notification_status = aws_response_dict['notification_status']
-        reference = ses_message['mail']['messageId']
+        notification_status = aws_response_dict["notification_status"]
+        reference = ses_message["mail"]["messageId"]
 
         try:
-            notification = notifications_dao.dao_get_notification_or_history_by_reference(reference=reference)
+            notification = notifications_dao.dao_get_notification_by_reference(reference)
         except NoResultFound:
-            message_time = iso8601.parse_date(ses_message['mail']['timestamp']).replace(tzinfo=None)
+            message_time = iso8601.parse_date(ses_message["mail"]["timestamp"]).replace(tzinfo=None)
             if datetime.utcnow() - message_time < timedelta(minutes=5):
                 current_app.logger.info(
-                    f"notification not found for reference: {reference} (update to {notification_status}). "
+                    f"notification not found for reference: {reference} (while attempting update to {notification_status}). "
                     f"Callback may have arrived before notification was persisted to the DB. Adding task to retry queue"
                 )
                 self.retry(queue=QueueNames.RETRY)
             else:
                 current_app.logger.warning(
-                    f"notification not found for reference: {reference} (update to {notification_status})"
+                    "notification not found for reference: {} (while attempting update to {})".format(reference, notification_status)
                 )
             return
 
         if bounce_message:
             current_app.logger.info(f"SES bounce for notification ID {notification.id}: {bounce_message}")
 
-        if notification.status not in [NOTIFICATION_SENDING, NOTIFICATION_PENDING]:
+        if notification.status not in {NOTIFICATION_SENDING, NOTIFICATION_PENDING}:
             notifications_dao._duplicate_update_warning(
-                notification=notification,
-                status=notification_status
+                notification, 
+                notification_status
             )
             return
+
+        notifications_dao._update_notification_status(
+            notification=notification,
+            status=notification_status,
+            provider_response=aws_response_dict["provider_response"],
+        )
+
+        if not aws_response_dict["success"]:
+            current_app.logger.info(
+                "SES delivery failed: notification id {} and reference {} has error found. Status {}".format(
+                    notification.id, reference, aws_response_dict["message"]
+                )
+            )
         else:
-            notifications_dao.dao_update_notifications_by_reference(
-                references=[reference],
-                update_dict={'status': notification_status}
+            current_app.logger.info(
+                "SES callback return status of {} for notification: {}".format(notification_status, notification.id)
             )
 
-        statsd_client.incr('callback.ses.{}'.format(notification_status))
+        statsd_client.incr("callback.ses.{}".format(notification_status))
 
         if notification.sent_at:
-            statsd_client.timing_with_dates(
-                f'callback.ses.{notification_status}.elapsed-time',
-                datetime.utcnow(),
-                notification.sent_at
-            )
+            statsd_client.timing_with_dates("callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at)
 
         check_and_queue_callback_task(notification)
 
@@ -84,5 +99,133 @@ def process_ses_results(self, response):
         raise
 
     except Exception as e:
-        current_app.logger.exception('Error processing SES results: {}'.format(type(e)))
+        current_app.logger.exception("Error processing SES results: {}".format(type(e)))
         self.retry(queue=QueueNames.RETRY)
+
+def determine_notification_bounce_type(ses_message):
+    notification_type = ses_message["notificationType"]
+    if notification_type in ["Delivery", "Complaint"]:
+        return notification_type
+
+    if notification_type != "Bounce":
+        raise KeyError(f"Unhandled sns notification type {notification_type}")
+
+    remove_emails_from_bounce(ses_message)
+    current_app.logger.info("SES bounce dict: {}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
+    if ses_message["bounce"]["bounceType"] == "Permanent":
+        return "Permanent"
+    return "Temporary"
+
+def determine_notification_type(ses_message):
+    notification_type = ses_message["notificationType"]
+    if notification_type not in ["Bounce","Complaint","Delivery"]:
+        raise KeyError(f"Unhandled sns notification type {notification_type}")
+    if notification_type == 'Bounce':
+        return determine_notification_bounce_type(ses_message)
+    return notification_type
+
+def _determine_provider_response(ses_message):
+    if ses_message["notificationType"] != "Bounce":
+        return None
+
+    bounce_type = ses_message["bounce"]["bounceType"]
+    bounce_subtype = ses_message["bounce"]["bounceSubType"]
+
+    # See https://docs.aws.amazon.com/ses/latest/DeveloperGuide/event-publishing-retrieving-sns-contents.html
+    if bounce_type == "Permanent" and bounce_subtype == "Suppressed":
+        return "The email address is on our email provider suppression list"
+    elif bounce_type == "Permanent" and bounce_subtype == "OnAccountSuppressionList":
+        return "The email address is on the GC Notify suppression list"
+    elif bounce_type == "Transient" and bounce_subtype == "AttachmentRejected":
+        return "The email was rejected because of its attachments"
+
+    return None
+
+
+def get_aws_responses(ses_message):
+    status = determine_notification_type(ses_message)
+
+    base = {
+        "Permanent": {
+            "message": "Hard bounced",
+            "success": False,
+            "notification_status": "permanent-failure",
+        },
+        "Temporary": {
+            "message": "Soft bounced",
+            "success": False,
+            "notification_status": "temporary-failure",
+        },
+        "Delivery": {
+            "message": "Delivered",
+            "success": True,
+            "notification_status": "delivered",
+        },
+        "Complaint": {
+            "message": "Complaint",
+            "success": True,
+            "notification_status": "delivered",
+        },
+    }[status]
+
+    base["provider_response"] = _determine_provider_response(ses_message)
+
+    return base
+
+
+def handle_complaint(ses_message):
+    recipient_email = remove_emails_from_complaint(ses_message)[0]
+    current_app.logger.info("Complaint from SES: \n{}".format(json.dumps(ses_message).replace("{", "(").replace("}", ")")))
+    try:
+        reference = ses_message["mail"]["messageId"]
+    except KeyError as e:
+        current_app.logger.exception(f"Complaint from SES failed to get reference from message with error: {e}")
+        return
+    notification = dao_get_notification_history_by_reference(reference)
+    ses_complaint = ses_message.get("complaint", None)
+
+    complaint = Complaint(
+        notification_id=notification.id,
+        service_id=notification.service_id,
+        ses_feedback_id=ses_complaint.get("feedbackId", None) if ses_complaint else None,
+        complaint_type=ses_complaint.get("complaintFeedbackType", None) if ses_complaint else None,
+        complaint_date=ses_complaint.get("timestamp", None) if ses_complaint else None,
+    )
+    save_complaint(complaint)
+    return complaint, notification, recipient_email
+
+
+def remove_mail_headers(dict_to_edit):
+    if dict_to_edit["mail"].get("headers"):
+        dict_to_edit["mail"].pop("headers")
+    if dict_to_edit["mail"].get("commonHeaders"):
+        dict_to_edit["mail"].pop("commonHeaders")
+
+
+def remove_emails_from_bounce(bounce_dict):
+    remove_mail_headers(bounce_dict)
+    bounce_dict["mail"].pop("destination", None)
+    bounce_dict["bounce"].pop("bouncedRecipients", None)
+
+
+def remove_emails_from_complaint(complaint_dict):
+    remove_mail_headers(complaint_dict)
+    complaint_dict["complaint"].pop("complainedRecipients")
+    return complaint_dict["mail"].pop("destination")
+
+
+def check_and_queue_callback_task(notification):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
+        send_delivery_status_to_service.apply_async([str(notification.id), notification_data], queue=QueueNames.CALLBACKS)
+
+
+def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
+        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
+        

app/celery/process_sms_client_response_tasks.py

@@ -6,13 +6,11 @@ from flask import current_app
 from notifications_utils.template import SMSMessageTemplate
 
 from app import notify_celery, statsd_client
+from app.celery.process_ses_receipts_tasks import check_and_queue_callback_task
 from app.clients import ClientException
 from app.dao import notifications_dao
 from app.dao.templates_dao import dao_get_template_by_id
 from app.models import NOTIFICATION_PENDING
-from app.notifications.notifications_ses_callback import (
-    check_and_queue_callback_task,
-)
 
 sms_response_mapper = {
     # 'MMG': get_mmg_responses,

app/celery/service_callback_tasks.py

@@ -114,6 +114,7 @@ def create_delivery_status_callback_data(notification, service_callback_api):
         "notification_client_reference": notification.client_reference,
         "notification_to": notification.to,
         "notification_status": notification.status,
+        "notification_provider_response": notification.provider_response, # TODO do we have a test for provider_response
         "notification_created_at": notification.created_at.strftime(DATETIME_FORMAT),
         "notification_updated_at":
             notification.updated_at.strftime(DATETIME_FORMAT) if notification.updated_at else None,

app/celery/tasks.py

@@ -21,7 +21,7 @@ from app.dao.inbound_sms_dao import dao_get_inbound_sms_by_id
 from app.dao.jobs_dao import dao_get_job_by_id, dao_update_job
 from app.dao.notifications_dao import (
     dao_get_last_notification_added_for_job_id,
-    dao_get_notification_or_history_by_reference,
+    dao_get_notification_history_by_reference,
     dao_update_notifications_by_reference,
     get_notification_by_id,
     update_notification_status_by_reference,
@@ -548,7 +548,7 @@ def update_letter_notification(filename, temporary_failures, update):
 
 
 def check_billable_units(notification_update):
-    notification = dao_get_notification_or_history_by_reference(notification_update.reference)
+    notification = dao_get_notification_history_by_reference(notification_update.reference)
 
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} has {} billable_units but DVLA says page count is {}'.format(

app/cloudfoundry_config.py

@@ -13,10 +13,9 @@ def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
 
     # Postgres config
-    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres',
-                                                                                                      'postgresql')
+    os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['aws-rds'][0]['credentials']['uri'].replace('postgres','postgresql')
     # Redis config
-    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis', 'rediss')
+    os.environ['REDIS_URL'] = vcap_services['aws-elasticache-redis'][0]['credentials']['uri'].replace('redis://','rediss://')
 
     # CSV Upload Bucket Name
     bucket_service = find_by_service_name(

app/commands.py

@@ -460,7 +460,7 @@ def replay_daily_sorted_count_files(file_extension):
               help="Pipe delimited file containing organisation name, sector, crown, argeement_signed, domains")
 def populate_organisations_from_file(file_name):
     # [0] organisation name:: name of the organisation insert if organisation is missing.
-    # [1] sector:: Central | Local | NHS only
+    # [1] sector:: Federal | State only
     # [2] crown:: TRUE | FALSE only
     # [3] argeement_signed:: TRUE | FALSE
     # [4] domains:: comma separated list of domains related to the organisation

app/config.py

@@ -99,9 +99,16 @@ class Config(object):
     # Firetext API Key
     FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
     FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
+    
+    # Whether to ignore POSTs from SNS for replies to SMS we sent
+    RECEIVE_INBOUND_SMS = False
 
     # Use notify.sandbox.10x sending domain unless overwritten by environment
     NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
+    
+    # AWS SNS topics for delivery receipts
+    VALIDATE_SNS_TOPICS = True
+    VALID_SNS_TOPICS = ['notify_test_bounce', 'notify_test_success', 'notify_test_complaint', 'notify_test_sms_inbound']
 
     # URL of redis instance
     REDIS_URL = os.environ.get('REDIS_URL')
@@ -175,7 +182,7 @@ class Config(object):
     MOU_SIGNER_RECEIPT_TEMPLATE_ID = '4fd2e43c-309b-4e50-8fb8-1955852d9d71'
     MOU_SIGNED_ON_BEHALF_SIGNER_RECEIPT_TEMPLATE_ID = 'c20206d5-bf03-4002-9a90-37d5032d9e84'
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
-    NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
+    NOTIFY_INTERNATIONAL_SMS_SENDER = '18446120782'
     LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
     NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
     # we only need real email in Live environment (production)

app/dao/annual_billing_dao.py

@@ -55,46 +55,21 @@ def dao_get_all_free_sms_fragment_limit(service_id):
 
 def set_default_free_allowance_for_service(service, year_start=None):
     default_free_sms_fragment_limits = {
-        'central': {
+        'federal': {
             2020: 250_000,
             2021: 150_000,
             2022: 40_000,
         },
-        'local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_central': {
+        'state': {
             2020: 250_000,
             2021: 150_000,
             2022: 40_000,
         },
-        'nhs_local': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'nhs_gp': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
-        'emergency_service': {
-            2020: 25_000,
-            2021: 25_000,
-            2022: 20_000,
-        },
-        'school_or_college': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
         'other': {
-            2020: 25_000,
-            2021: 10_000,
-            2022: 10_000,
-        },
+            2020: 250_000,
+            2021: 150_000,
+            2022: 40_000,
+        }
     }
     if not year_start:
         year_start = get_current_financial_year_start_year()

app/dao/notifications_dao.py

@@ -87,16 +87,21 @@ def country_records_delivery(phone_prefix):
     dlr = INTERNATIONAL_BILLING_RATES[phone_prefix]['attributes']['dlr']
     return dlr and dlr.lower() == 'yes'
 
+def _decide_permanent_temporary_failure(current_status, status):
+    # If we go from pending to delivered we need to set failure type as temporary-failure
+    if current_status == NOTIFICATION_PENDING and status == NOTIFICATION_PERMANENT_FAILURE:
+        status = NOTIFICATION_TEMPORARY_FAILURE
+    return status
 
-def _update_notification_status(notification, status, detailed_status_code=None):
-    # status = _decide_permanent_temporary_failure(
-    #     status=status, notification=notification, detailed_status_code=detailed_status_code
-    # )
-    # notification.status = status
-    # dao_update_notification(notification)
+
+def _update_notification_status(notification, status, provider_response=None):
+    status = _decide_permanent_temporary_failure(current_status=notification.status, status=status)
+    notification.status = status
+    if provider_response:
+        notification.provider_response = provider_response
+    dao_update_notification(notification)
     return notification
 
-
 @autocommit
 def update_notification_status_by_id(notification_id, status, sent_by=None, detailed_status_code=None):
     notification = Notification.query.with_for_update().filter(Notification.id == notification_id).first()
@@ -587,17 +592,13 @@ def dao_get_notification_by_reference(reference):
     ).one()
 
 
-def dao_get_notification_or_history_by_reference(reference):
+def dao_get_notification_history_by_reference(reference):
     try:
         # This try except is necessary because in test keys and research mode does not create notification history.
         # Otherwise we could just search for the NotificationHistory object
-        return Notification.query.filter(
-            Notification.reference == reference
-        ).one()
+        return Notification.query.filter(Notification.reference == reference).one()
     except NoResultFound:
-        return NotificationHistory.query.filter(
-            NotificationHistory.reference == reference
-        ).one()
+        return NotificationHistory.query.filter(NotificationHistory.reference == reference).one()
 
 
 def dao_get_notifications_processing_time_stats(start_date, end_date):

app/dao/services_dao.py

@@ -16,14 +16,11 @@ from app.dao.service_sms_sender_dao import insert_service_sms_sender
 from app.dao.service_user_dao import dao_get_service_user
 from app.dao.template_folder_dao import dao_get_valid_template_folders_by_id
 from app.models import (
-    CROWN_ORGANISATION_TYPES,
     EMAIL_TYPE,
     INTERNATIONAL_LETTERS,
     INTERNATIONAL_SMS_TYPE,
     KEY_TYPE_TEST,
     LETTER_TYPE,
-    NHS_ORGANISATION_TYPES,
-    NON_CROWN_ORGANISATION_TYPES,
     NOTIFICATION_PERMANENT_FAILURE,
     SMS_TYPE,
     UPLOAD_LETTERS,
@@ -324,16 +321,8 @@ def dao_create_service(
         if organisation.letter_branding:
             service.letter_branding = organisation.letter_branding
 
-    elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
-        service.email_branding = dao_get_email_branding_by_name('NHS')
-        service.letter_branding = dao_get_letter_branding_by_name('NHS')
-
     if organisation:
         service.crown = organisation.crown
-    elif service.organisation_type in CROWN_ORGANISATION_TYPES:
-        service.crown = True
-    elif service.organisation_type in NON_CROWN_ORGANISATION_TYPES:
-        service.crown = False
     service.count_as_live = not user.platform_admin
 
     db.session.add(service)

app/delivery/send_to_providers.py

@@ -163,7 +163,10 @@ def update_notification_to_sending(notification, provider):
     notification.sent_at = datetime.utcnow()
     notification.sent_by = provider.name
     if notification.status not in NOTIFICATION_STATUS_TYPES_COMPLETED:
-        notification.status = NOTIFICATION_SENT if notification.international else NOTIFICATION_SENDING
+        # We currently have no callback method for SMS deliveries
+        # TODO create celery task to request SMS delivery receipts from cloudwatch api
+        notification.status = NOTIFICATION_SENT if notification.notification_type == "sms" else NOTIFICATION_SENDING
+    
     dao_update_notification(notification)
 
 

app/inbound_sms/rest.py

@@ -30,9 +30,10 @@ def post_inbound_sms_for_service(service_id):
     form = validate(request.get_json(), get_inbound_sms_for_service_schema)
     user_number = form.get('phone_number')
 
-    if user_number:
-        # we use this to normalise to an international phone number - but this may fail if it's an alphanumeric
-        user_number = try_validate_and_format_phone_number(user_number, international=True)
+    # TODO update this for US formatting
+    # if user_number:
+    #     # we use this to normalise to an international phone number - but this may fail if it's an alphanumeric
+    #     user_number = try_validate_and_format_phone_number(user_number, international=True)
 
     inbound_data_retention = fetch_service_data_retention_by_notification_type(service_id, 'sms')
     limit_days = inbound_data_retention.days_of_retention if inbound_data_retention else 7

app/models.py

@@ -348,13 +348,9 @@ class Domain(db.Model):
 
 
 ORGANISATION_TYPES = [
-    "central", "local", "nhs_central", "nhs_local", "nhs_gp", "emergency_service", "school_or_college", "other",
+    "federal", "state", "other"
 ]
 
-CROWN_ORGANISATION_TYPES = ["nhs_central"]
-NON_CROWN_ORGANISATION_TYPES = ["local", "nhs_local", "nhs_gp", "emergency_service", "school_or_college"]
-NHS_ORGANISATION_TYPES = ["nhs_central", "nhs_local", "nhs_gp"]
-
 
 class OrganisationTypes(db.Model):
     __tablename__ = 'organisation_types'
@@ -1488,6 +1484,8 @@ class Notification(db.Model):
     document_download_count = db.Column(db.Integer, nullable=True)
 
     postage = db.Column(db.String, nullable=True)
+    provider_response = db.Column(db.Text, nullable=True)
+    # queue_name = db.Column(db.Text, nullable=True)
 
     __table_args__ = (
         db.ForeignKeyConstraint(
@@ -1690,6 +1688,7 @@ class Notification(db.Model):
             "postcode": None,
             "type": self.notification_type,
             "status": self.get_letter_status() if self.notification_type == LETTER_TYPE else self.status,
+            "provider_response": self.provider_response,
             "template": template_dict,
             "body": self.content,
             "subject": self.subject,

app/notifications/notifications_ses_callback.py

@@ -1,85 +1,31 @@
-from flask import current_app
+from datetime import timedelta
 
-from app.celery.service_callback_tasks import (
-    create_complaint_callback_data,
-    create_delivery_status_callback_data,
-    send_complaint_to_service,
-    send_delivery_status_to_service,
-)
+from flask import Blueprint, jsonify, request
+
+from app.celery.process_ses_receipts_tasks import process_ses_results
 from app.config import QueueNames
-from app.dao.complaint_dao import save_complaint
-from app.dao.notifications_dao import (
-    dao_get_notification_or_history_by_reference,
-)
-from app.dao.service_callback_api_dao import (
-    get_service_complaint_callback_api_for_service,
-    get_service_delivery_status_callback_api_for_service,
-)
-from app.models import Complaint
+from app.errors import InvalidRequest
+from app.notifications.sns_handlers import sns_notification_handler
 
+ses_callback_blueprint = Blueprint('notifications_ses_callback', __name__)
+DEFAULT_MAX_AGE = timedelta(days=10000)
 
-def determine_notification_bounce_type(notification_type, ses_message):
-    remove_emails_from_bounce(ses_message)
-    if ses_message['bounce']['bounceType'] == 'Permanent':
-        notification_type = ses_message['bounce']['bounceType']  # permanent or not
-    else:
-        notification_type = 'Temporary'
-    return notification_type, ses_message
-
-
-def handle_complaint(ses_message):
-    recipient_email = remove_emails_from_complaint(ses_message)[0]
-    current_app.logger.info("Complaint from SES: \n{}".format(ses_message))
+# 400 counts as a permanent failure so SNS will not retry.
+# 500 counts as a failed delivery attempt so SNS will retry.
+# See https://docs.aws.amazon.com/sns/latest/dg/DeliveryPolicies.html#DeliveryPolicies
+@ses_callback_blueprint.route('/notifications/email/ses', methods=['POST'])
+def email_ses_callback_handler():
     try:
-        reference = ses_message['mail']['messageId']
-    except KeyError as e:
-        current_app.logger.exception("Complaint from SES failed to get reference from message", e)
-        return
-    notification = dao_get_notification_or_history_by_reference(reference)
-    ses_complaint = ses_message.get('complaint', None)
+        data = sns_notification_handler(request.data, request.headers)
+    except InvalidRequest as e:
+        return jsonify(
+            result="error", message=str(e.message)
+        ), e.status_code
+    
+    message = data.get("Message")
+    if "mail" in message:
+        process_ses_results.apply_async([{"Message": message}], queue=QueueNames.NOTIFY)
 
-    complaint = Complaint(
-        notification_id=notification.id,
-        service_id=notification.service_id,
-        ses_feedback_id=ses_complaint.get('feedbackId', None) if ses_complaint else None,
-        complaint_type=ses_complaint.get('complaintFeedbackType', None) if ses_complaint else None,
-        complaint_date=ses_complaint.get('timestamp', None) if ses_complaint else None
-    )
-    save_complaint(complaint)
-    return complaint, notification, recipient_email
-
-
-def remove_mail_headers(dict_to_edit):
-    if dict_to_edit['mail'].get('headers'):
-        dict_to_edit['mail'].pop('headers')
-    if dict_to_edit['mail'].get('commonHeaders'):
-        dict_to_edit['mail'].pop('commonHeaders')
-
-
-def remove_emails_from_bounce(bounce_dict):
-    remove_mail_headers(bounce_dict)
-    bounce_dict['mail'].pop('destination')
-    bounce_dict['bounce'].pop('bouncedRecipients')
-
-
-def remove_emails_from_complaint(complaint_dict):
-    remove_mail_headers(complaint_dict)
-    complaint_dict['complaint'].pop('complainedRecipients')
-    return complaint_dict['mail'].pop('destination')
-
-
-def check_and_queue_callback_task(notification):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async([str(notification.id), notification_data],
-                                                    queue=QueueNames.CALLBACKS)
-
-
-def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
-    # queue callback task only if the service_callback_api exists
-    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
-    if service_callback_api:
-        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
-        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
+    return jsonify(
+        result="success", message="SES-SNS callback succeeded"
+    ), 200

app/notifications/notifications_sms_callback.py

@@ -9,6 +9,7 @@ from app.errors import InvalidRequest, register_errors
 sms_callback_blueprint = Blueprint("sms_callback", __name__, url_prefix="/notifications/sms")
 register_errors(sms_callback_blueprint)
 
+# TODO SNS SMS delivery receipts delivered here
 
 # @sms_callback_blueprint.route('/mmg', methods=['POST'])
 # def process_mmg_response():

app/notifications/receive_notifications.py

@@ -2,7 +2,7 @@ from datetime import datetime
 from urllib.parse import unquote
 
 import iso8601
-from flask import Blueprint, abort, current_app, jsonify, request
+from flask import Blueprint, abort, current_app, json, jsonify, request
 from gds_metrics.metrics import Counter
 from notifications_utils.recipients import try_validate_and_format_phone_number
 
@@ -10,8 +10,9 @@ from app.celery import tasks
 from app.config import QueueNames
 from app.dao.inbound_sms_dao import dao_create_inbound_sms
 from app.dao.services_dao import dao_fetch_service_by_inbound_number
-from app.errors import register_errors
+from app.errors import InvalidRequest, register_errors
 from app.models import INBOUND_SMS_TYPE, SMS_TYPE, InboundSms
+from app.notifications.sns_handlers import sns_notification_handler
 
 receive_notifications_blueprint = Blueprint('receive_notifications', __name__)
 register_errors(receive_notifications_blueprint)
@@ -23,6 +24,71 @@ INBOUND_SMS_COUNTER = Counter(
     ['provider']
 )
 
+@receive_notifications_blueprint.route('/notifications/sms/receive/sns', methods=['POST'])
+def receive_sns_sms():
+    """
+    Expected value of the 'Message' key in the incoming payload from SNS
+    {
+        "originationNumber":"+14255550182",
+        "destinationNumber":"+12125550101",
+        "messageKeyword":"JOIN", # unique to our sending number
+        "messageBody":"EXAMPLE",
+        "inboundMessageId":"cae173d2-66b9-564c-8309-21f858e9fb84",
+        "previousPublishedMessageId":"wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+    }
+    """
+    
+    # Whether or not to ignore inbound SMS replies
+    if not current_app.config['RECEIVE_INBOUND_SMS']:
+        return jsonify(
+            result="success", message="SMS-SNS callback succeeded"
+        ), 200
+    
+    try:
+        post_data = sns_notification_handler(request.data, request.headers)
+    except Exception as e:
+        raise InvalidRequest(f"SMS-SNS callback failed with error: {e}", 400)
+
+    message = json.loads(post_data.get("Message"))
+    # TODO wrap this up
+    if "inboundMessageId" in message:
+        # TODO use standard formatting we use for all US numbers
+        inbound_number = message['destinationNumber'].replace('+','')
+
+        service = fetch_potential_service(inbound_number, 'sns')
+        if not service:
+            # since this is an issue with our service <-> number mapping, or no inbound_sms service permission
+            # we should still tell SNS that we received it successfully
+            current_app.logger.warning(f"Mapping between service and inbound number: {inbound_number} is broken, or service does not have permission to receive inbound sms")
+            return jsonify(
+                result="success", message="SMS-SNS callback succeeded"
+            ), 200
+
+        INBOUND_SMS_COUNTER.labels("sns").inc()
+
+        content = message.get("messageBody")
+        from_number = message.get('originationNumber')
+        provider_ref = message.get('inboundMessageId')
+        date_received = post_data.get('Timestamp')
+        provider_name = "sns"
+
+        inbound = create_inbound_sms_object(service,
+                                            content=content,
+                                            from_number=from_number,
+                                            provider_ref=provider_ref,
+                                            date_received=date_received,
+                                            provider_name=provider_name)
+
+        # TODO ensure inbound sms callback endpoints are accessible and functioning for notify api users, then uncomment the task below
+        tasks.send_inbound_sms_to_service.apply_async([str(inbound.id), str(service.id)], queue=QueueNames.NOTIFY)
+
+        current_app.logger.debug(
+            '{} received inbound SMS with reference {} from SNS'.format(service.id, inbound.provider_reference))
+
+    return jsonify(
+        result="success", message="SMS-SNS callback succeeded"
+    ), 200
+
 
 @receive_notifications_blueprint.route('/notifications/sms/receive/mmg', methods=['POST'])
 def receive_mmg_sms():

app/notifications/sns_cert_validator.py

@@ -0,0 +1,110 @@
+import base64
+import re
+from urllib.parse import urlparse
+
+import oscrypto.asymmetric
+import oscrypto.errors
+import requests
+import six
+
+from app import redis_store
+from app.config import Config
+
+VALIDATE_SNS_TOPICS = Config.VALIDATE_SNS_TOPICS
+VALID_SNS_TOPICS = Config.VALID_SNS_TOPICS
+
+
+_signing_cert_cache = {}
+_cert_url_re = re.compile(
+    r'sns\.([a-z]{1,3}-[a-z]+-[0-9]{1,2})\.amazonaws\.com',
+)
+
+class ValidationError(Exception):
+    """
+    ValidationError. Raised when a message fails integrity checks.
+    """
+
+
+def get_certificate(url):
+    res = redis_store.get(url)
+    if res is not None:
+        return res
+    res = requests.get(url).text
+    redis_store.set(url, res, ex=60 * 60)  # 60 minutes
+    _signing_cert_cache[url] = res
+    return res
+
+
+def validate_arn(sns_payload):
+    if VALIDATE_SNS_TOPICS:
+        arn = sns_payload.get('TopicArn')
+        topic_name = arn.split(':')[5]
+        if topic_name not in VALID_SNS_TOPICS:
+            raise ValidationError("Invalid Topic Name")
+
+
+def get_string_to_sign(sns_payload):
+    payload_type = sns_payload.get('Type')
+    if payload_type in ['SubscriptionConfirmation', 'UnsubscribeConfirmation']:
+        fields = ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']
+    elif payload_type == 'Notification':
+        fields = ['Message', 'MessageId', 'Subject', 'Timestamp', 'TopicArn', 'Type']
+    else:
+        raise ValidationError("Unexpected Message Type")
+
+    string_to_sign = ''
+    for field in fields:
+        field_value = sns_payload.get(field)
+        if not isinstance(field_value, str):
+            if field == 'Subject' and field_value == None:
+                continue
+            raise ValidationError(f"In {field}, found non-string value: {field_value}")
+        string_to_sign += field + '\n' + field_value + '\n'
+    if isinstance(string_to_sign, six.text_type):
+        string_to_sign = string_to_sign.encode()
+    return string_to_sign
+
+
+def validate_sns_cert(sns_payload):
+    """
+    Adapted from the solution posted at
+    https://github.com/boto/boto3/issues/2508#issuecomment-992931814
+    Modified to swap m2crypto for oscrypto
+    """
+    if not isinstance(sns_payload, dict):
+        raise ValidationError("Unexpected message type {!r}".format(type(sns_payload).__name__))
+
+    # Amazon SNS currently supports signature version 1.
+    if sns_payload.get('SignatureVersion') != '1':
+        raise ValidationError("Wrong Signature Version (expected 1)")
+    
+    validate_arn(sns_payload)
+    
+    string_to_sign = get_string_to_sign(sns_payload)
+
+    # Key signing cert url via Lambda and via webhook are slightly different
+    signing_cert_url = sns_payload.get('SigningCertUrl') if 'SigningCertUrl' in sns_payload else sns_payload.get('SigningCertURL')
+    if not isinstance(signing_cert_url, str):
+        raise ValidationError("Signing cert url must be a string")
+    cert_scheme, cert_netloc, *_ = urlparse(signing_cert_url)
+    if cert_scheme != 'https' or not re.match(_cert_url_re, cert_netloc):
+        raise ValidationError("Cert does not appear to be from AWS")
+    
+    certificate = _signing_cert_cache.get(signing_cert_url)
+    if certificate is None:
+        certificate = get_certificate(signing_cert_url)
+    if isinstance(certificate, six.text_type):
+        certificate = certificate.encode()
+    
+    signature = base64.b64decode(sns_payload["Signature"])
+
+    try:
+        oscrypto.asymmetric.rsa_pkcs1v15_verify(
+            oscrypto.asymmetric.load_certificate(certificate),
+            signature,
+            string_to_sign,
+            "sha1"
+        )
+        return True
+    except oscrypto.errors.SignatureError:
+        raise ValidationError("Invalid signature")

app/notifications/sns_handlers.py

@@ -0,0 +1,66 @@
+import enum
+from datetime import timedelta
+from json import decoder
+
+import requests
+from flask import current_app, json
+
+from app.errors import InvalidRequest
+from app.notifications.sns_cert_validator import validate_sns_cert
+
+DEFAULT_MAX_AGE = timedelta(days=10000)
+
+
+class SNSMessageType(enum.Enum):
+    SubscriptionConfirmation = 'SubscriptionConfirmation'
+    Notification = 'Notification'
+    UnsubscribeConfirmation = 'UnsubscribeConfirmation'
+
+
+class InvalidMessageTypeException(Exception):
+    pass
+
+
+def verify_message_type(message_type: str):
+    try:
+        SNSMessageType(message_type)
+    except ValueError:
+        raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
+
+
+def sns_notification_handler(data, headers):
+    message_type = headers.get('x-amz-sns-message-type')
+    try:
+        verify_message_type(message_type)
+    except InvalidMessageTypeException:
+        current_app.logger.exception(f"Response headers: {headers}\nResponse data: {data}")
+        raise InvalidRequest("SES-SNS callback failed: invalid message type", 400)
+
+    try:
+        message = json.loads(data.decode('utf-8'))
+    except decoder.JSONDecodeError:
+        current_app.logger.exception(f"Response headers: {headers}\nResponse data: {data}")
+        raise InvalidRequest("SES-SNS callback failed: invalid JSON given", 400)
+
+    try:
+        validate_sns_cert(message)
+    except Exception as e:
+        current_app.logger.error(f"SES-SNS callback failed: validation failed with error: Signature validation failed with error {e}")
+        raise InvalidRequest("SES-SNS callback failed: validation failed", 400)
+
+    if message.get('Type') == 'SubscriptionConfirmation':
+        # NOTE once a request is sent to SubscribeURL, AWS considers Notify a confirmed subscriber to this topic
+        url = message.get('SubscribeUrl') if 'SubscribeUrl' in message else message.get('SubscribeURL')
+        response = requests.get(url)
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            current_app.logger.warning(f"Attempt to raise_for_status()SubscriptionConfirmation Type message files for response: {response.text} with error {e}")
+            raise InvalidRequest("SES-SNS callback failed: attempt to raise_for_status()SubscriptionConfirmation Type message failed", 400)
+        current_app.logger.info("SES-SNS auto-confirm subscription callback succeeded")
+        return message
+
+    # TODO remove after smoke testing on prod is implemented
+    current_app.logger.info(f"SNS message: {message} is a valid message. Attempting to process it now.")
+    
+    return message

app/organisation/rest.py

@@ -22,7 +22,7 @@ from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
 from app.errors import InvalidRequest, register_errors
-from app.models import KEY_TYPE_NORMAL, NHS_ORGANISATION_TYPES, Organisation
+from app.models import KEY_TYPE_NORMAL, Organisation
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
@@ -93,9 +93,6 @@ def create_organisation():
 
     validate(data, post_create_organisation_schema)
 
-    if data["organisation_type"] in NHS_ORGANISATION_TYPES:
-        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
-
     organisation = Organisation(**data)
     dao_create_organisation(organisation)
     return jsonify(organisation.serialize()), 201
@@ -108,9 +105,6 @@ def update_organisation(organisation_id):
 
     organisation = dao_get_organisation_by_id(organisation_id)
 
-    if data.get('organisation_type') in NHS_ORGANISATION_TYPES and not organisation.email_branding_id:
-        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
-
     result = dao_update_organisation(organisation_id, **data)
 
     if data.get('agreement_signed') is True:

deploy-config/production.yml

@@ -0,0 +1,5 @@
+env: production
+web_instances: 2
+web_memory: 1G
+worker_instances: 1
+worker_memory: 512M

deploy-config/staging.yml

@@ -0,0 +1,5 @@
+env: staging
+web_instances: 1
+web_memory: 1G
+worker_instances: 1
+worker_memory: 512M

devcontainer-api/.devcontainer.json

@@ -16,7 +16,8 @@
 		"python.defaultInterpreterPath": "/usr/bin/python3",
 		"python.linting.pylintPath": "/usr/local/share/pip-global/bin/pylint",
 		"python.analysis.extraPaths": [
-			"/home/vscode/.local/lib/python3.9/site-packages"
+			"/home/vscode/.local/lib/python3.9/site-packages",
+			"/home/vscode/.local/bin"
 		]
 	},
 	"features": {

manifest.yml

@@ -17,6 +17,16 @@ applications:
       - notifications-api-csv-upload-bucket-((env))
       - notifications-api-contact-list-bucket-((env))
 
+    processes:
+      - type: web
+        instances: ((web_instances))
+        memory: ((web_memory))
+        command: ./scripts/migrate_and_run_web.sh
+      - type: worker
+        instances: ((worker_instances))
+        memory: ((worker_memory))
+        command: ./scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4
+
     env:
       NOTIFY_APP_NAME: api
       NOTIFY_LOG_PATH: /home/vcap/logs/app.log

migrations/versions/0375_fix_service_name.py

@@ -6,17 +6,13 @@ Create Date: 2022-08-29 11:04:15.888017
 
 """
 
-# revision identifiers, used by Alembic.
-from datetime import datetime
-
 revision = '0375_fix_service_name'
 down_revision = '0374_fix_reg_template_history'
 
 from alembic import op
-import sqlalchemy as sa
+from flask import current_app
 
-service_id = 'd6aa2c68-a2d9-4437-ab19-3ae8eb202553'
-user_id= '6af522d0-2915-4e52-83a3-3690455a5fe6'
+service_id = current_app.config['NOTIFY_SERVICE_ID']
 
 def upgrade():
     op.get_bind()

migrations/versions/0376_add_provider_response.py

@@ -0,0 +1,30 @@
+"""empty message
+
+Revision ID: 0376_add_provider_response
+Revises: 0375_fix_service_name
+Create Date: 2022-09-14 11:04:15.888017
+
+"""
+# revision identifiers, used by Alembic.
+from datetime import datetime
+
+revision = '0376_add_provider_response'
+down_revision = '0375_fix_service_name'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('notifications', sa.Column('provider_response', sa.Text(), nullable=True))
+    op.add_column('notifications', sa.Column('queue_name', sa.Text(), nullable=True))
+    ### end Alembic commands ###
+
+
+def downgrade():
+    ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('notifications', 'provider_response')
+    op.drop_column('notifications', 'queue_name')
+    ### end Alembic commands ###
+    

migrations/versions/0377_add_inbound_sms_number.py

@@ -0,0 +1,60 @@
+"""empty message
+
+Revision ID: 0377_add_inbound_sms_number
+Revises: 0376_add_provider_response
+Create Date: 2022-09-30 11:04:15.888017
+
+"""
+import uuid
+
+from alembic import op
+from flask import current_app
+
+
+revision = '0377_add_inbound_sms_number'
+down_revision = '0376_add_provider_response'
+
+INBOUND_NUMBER_ID = '9b5bc009-b847-4b1f-8a54-f3b5f95cff18'
+INBOUND_NUMBER = current_app.config['NOTIFY_INTERNATIONAL_SMS_SENDER']
+DEFAULT_SERVICE_ID = current_app.config['NOTIFY_SERVICE_ID']
+
+def upgrade():
+    op.get_bind()
+
+    # delete the previous inbound_number with mmg as provider
+    table_name = 'inbound_numbers'
+    select_by_col = 'number'
+    select_by_val = INBOUND_NUMBER
+    op.execute(f"delete from {table_name} where {select_by_col} = '{select_by_val}'")
+    
+    # add the inbound number for the default service to inbound_numbers
+    table_name = 'inbound_numbers'
+    provider = 'sns'
+    active = 'true'
+    op.execute(f"insert into {table_name} (id, number, provider, service_id, active, created_at) VALUES('{INBOUND_NUMBER_ID}', '{INBOUND_NUMBER}', '{provider}','{DEFAULT_SERVICE_ID}', '{active}', 'now()')")
+    
+    # add the inbound number for the default service to service_sms_senders
+    table_name = 'service_sms_senders'
+    sms_sender = INBOUND_NUMBER
+    select_by_col = 'id'
+    select_by_val = '286d6176-adbe-7ea7-ba26-b7606ee5e2a4'
+    op.execute(f"update {table_name} set {'sms_sender'}='{sms_sender}' where {select_by_col} = '{select_by_val}'")
+    
+    # add the inbound number for the default service to inbound_numbers
+    table_name = 'service_permissions'
+    permission = 'inbound_sms'
+    active = 'true'
+    op.execute(f"insert into {table_name} (service_id, permission, created_at) VALUES('{DEFAULT_SERVICE_ID}', '{permission}', 'now()')")
+    # pass
+
+
+def downgrade():
+    delete_sms_sender = f"delete from service_sms_senders where inbound_number_id = '{INBOUND_NUMBER_ID}'"
+    delete_inbound_number = f"delete from inbound_numbers where number = '{INBOUND_NUMBER}'"
+    delete_service_inbound_permission = f"delete from service_permissions where service_id = '{DEFAULT_SERVICE_ID}' and permission = 'inbound_sms'"
+    recreate_mmg_inbound_number = f"insert into inbound_numbers (id, number, provider, service_id, active, created_at) VALUES('d7aea27f-340b-4428-9b20-4470dd978bda', '{INBOUND_NUMBER}', 'mmg', 'null', 'false', 'now()')"
+    op.execute(delete_sms_sender)
+    op.execute(delete_inbound_number)
+    op.execute(delete_service_inbound_permission)
+    op.execute(recreate_mmg_inbound_number)
+    # pass

migrations/versions/0378_add_org_names.py

@@ -0,0 +1,53 @@
+"""
+
+Revision ID: 0378_add_org_names
+Revises: 0377_add_inbound_sms_number
+Create Date: 2022-09-23 20:04:00.766980
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0378_add_org_names'
+down_revision = '0377_add_inbound_sms_number'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.get_bind()
+
+    # bluntly swap out data
+    op.execute("INSERT INTO organisation_types VALUES ('state','f','250000'),('federal','f','250000');")
+    op.execute("UPDATE services SET organisation_type = 'federal';")
+    op.execute("UPDATE organisation SET organisation_type = 'federal';")
+    op.execute("UPDATE services_history SET organisation_type = 'federal';")
+    
+    # remove uk values
+    service_delete = """DELETE FROM organisation_types WHERE name IN 
+                    ('central','local','nhs','nhs_central','nhs_local','emergency_service','school_or_college','nhs_gp')
+                """
+    op.execute(service_delete)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    service_insert = """INSERT INTO organisation_types VALUES 
+                    ('central','','250000')
+                    ('local','f','25000')
+                    ('nhs','','25000')
+                    ('nhs_central','t','250000')
+                    ('nhs_local','f','25000')
+                    ('emergency_service','f','25000')
+                    ('school_or_college','f','25000')
+                    ('nhs_gp','f','25000')
+                """
+    op.execute(service_insert)
+    op.execute("UPDATE services SET organisation_type = 'central';")
+    op.execute("UPDATE organisation SET organisation_type = 'central';")
+    op.execute("UPDATE services_history SET organisation_type = 'central';")
+    op.execute("DELETE FROM organisation_types WHERE name IN ('federal','state')")
+
+    # ### end Alembic commands ###

requirements.in

@@ -25,6 +25,7 @@ lxml==4.9.1
 defusedxml==0.7.1
 Werkzeug==2.1.1
 python-dotenv==0.20.0
+oscrypto==1.3.0
 
 notifications-python-client==6.3.0
 

requirements.txt

@@ -10,6 +10,8 @@ amqp==5.1.1
     # via kombu
 arrow==1.2.2
     # via isoduration
+asn1crypto==1.5.1
+    # via oscrypto
 async-timeout==4.0.2
     # via redis
 attrs==21.4.0
@@ -169,6 +171,8 @@ notifications-utils @ git+https://github.com/GSA/notifications-utils.git
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
+oscrypto==1.3.0
+    # via -r requirements.in
 packaging==21.3
     # via
     #   bleach

scripts/migrate_and_run_web.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ $CF_INSTANCE_INDEX -eq 0 ]]; then
+  flask db upgrade
+fi
+
+${HOME}/scripts/run_app_paas.sh gunicorn -c ${HOME}/gunicorn_config.py application

scripts/paas_app_wrapper.sh

@@ -1,63 +0,0 @@
-#!/bin/bash
-case $NOTIFY_APP_NAME in
-  api)
-    unset GUNICORN_CMD_ARGS
-    exec scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
-    ;;
-  delivery-worker-retry-tasks)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q retry-tasks 2> /dev/null
-    ;;
-  delivery-worker-letters)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null
-    ;;
-  delivery-worker-jobs)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q database-tasks,job-tasks 2> /dev/null
-    ;;
-  delivery-worker-research)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q research-mode-tasks 2> /dev/null
-    ;;
-  delivery-worker-sender)
-    exec scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 4 -A run_celery.notify_celery --loglevel=INFO \
-    --logfile=/dev/null --pidfile=/tmp/celery%N.pid -Q send-sms-tasks,send-email-tasks
-    ;;
-  delivery-worker-periodic)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
-    -Q periodic-tasks 2> /dev/null
-    ;;
-  delivery-worker-reporting)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q reporting-tasks 2> /dev/null
-    ;;
-  delivery-worker-priority)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q priority-tasks 2> /dev/null
-    ;;
-  # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
-  delivery-worker-internal)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q notify-internal-tasks 2> /dev/null
-    ;;
-  delivery-worker-receipts)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q ses-callbacks,sms-callbacks 2> /dev/null
-    ;;
-  delivery-worker-service-callbacks)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q service-callbacks,service-callbacks-retry 2> /dev/null
-    ;;
-  delivery-worker-save-api-notifications)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
-    -Q save-api-email-tasks,save-api-sms-tasks 2> /dev/null
-    ;;
-  delivery-celery-beat)
-    exec scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
-    ;;
-  *)
-    echo "Unknown notify_app_name $NOTIFY_APP_NAME"
-    exit 1
-    ;;
-esac

scripts/run_multi_worker_app_paas.sh

@@ -1,167 +0,0 @@
-#!/bin/bash
-
-set -e -o pipefail
-
-TERMINATE_TIMEOUT=9
-MAX_DISK_SPACE_USAGE=75
-readonly LOGS_DIR="/home/vcap/logs"
-
-function check_params {
-  if [ -z "${NOTIFY_APP_NAME}" ]; then
-    echo "You must set NOTIFY_APP_NAME"
-    exit 1
-  fi
-
-  if [ -z "${CW_APP_NAME}" ]; then
-    CW_APP_NAME=${NOTIFY_APP_NAME}
-  fi
-}
-
-function configure_aws_logs {
-  # create files so that aws logs agent doesn't complain
-  touch ${LOGS_DIR}/gunicorn_error.log
-  touch ${LOGS_DIR}/app.log.json
-
-  aws configure set plugins.cwlogs cwlogs
-
-  cat > /home/vcap/app/awslogs.conf << EOF
-[general]
-state_file = ${LOGS_DIR}/awslogs-state
-
-[${LOGS_DIR}/app.log]
-file = ${LOGS_DIR}/app.log.json
-log_group_name = paas-${CW_APP_NAME}-application
-log_stream_name = {hostname}
-
-[${LOGS_DIR}/gunicorn_error.log]
-file = ${LOGS_DIR}/gunicorn_error.log
-log_group_name = paas-${CW_APP_NAME}-gunicorn
-log_stream_name = {hostname}
-EOF
-}
-
-# For every PID, check if it's still running. if it is, send the sigterm. then wait 9 seconds before sending sigkill
-function on_exit {
-  echo "multi worker app exiting"
-  wait_time=0
-
-  send_signal_to_celery_processes TERM
-
-  # check if the apps are still running every second
-  while [[ "$wait_time" -le "$TERMINATE_TIMEOUT" ]]; do
-    get_celery_pids
-    ensure_celery_is_running
-    let wait_time=wait_time+1
-    sleep 1
-  done
-
-  send_signal_to_celery_processes KILL
-}
-
-function check_disk_space {
-    # get something like:
-    #
-    # Filesystem     Use%
-    # overlay         56%
-    # tmpfs            0%
-    #
-    # and only keep '56'
-    SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
-
-    if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
-        echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
-        echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
-        exit
-    fi
-}
-
-function get_celery_pids {
-  # get the PIDs of the process whose parent is the root process
-  # print only pid and their command, get the ones with "celery" in their name
-  # and keep only these PIDs
-
-  set +o pipefail # so grep returning no matches does not premature fail pipe
-  APP_PIDS=$(pgrep -P 1 | xargs ps -o pid=,command= -p | grep celery | cut -f1 -d/)
-  set -o pipefail # pipefail should be set everywhere else
-}
-
-function send_signal_to_celery_processes {
-  # refresh pids to account for the case that some workers may have terminated but others not
-  get_celery_pids
-  # send signal to all remaining apps
-  echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: "
-  echo ${APP_PIDS} | xargs kill -s ${1}
-}
-
-function start_application {
-  echo "Starting application..."
-  eval "$@"
-  get_celery_pids
-  echo "Application process pids: "${APP_PIDS}
-}
-
-function start_aws_logs_agent {
-  echo "Starting aws logs agent..."
-  exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf &
-  AWSLOGS_AGENT_PID=$!
-  echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
-}
-
-function start_logs_tail {
-  echo "Starting logs tail..."
-  exec tail -n0 -f ${LOGS_DIR}/app.log.json &
-  LOGS_TAIL_PID=$!
-  echo "tail pid: ${LOGS_TAIL_PID}"
-}
-
-function ensure_celery_is_running {
-  if [ "${APP_PIDS}" = "" ]; then
-    echo "There are no celery processes running, this container is bad"
-
-    echo "Exporting CF information for diagnosis"
-
-    env | grep CF
-
-    echo "Sleeping 15 seconds for logs to get shipped"
-
-    sleep 15
-
-    echo "Killing awslogs_agent and tail"
-    kill -9 ${AWSLOGS_AGENT_PID}
-    kill -9 ${LOGS_TAIL_PID}
-
-    exit 1
-  fi
-}
-
-function run {
-  while true; do
-    check_disk_space
-    get_celery_pids
-
-    ensure_celery_is_running
-
-    for APP_PID in ${APP_PIDS}; do
-        kill -0 ${APP_PID} 2&>/dev/null || return 1
-    done
-    kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
-    kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail
-    sleep 1
-  done
-}
-
-echo "Run script pid: $$"
-
-check_params
-
-trap "on_exit" EXIT TERM
-
-configure_aws_logs
-
-# The application has to start first!
-start_application "$@"
-
-start_aws_logs_agent
-start_logs_tail
-
-run

tests/app/billing/test_rest.py

@@ -108,7 +108,7 @@ def test_get_free_sms_fragment_limit_current_year_creates_new_row_if_annual_bill
     )
 
     assert json_response['financial_year_start'] == 2021
-    assert json_response['free_sms_fragment_limit'] == 10000  # based on other organisation type
+    assert json_response['free_sms_fragment_limit'] == 150000  # based on other organisation type
 
 
 def test_update_free_sms_fragment_limit_data(client, sample_service):

tests/app/celery/test_process_ses_receipts_tasks.py

@@ -4,18 +4,22 @@ from datetime import datetime
 from freezegun import freeze_time
 
 from app import encryption, statsd_client
-from app.celery.process_ses_receipts_tasks import process_ses_results
+from app.celery.process_ses_receipts_tasks import (
+    process_ses_results,
+    remove_emails_from_bounce,
+    remove_emails_from_complaint,
+)
 from app.celery.research_mode_tasks import (
     ses_hard_bounce_callback,
     ses_notification_callback,
     ses_soft_bounce_callback,
 )
+from app.celery.service_callback_tasks import (
+    create_delivery_status_callback_data,
+)
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import Complaint, Notification
-from app.notifications.notifications_ses_callback import (
-    remove_emails_from_bounce,
-    remove_emails_from_complaint,
-)
+from tests.app.conftest import create_sample_notification
 from tests.app.db import (
     create_notification,
     create_service_callback_api,
@@ -23,6 +27,79 @@ from tests.app.db import (
 )
 
 
+def test_notifications_ses_400_with_invalid_header(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json')]
+    )
+    assert response.status_code == 400
+
+
+def test_notifications_ses_400_with_invalid_message_type(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'foo')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: invalid message type" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_400_with_invalid_json(client):
+    data = "FOOO"
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: invalid JSON given" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_400_with_certificate(client):
+    data = json.dumps({"foo": "bar"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+    assert response.status_code == 400
+    assert "SES-SNS callback failed: validation failed" in response.get_data(as_text=True)
+
+
+def test_notifications_ses_200_autoconfirms_subscription(client, mocker):
+    mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True)
+    requests_mock = mocker.patch("requests.get")
+    data = json.dumps({"Type": "SubscriptionConfirmation", "SubscribeURL": "https://foo", "Message": "foo"})
+    response = client.post(
+        path='/notifications/email/ses',
+        data=data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'SubscriptionConfirmation')]
+    )
+
+    requests_mock.assert_called_once_with("https://foo")
+    assert response.status_code == 200
+
+
+def test_notifications_ses_200_call_process_task(client, mocker):
+    process_mock = mocker.patch("app.notifications.notifications_ses_callback.process_ses_results.apply_async")
+    mocker.patch("app.notifications.sns_handlers.validate_sns_cert", return_value=True)
+    data = {"Type": "Notification", "foo": "bar", "Message": {"mail": "baz"} }
+    mocker.patch("app.notifications.sns_handlers.sns_notification_handler", return_value=data)
+    json_data = json.dumps(data)
+    response = client.post(
+        path='/notifications/email/ses',
+        data=json_data,
+        headers=[('Content-Type', 'application/json'), ('x-amz-sns-message-type', 'Notification')]
+    )
+
+    process_mock.assert_called_once_with([{'Message': {"mail": "baz"}}], queue='notify-internal-tasks')
+    assert response.status_code == 200
+
+
 def test_process_ses_results(sample_email_template):
     create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
 
@@ -31,8 +108,7 @@ def test_process_ses_results(sample_email_template):
 
 def test_process_ses_results_retry_called(sample_email_template, mocker):
     create_notification(sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending')
-
-    mocker.patch("app.dao.notifications_dao.dao_update_notifications_by_reference", side_effect=Exception("EXPECTED"))
+    mocker.patch("app.dao.notifications_dao._update_notification_status", side_effect=Exception("EXPECTED"))
     mocked = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     process_ses_results(response=ses_notification_callback(reference='ref1'))
     assert mocked.call_count != 0
@@ -62,6 +138,7 @@ def test_remove_email_from_bounce():
 
 def test_ses_callback_should_update_notification_status(
         client,
+        _notify_db,
         notify_db_session,
         sample_email_template,
         mocker):
@@ -69,140 +146,159 @@ def test_ses_callback_should_update_notification_status(
         mocker.patch('app.statsd_client.incr')
         mocker.patch('app.statsd_client.timing_with_dates')
         send_mock = mocker.patch(
-            'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+            'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
         )
-        notification = create_notification(
+        notification = create_sample_notification(
+            _notify_db,
+            notify_db_session,
             template=sample_email_template,
-            status='sending',
             reference='ref',
+            status='sending',
+            sent_at=datetime.utcnow()
         )
+        callback_api = create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
         assert get_notification_by_id(notification.id).status == 'sending'
-
         assert process_ses_results(ses_notification_callback(reference='ref'))
         assert get_notification_by_id(notification.id).status == 'delivered'
         statsd_client.timing_with_dates.assert_any_call(
-            "callback.ses.delivered.elapsed-time", datetime.utcnow(), notification.sent_at
+            "callback.ses.elapsed-time", datetime.utcnow(), notification.sent_at
         )
         statsd_client.incr.assert_any_call("callback.ses.delivered")
         updated_notification = Notification.query.get(notification.id)
-        send_mock.assert_called_once_with(updated_notification)
+        encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api)
+        send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks")
 
 
 def test_ses_callback_should_not_update_notification_status_if_already_delivered(sample_email_template, mocker):
     mock_dup = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._duplicate_update_warning')
-    mock_upd = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.notifications_dao.dao_update_notifications_by_reference'
-    )
+    mock_upd = mocker.patch('app.celery.process_ses_receipts_tasks.notifications_dao._update_notification_status')
     notification = create_notification(template=sample_email_template, reference='ref', status='delivered')
-
     assert process_ses_results(ses_notification_callback(reference='ref')) is None
     assert get_notification_by_id(notification.id).status == 'delivered'
-
-    mock_dup.assert_called_once_with(notification=notification, status='delivered')
+    mock_dup.assert_called_once_with(notification, 'delivered')
     assert mock_upd.call_count == 0
 
 
-def test_ses_callback_should_retry_if_notification_is_new(client, notify_db_session, mocker):
+def test_ses_callback_should_retry_if_notification_is_new(mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
-
     with freeze_time('2017-11-17T12:14:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_logger.call_count == 0
         assert mock_retry.call_count == 1
-
-
-def test_ses_callback_should_log_if_notification_is_missing(client, notify_db_session, mocker):
+def test_ses_callback_should_log_if_notification_is_missing(client, _notify_db, mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.warning')
-
     with freeze_time('2017-11-17T12:34:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_retry.call_count == 0
-        mock_logger.assert_called_once_with('notification not found for reference: ref (update to delivered)')
-
-
-def test_ses_callback_should_not_retry_if_notification_is_old(client, notify_db_session, mocker):
+        mock_logger.assert_called_once_with('notification not found for reference: ref (while attempting update to delivered)')
+def test_ses_callback_should_not_retry_if_notification_is_old(mocker):
     mock_retry = mocker.patch('app.celery.process_ses_receipts_tasks.process_ses_results.retry')
     mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.error')
-
     with freeze_time('2017-11-21T12:14:03.646Z'):
         assert process_ses_results(ses_notification_callback(reference='ref')) is None
         assert mock_logger.call_count == 0
         assert mock_retry.call_count == 0
-
-
-def test_ses_callback_should_update_multiple_notification_status_sent(
+def test_ses_callback_does_not_call_send_delivery_status_if_no_db_entry(
         client,
+        _notify_db,
+        notify_db_session,
+        sample_email_template,
+        mocker):
+    with freeze_time('2001-01-01T12:00:00'):
+        send_mock = mocker.patch(
+            'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
+        )
+        notification = create_sample_notification(
+            _notify_db,
+            notify_db_session,
+            template=sample_email_template,
+            reference='ref',
+            status='sending',
+            sent_at=datetime.utcnow()
+        )
+        assert get_notification_by_id(notification.id).status == 'sending'
+        assert process_ses_results(ses_notification_callback(reference='ref'))
+        assert get_notification_by_id(notification.id).status == 'delivered'
+        send_mock.assert_not_called()
+def test_ses_callback_should_update_multiple_notification_status_sent(
+        client,
+        _notify_db,
         notify_db_session,
         sample_email_template,
         mocker):
-
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    create_notification(
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref1',
-    )
-    create_notification(
+        sent_at=datetime.utcnow(),
+        status='sending')
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref2',
-    )
-    create_notification(
+        sent_at=datetime.utcnow(),
+        status='sending')
+    create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref3',
-    )
+        sent_at=datetime.utcnow(),
+        status='sending')
+    create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
     assert process_ses_results(ses_notification_callback(reference='ref1'))
     assert process_ses_results(ses_notification_callback(reference='ref2'))
     assert process_ses_results(ses_notification_callback(reference='ref3'))
     assert send_mock.called
-
-
 def test_ses_callback_should_set_status_to_temporary_failure(client,
+                                                             _notify_db,
                                                              notify_db_session,
                                                              sample_email_template,
                                                              mocker):
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info')
-    notification = create_notification(
+    notification = create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref',
+        status='sending',
+        sent_at=datetime.utcnow()
     )
+    create_service_callback_api(service=notification.service, url="https://original_url.com")
     assert get_notification_by_id(notification.id).status == 'sending'
     assert process_ses_results(ses_soft_bounce_callback(reference='ref'))
     assert get_notification_by_id(notification.id).status == 'temporary-failure'
     assert send_mock.called
-    assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0]
-
-
 def test_ses_callback_should_set_status_to_permanent_failure(client,
+                                                             _notify_db,
                                                              notify_db_session,
                                                              sample_email_template,
                                                              mocker):
     send_mock = mocker.patch(
-        'app.celery.process_ses_receipts_tasks.check_and_queue_callback_task'
+        'app.celery.service_callback_tasks.send_delivery_status_to_service.apply_async'
     )
-    mock_logger = mocker.patch('app.celery.process_ses_receipts_tasks.current_app.logger.info')
-    notification = create_notification(
+    notification = create_sample_notification(
+        _notify_db,
+        notify_db_session,
         template=sample_email_template,
-        status='sending',
         reference='ref',
+        status='sending',
+        sent_at=datetime.utcnow()
     )
-
+    create_service_callback_api(service=sample_email_template.service, url="https://original_url.com")
     assert get_notification_by_id(notification.id).status == 'sending'
     assert process_ses_results(ses_hard_bounce_callback(reference='ref'))
     assert get_notification_by_id(notification.id).status == 'permanent-failure'
     assert send_mock.called
-    assert f'SES bounce for notification ID {notification.id}: ' in mock_logger.call_args[0][0]
-
-
 def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker):
     send_mock = mocker.patch(
         'app.celery.service_callback_tasks.send_complaint_to_service.apply_async'
@@ -210,13 +306,11 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
     create_service_callback_api(
         service=sample_email_template.service, url="https://original_url.com", callback_type="complaint"
     )
-
     notification = create_notification(
         template=sample_email_template, reference='ref1', sent_at=datetime.utcnow(), status='sending'
     )
     response = ses_complaint_callback()
     assert process_ses_results(response)
-
     assert send_mock.call_count == 1
     assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
         'complaint_date': '2018-06-05T13:59:58.000000Z',
@@ -227,3 +321,4 @@ def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email
         'service_callback_api_url': 'https://original_url.com',
         'to': 'recipient1@example.com'
     }
+    

tests/app/conftest.py

@@ -24,6 +24,7 @@ from app.models import (
     KEY_TYPE_TEAM,
     KEY_TYPE_TEST,
     LETTER_TYPE,
+    NOTIFICATION_STATUS_TYPES_COMPLETED,
     SERVICE_PERMISSION_TYPES,
     SMS_TYPE,
     ApiKey,
@@ -62,6 +63,86 @@ def rmock():
         yield rmock
 
 
+def create_sample_notification(
+    notify_db,
+    notify_db_session,
+    service=None,
+    template=None,
+    job=None,
+    job_row_number=None,
+    to_field=None,
+    status="created",
+    provider_response=None,
+    reference=None,
+    created_at=None,
+    sent_at=None,
+    billable_units=1,
+    personalisation=None,
+    api_key=None,
+    key_type=KEY_TYPE_NORMAL,
+    sent_by=None,
+    international=False,
+    client_reference=None,
+    rate_multiplier=1.0,
+    scheduled_for=None,
+    normalised_to=None,
+    postage=None,
+):
+    if created_at is None:
+        created_at = datetime.utcnow()
+    if service is None:
+        service = create_service(check_if_service_exists=True)
+    if template is None:
+        template = create_template(service=service)
+
+    if job is None and api_key is None:
+        # we didn't specify in test - lets create it
+        api_key = ApiKey.query.filter(ApiKey.service == template.service, ApiKey.key_type == key_type).first()
+        if not api_key:
+            api_key = create_api_key(template.service, key_type=key_type)
+
+    notification_id = uuid.uuid4()
+
+    if to_field:
+        to = to_field
+    else:
+        to = "+16502532222"
+
+    data = {
+        "id": notification_id,
+        "to": to,
+        "job_id": job.id if job else None,
+        "job": job,
+        "service_id": service.id,
+        "service": service,
+        "template_id": template.id,
+        "template_version": template.version,
+        "status": status,
+        "provider_response": provider_response,
+        "reference": reference,
+        "created_at": created_at,
+        "sent_at": sent_at,
+        "billable_units": billable_units,
+        "personalisation": personalisation,
+        "notification_type": template.template_type,
+        "api_key": api_key,
+        "api_key_id": api_key and api_key.id,
+        "key_type": api_key.key_type if api_key else key_type,
+        "sent_by": sent_by,
+        "updated_at": created_at if status in NOTIFICATION_STATUS_TYPES_COMPLETED else None,
+        "client_reference": client_reference,
+        "rate_multiplier": rate_multiplier,
+        "normalised_to": normalised_to,
+        "postage": postage,
+    }
+    if job_row_number is not None:
+        data["job_row_number"] = job_row_number
+    notification = Notification(**data)
+    dao_create_notification(notification)
+
+    return notification
+
+
 @pytest.fixture(scope='function')
 def service_factory(sample_user):
     class ServiceFactory(object):

tests/app/dao/notification_dao/test_notification_dao.py

@@ -15,7 +15,7 @@ from app.dao.notifications_dao import (
     dao_get_letters_to_be_printed,
     dao_get_notification_by_reference,
     dao_get_notification_count_for_job_id,
-    dao_get_notification_or_history_by_reference,
+    dao_get_notification_history_by_reference,
     dao_get_notifications_by_recipient_or_reference,
     dao_timeout_notifications,
     dao_update_notification,
@@ -1607,28 +1607,28 @@ def test_dao_get_notification_by_reference_with_no_matches_raises_error(notify_d
         dao_get_notification_by_reference('REF1')
 
 
-def test_dao_get_notification_or_history_by_reference_with_one_match_returns_notification(
+def test_dao_get_notification_history_by_reference_with_one_match_returns_notification(
         sample_letter_template
 ):
     create_notification(template=sample_letter_template, reference='REF1')
-    notification = dao_get_notification_or_history_by_reference('REF1')
+    notification = dao_get_notification_history_by_reference('REF1')
 
     assert notification.reference == 'REF1'
 
 
-def test_dao_get_notification_or_history_by_reference_with_multiple_matches_raises_error(
+def test_dao_get_notification_history_by_reference_with_multiple_matches_raises_error(
         sample_letter_template
 ):
     create_notification(template=sample_letter_template, reference='REF1')
     create_notification(template=sample_letter_template, reference='REF1')
 
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1')
+        dao_get_notification_history_by_reference('REF1')
 
 
-def test_dao_get_notification_or_history_by_reference_with_no_matches_raises_error(notify_db_session):
+def test_dao_get_notification_history_by_reference_with_no_matches_raises_error(notify_db_session):
     with pytest.raises(SQLAlchemyError):
-        dao_get_notification_or_history_by_reference('REF1')
+        dao_get_notification_history_by_reference('REF1')
 
 
 @pytest.mark.parametrize("notification_type",

tests/app/dao/test_annual_billing_dao.py

@@ -47,31 +47,17 @@ def test_dao_update_annual_billing_for_future_years(notify_db_session, sample_se
 
 
 @pytest.mark.parametrize('org_type, year, expected_default',
-                         [('central', 2021, 150000),
-                          ('local', 2021, 25000),
-                          ('nhs_central', 2021, 150000),
-                          ('nhs_local', 2021, 25000),
-                          ('nhs_gp', 2021, 10000),
-                          ('emergency_service', 2021, 25000),
-                          ('school_or_college', 2021, 10000),
-                          ('other', 2021, 10000),
-                          (None, 2021, 10000),
-                          ('central', 2020, 250000),
-                          ('local', 2020, 25000),
-                          ('nhs_central', 2020, 250000),
-                          ('nhs_local', 2020, 25000),
-                          ('nhs_gp', 2020, 25000),
-                          ('emergency_service', 2020, 25000),
-                          ('school_or_college', 2020, 25000),
-                          ('other', 2020, 25000),
-                          (None, 2020, 25000),
-                          ('central', 2019, 250000),
-                          ('school_or_college', 2022, 10000),
-                          ('central', 2022, 40000),
-                          ('local', 2022, 20000),
-                          ('nhs_local', 2022, 20000),
-                          ('emergency_service', 2022, 20000),
-                          ('central', 2023, 40000),
+                         [('federal', 2021, 150000),
+                          ('state', 2021, 150000),
+                          (None, 2021, 150000),
+                          ('federal', 2020, 250000),
+                          ('state', 2020, 250000),
+                          ('other', 2020, 250000),
+                          (None, 2020, 250000),
+                          ('federal', 2019, 250000),
+                          ('federal', 2022, 40000),
+                          ('state', 2022, 40000),
+                          ('federal', 2023, 40000),
                           ])
 def test_set_default_free_allowance_for_service(notify_db_session, org_type, year, expected_default):
 
@@ -93,7 +79,7 @@ def test_set_default_free_allowance_for_service_using_correct_year(sample_servic
 
     mock_dao.assert_called_once_with(
         sample_service.id,
-        25000,
+        250000,
         2020
     )
 
@@ -105,9 +91,9 @@ def test_set_default_free_allowance_for_service_updates_existing_year(sample_ser
     assert not sample_service.organisation_type
     assert len(annual_billing) == 1
     assert annual_billing[0].service_id == sample_service.id
-    assert annual_billing[0].free_sms_fragment_limit == 10000
+    assert annual_billing[0].free_sms_fragment_limit == 150000
 
-    sample_service.organisation_type = 'central'
+    sample_service.organisation_type = 'federal'
 
     set_default_free_allowance_for_service(service=sample_service, year_start=None)
     annual_billing = AnnualBilling.query.all()

tests/app/dao/test_organisation_dao.py

@@ -65,7 +65,7 @@ def test_update_organisation(notify_db_session):
     data = {
         'name': 'new name',
         "crown": True,
-        "organisation_type": 'local',
+        "organisation_type": 'state',
         "agreement_signed": True,
         "agreement_signed_at": datetime.datetime.utcnow(),
         "agreement_signed_by_id": user.id,
@@ -124,8 +124,8 @@ def test_update_organisation_does_not_update_the_service_if_certain_attributes_n
     email_branding = create_email_branding()
     letter_branding = create_letter_branding()
 
-    sample_service.organisation_type = 'local'
-    sample_organisation.organisation_type = 'central'
+    sample_service.organisation_type = 'state'
+    sample_organisation.organisation_type = 'federal'
     sample_organisation.email_branding = email_branding
     sample_organisation.letter_branding = letter_branding
 
@@ -138,8 +138,8 @@ def test_update_organisation_does_not_update_the_service_if_certain_attributes_n
 
     assert sample_organisation.name == 'updated org name'
 
-    assert sample_organisation.organisation_type == 'central'
-    assert sample_service.organisation_type == 'local'
+    assert sample_organisation.organisation_type == 'federal'
+    assert sample_service.organisation_type == 'state'
 
     assert sample_organisation.email_branding == email_branding
     assert sample_service.email_branding is None
@@ -152,20 +152,20 @@ def test_update_organisation_updates_the_service_org_type_if_org_type_is_provide
     sample_service,
     sample_organisation,
 ):
-    sample_service.organisation_type = 'local'
-    sample_organisation.organisation_type = 'local'
+    sample_service.organisation_type = 'state'
+    sample_organisation.organisation_type = 'state'
 
     sample_organisation.services.append(sample_service)
     db.session.commit()
 
-    dao_update_organisation(sample_organisation.id, organisation_type='central')
+    dao_update_organisation(sample_organisation.id, organisation_type='federal')
 
-    assert sample_organisation.organisation_type == 'central'
-    assert sample_service.organisation_type == 'central'
+    assert sample_organisation.organisation_type == 'federal'
+    assert sample_service.organisation_type == 'federal'
     assert Service.get_history_model().query.filter_by(
         id=sample_service.id,
         version=2
-    ).one().organisation_type == 'central'
+    ).one().organisation_type == 'federal'
 
 
 def test_update_organisation_updates_the_service_branding_if_branding_is_provided(
@@ -228,8 +228,8 @@ def test_update_organisation_updates_services_with_new_crown_type(
 def test_add_service_to_organisation(sample_service, sample_organisation):
     assert sample_organisation.services == []
 
-    sample_service.organisation_type = "central"
-    sample_organisation.organisation_type = "local"
+    sample_service.organisation_type = "federal"
+    sample_organisation.organisation_type = "state"
     sample_organisation.crown = False
 
     dao_add_service_to_organisation(sample_service, sample_organisation.id)

tests/app/dao/test_services_dao.py

@@ -98,7 +98,7 @@ def test_create_service(notify_db_session):
                       email_from="email_from",
                       message_limit=1000,
                       restricted=False,
-                      organisation_type='central',
+                      organisation_type='federal',
                       created_by=user)
     dao_create_service(service, user)
     assert Service.query.count() == 1
@@ -110,7 +110,7 @@ def test_create_service(notify_db_session):
     assert service_db.prefix_sms is True
     assert service.active is True
     assert user in service_db.users
-    assert service_db.organisation_type == 'central'
+    assert service_db.organisation_type == 'federal'
     assert service_db.crown is None
     assert not service.letter_branding
     assert not service.organisation_id
@@ -119,13 +119,13 @@ def test_create_service(notify_db_session):
 def test_create_service_with_organisation(notify_db_session):
     user = create_user(email='local.authority@local-authority.gov.uk')
     organisation = create_organisation(
-        name='Some local authority', organisation_type='local', domains=['local-authority.gov.uk'])
+        name='Some local authority', organisation_type='state', domains=['local-authority.gov.uk'])
     assert Service.query.count() == 0
     service = Service(name="service_name",
                       email_from="email_from",
                       message_limit=1000,
                       restricted=False,
-                      organisation_type='central',
+                      organisation_type='federal',
                       created_by=user)
     dao_create_service(service, user)
     assert Service.query.count() == 1
@@ -138,7 +138,7 @@ def test_create_service_with_organisation(notify_db_session):
     assert service_db.prefix_sms is True
     assert service.active is True
     assert user in service_db.users
-    assert service_db.organisation_type == 'local'
+    assert service_db.organisation_type == 'state'
     assert service_db.crown is None
     assert not service.letter_branding
     assert service.organisation_id == organisation.id
@@ -162,6 +162,7 @@ def test_create_service_with_organisation(notify_db_session):
     # the NHS branding set up
     ('SHN', False),
 ))
+@pytest.mark.skip(reason='Update for TTS')
 def test_create_nhs_service_get_default_branding_based_on_email_address(
     notify_db_session,
     branding_name_to_create,
@@ -446,7 +447,7 @@ def test_get_all_user_services_should_return_empty_list_if_no_services_for_user(
 
 @freeze_time('2019-04-23T10:00:00')
 def test_dao_fetch_live_services_data(sample_user):
-    org = create_organisation(organisation_type='nhs_central')
+    org = create_organisation(organisation_type='federal')
     service = create_service(go_live_user=sample_user, go_live_at='2014-04-20T10:00:00')
     sms_template = create_template(service=service)
     service_2 = create_service(service_name='second', go_live_at='2017-04-20T10:00:00', go_live_user=sample_user)
@@ -484,7 +485,7 @@ def test_dao_fetch_live_services_data(sample_user):
     # checks the results and that they are ordered by date:
     assert results == [
         {'service_id': mock.ANY, 'service_name': 'Sample service', 'organisation_name': 'test_org_1',
-            'organisation_type': 'nhs_central', 'consent_to_research': None, 'contact_name': 'Test User',
+            'organisation_type': 'federal', 'consent_to_research': None, 'contact_name': 'Test User',
             'contact_email': 'notify@digital.cabinet-office.gov.uk', 'contact_mobile': '+447700900986',
             'live_date': datetime(2014, 4, 20, 10, 0), 'sms_volume_intent': None, 'email_volume_intent': None,
             'letter_volume_intent': None, 'sms_totals': 2, 'email_totals': 1, 'letter_totals': 1,

tests/app/db.py

@@ -116,7 +116,7 @@ def create_service(
         email_from=None,
         prefix_sms=True,
         message_limit=1000,
-        organisation_type='central',
+        organisation_type='federal',
         check_if_service_exists=False,
         go_live_user=None,
         go_live_at=None,
@@ -437,17 +437,17 @@ def create_service_permission(service_id, permission=EMAIL_TYPE):
 def create_inbound_sms(
         service,
         notify_number=None,
-        user_number='447700900111',
+        user_number='12025550104',
         provider_date=None,
         provider_reference=None,
         content='Hello',
-        provider="mmg",
+        provider="sns",
         created_at=None
 ):
     if not service.inbound_number:
         create_inbound_number(
             # create random inbound number
-            notify_number or '07{:09}'.format(random.randint(0, 1e9 - 1)),
+            notify_number or '1'+str(random.randint(1001001000, 9999999999)),
             provider=provider,
             service_id=service.id
         )
@@ -560,7 +560,7 @@ def create_api_key(service, key_type=KEY_TYPE_NORMAL, key_name=None):
     return api_key
 
 
-def create_inbound_number(number, provider='mmg', active=True, service_id=None):
+def create_inbound_number(number, provider='sns', active=True, service_id=None):
     inbound_number = InboundNumber(
         id=uuid.uuid4(),
         number=number,

tests/app/inbound_sms/test_rest.py

@@ -39,6 +39,7 @@ def test_post_to_get_inbound_sms_with_no_params(admin_request, sample_service):
     '+4407700900001',
     '447700900001',
 ])
+@pytest.mark.skip(reason="Needs updating for TTS. Don't need to test UK numbers right now")
 def test_post_to_get_inbound_sms_filters_user_number(admin_request, sample_service, user_number):
     # user_number in the db is international and normalised
     one = create_inbound_sms(sample_service, user_number='447700900001')
@@ -65,7 +66,7 @@ def test_post_to_get_inbound_sms_filters_international_user_number(admin_request
     create_inbound_sms(sample_service)
 
     data = {
-        'phone_number': '+1 (202) 555-0104'
+        'phone_number': '12025550104'
     }
 
     sms = admin_request.post(
@@ -74,9 +75,10 @@ def test_post_to_get_inbound_sms_filters_international_user_number(admin_request
         _data=data
     )['data']
 
-    assert len(sms) == 1
-    assert sms[0]['id'] == str(one.id)
-    assert sms[0]['user_number'] == str(one.user_number)
+    assert len(sms) == 2
+    print(f'sms is: {sms}')
+    assert sms[1]['id'] == str(one.id)
+    assert sms[1]['user_number'] == str(one.user_number)
 
 
 def test_post_to_get_inbound_sms_allows_badly_formatted_number(admin_request, sample_service):

tests/app/notifications/test_notifications_ses_callback.py

@@ -2,12 +2,12 @@ import pytest
 from flask import json
 from sqlalchemy.exc import SQLAlchemyError
 
-from app.dao.notifications_dao import get_notification_by_id
-from app.models import Complaint
-from app.notifications.notifications_ses_callback import (
+from app.celery.process_ses_receipts_tasks import (
     check_and_queue_callback_task,
     handle_complaint,
 )
+from app.dao.notifications_dao import get_notification_by_id
+from app.models import Complaint
 from tests.app.db import (
     create_notification,
     create_notification_history,
@@ -72,7 +72,7 @@ def test_process_ses_results_in_complaint_save_complaint_with_null_complaint_typ
 
 def test_check_and_queue_callback_task(mocker, sample_notification):
     mock_create = mocker.patch(
-        'app.notifications.notifications_ses_callback.create_delivery_status_callback_data'
+        'app.celery.process_ses_receipts_tasks.create_delivery_status_callback_data'
     )
 
     mock_send = mocker.patch(
@@ -86,6 +86,7 @@ def test_check_and_queue_callback_task(mocker, sample_notification):
 
     # callback_api doesn't match by equality for some
     # reason, so we need to take this approach instead
+    print(f'mock_create.mock_calls is: {mock_create.mock_calls}')
     mock_create_args = mock_create.mock_calls[0][1]
     assert mock_create_args[0] == sample_notification
     assert mock_create_args[1].id == callback_api.id

tests/app/organisation/test_rest.py

@@ -26,7 +26,7 @@ from tests.app.db import (
 
 
 def test_get_all_organisations(admin_request, notify_db_session):
-    create_organisation(name='inactive org', active=False, organisation_type='nhs_central')
+    create_organisation(name='inactive org', active=False, organisation_type='federal')
     create_organisation(name='active org', domains=['example.com'])
 
     response = admin_request.get(
@@ -52,7 +52,7 @@ def test_get_all_organisations(admin_request, notify_db_session):
     assert response[1]['active'] is False
     assert response[1]['count_of_live_services'] == 0
     assert response[1]['domains'] == []
-    assert response[1]['organisation_type'] == 'nhs_central'
+    assert response[1]['organisation_type'] == 'federal'
 
 
 def test_get_organisation_by_id(admin_request, notify_db_session):
@@ -169,7 +169,7 @@ def test_post_create_organisation(admin_request, notify_db_session, crown):
         'name': 'test organisation',
         'active': True,
         'crown': crown,
-        'organisation_type': 'local',
+        'organisation_type': 'state',
     }
 
     response = admin_request.post(
@@ -191,6 +191,7 @@ def test_post_create_organisation(admin_request, notify_db_session, crown):
 
 
 @pytest.mark.parametrize('org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+@pytest.mark.skip(reason='Update for TTS')
 def test_post_create_organisation_sets_default_nhs_branding_for_nhs_orgs(
     admin_request, notify_db_session, nhs_email_branding, org_type
 ):
@@ -218,7 +219,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample
         'name': sample_organisation.name,
         'active': True,
         'crown': True,
-        'organisation_type': 'central',
+        'organisation_type': 'federal',
     }
 
     response = admin_request.post(
@@ -237,12 +238,12 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample
     ({
         'active': False,
         'crown': True,
-        'organisation_type': 'central',
+        'organisation_type': 'federal',
     }, 'name is a required property'),
     ({
         'active': False,
         'name': 'Service name',
-        'organisation_type': 'central',
+        'organisation_type': 'federal',
     }, 'crown is a required property'),
     ({
         'active': False,
@@ -253,7 +254,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample
         'active': False,
         'name': 'Service name',
         'crown': None,
-        'organisation_type': 'central',
+        'organisation_type': 'federal',
     }, 'crown None is not of type boolean'),
     ({
         'active': False,
@@ -262,7 +263,7 @@ def test_post_create_organisation_existing_name_raises_400(admin_request, sample
         'organisation_type': 'foo',
     }, (
         'organisation_type foo is not one of '
-        '[central, local, nhs_central, nhs_local, nhs_gp, emergency_service, school_or_college, other]'
+        '[federal, state, other]'
     )),
 ))
 def test_post_create_organisation_with_missing_data_gives_validation_error(
@@ -295,7 +296,7 @@ def test_post_update_organisation_updates_fields(
         'name': 'new organisation name',
         'active': False,
         'crown': crown,
-        'organisation_type': 'central',
+        'organisation_type': 'federal',
     }
     assert org.crown is None
 
@@ -314,7 +315,7 @@ def test_post_update_organisation_updates_fields(
     assert organisation[0].active == data['active']
     assert organisation[0].crown == crown
     assert organisation[0].domains == []
-    assert organisation[0].organisation_type == 'central'
+    assert organisation[0].organisation_type == 'federal'
 
 
 @pytest.mark.parametrize('domain_list', (
@@ -371,6 +372,7 @@ def test_update_other_organisation_attributes_doesnt_clear_domains(
 
 
 @pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+@pytest.mark.skip(reason='Update for TTS')
 def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present(
     admin_request,
     nhs_email_branding,
@@ -398,6 +400,7 @@ def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present(
 
 
 @pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+@pytest.mark.skip(reason='Update for TTS')
 def test_post_update_organisation_to_nhs_type_does_not_update_branding_if_default_branding_set(
     admin_request,
     nhs_email_branding,
@@ -581,7 +584,7 @@ def test_post_link_service_to_organisation(admin_request, sample_service):
     data = {
         'service_id': str(sample_service.id)
     }
-    organisation = create_organisation(organisation_type='central')
+    organisation = create_organisation(organisation_type='federal')
 
     admin_request.post(
         'organisation.link_service_to_organisation',
@@ -590,7 +593,7 @@ def test_post_link_service_to_organisation(admin_request, sample_service):
         _expected_status=204
     )
     assert len(organisation.services) == 1
-    assert sample_service.organisation_type == 'central'
+    assert sample_service.organisation_type == 'federal'
 
 
 @freeze_time('2021-09-24 13:30')
@@ -598,7 +601,7 @@ def test_post_link_service_to_organisation_inserts_annual_billing(admin_request,
     data = {
         'service_id': str(sample_service.id)
     }
-    organisation = create_organisation(organisation_type='central')
+    organisation = create_organisation(organisation_type='federal')
     assert len(organisation.services) == 0
     assert len(AnnualBilling.query.all()) == 0
     admin_request.post(
@@ -623,7 +626,7 @@ def test_post_link_service_to_organisation_rollback_service_if_annual_billing_up
     }
     assert not sample_service.organisation_type
 
-    organisation = create_organisation(organisation_type='central')
+    organisation = create_organisation(organisation_type='federal')
     assert len(organisation.services) == 0
     assert len(AnnualBilling.query.all()) == 0
     with pytest.raises(expected_exception=SQLAlchemyError):
@@ -655,7 +658,7 @@ def test_post_link_service_to_another_org(
     assert len(sample_organisation.services) == 1
     assert not sample_service.organisation_type
 
-    new_org = create_organisation(organisation_type='central')
+    new_org = create_organisation(organisation_type='federal')
     admin_request.post(
         'organisation.link_service_to_organisation',
         _data=data,
@@ -664,7 +667,7 @@ def test_post_link_service_to_another_org(
     )
     assert not sample_organisation.services
     assert len(new_org.services) == 1
-    assert sample_service.organisation_type == 'central'
+    assert sample_service.organisation_type == 'federal'
     annual_billing = AnnualBilling.query.all()
     assert len(annual_billing) == 1
     assert annual_billing[0].free_sms_fragment_limit == 150000

tests/app/service/test_rest.py

@@ -687,7 +687,7 @@ def test_update_service(client, notify_db_session, sample_service):
         'email_from': 'updated.service.name',
         'created_by': str(sample_service.created_by.id),
         'email_branding': str(brand.id),
-        'organisation_type': 'school_or_college',
+        'organisation_type': 'federal',
     }
 
     auth_header = create_admin_authorization_header()
@@ -702,7 +702,7 @@ def test_update_service(client, notify_db_session, sample_service):
     assert result['data']['name'] == 'updated service name'
     assert result['data']['email_from'] == 'updated.service.name'
     assert result['data']['email_branding'] == str(brand.id)
-    assert result['data']['organisation_type'] == 'school_or_college'
+    assert result['data']['organisation_type'] == 'federal'
 
 
 def test_cant_update_service_org_type_to_random_value(client, sample_service):

tests/app/test_commands.py

@@ -21,9 +21,8 @@ def test_insert_inbound_numbers_from_file(notify_db_session, notify_api, tmpdir)
 
 
 @pytest.mark.parametrize("organisation_type, expected_allowance",
-                         [('central', 40000),
-                          ('local', 20000),
-                          ('nhs_gp', 10000)])
+                         [('federal', 40000),
+                          ('state', 40000)])
 def test_populate_annual_billing_with_defaults(
         notify_db_session, notify_api, organisation_type, expected_allowance
 ):
@@ -45,7 +44,7 @@ def test_populate_annual_billing_with_defaults(
 def test_populate_annual_billing_with_defaults_sets_free_allowance_to_zero_if_previous_year_is_zero(
         notify_db_session, notify_api
 ):
-    service = create_service(organisation_type='central')
+    service = create_service(organisation_type='federal')
     create_annual_billing(service_id=service.id, free_sms_fragment_limit=0, financial_year_start=2021)
     notify_api.test_cli_runner().invoke(
         populate_annual_billing_with_defaults, ['-y', 2022]

tests/app/test_schemas.py

@@ -106,7 +106,7 @@ def test_provider_details_schema_returns_user_details(
     restore_provider_details
 ):
     from app.schemas import provider_details_schema
-    current_sms_provider = get_provider_details_by_identifier('mmg')
+    current_sms_provider = get_provider_details_by_identifier('sns')
     current_sms_provider.created_by = sample_user
     data = provider_details_schema.dump(current_sms_provider)
 
@@ -119,7 +119,7 @@ def test_provider_details_history_schema_returns_user_details(
     restore_provider_details,
 ):
     from app.schemas import provider_details_schema
-    current_sms_provider = get_provider_details_by_identifier('mmg')
+    current_sms_provider = get_provider_details_by_identifier('sns')
     current_sms_provider.created_by_id = sample_user.id
     data = provider_details_schema.dump(current_sms_provider)
 

tests/app/v2/notifications/test_get_notifications.py

@@ -68,6 +68,7 @@ def test_get_notification_by_id_returns_200(
         'completed_at': sample_notification.completed_at(),
         'scheduled_for': None,
         'postage': None,
+        'provider_response': None
     }
 
     assert json_response == expected_response
@@ -120,6 +121,7 @@ def test_get_notification_by_id_with_placeholders_returns_200(
         'completed_at': sample_notification.completed_at(),
         'scheduled_for': None,
         'postage': None,
+        'provider_response': None
     }
 
     assert json_response == expected_response

tests/test_all_queues_used.py

@@ -1,18 +0,0 @@
-from app.config import QueueNames
-
-
-def test_queue_names_set_in_paas_app_wrapper():
-    with open("scripts/paas_app_wrapper.sh", 'r') as stream:
-        search = ' -Q '
-
-        watched_queues = set()
-        for line in stream.readlines():
-            start_of_queue_arg = line.find(search)
-            if start_of_queue_arg > 0:
-                start_of_queue_names = start_of_queue_arg + len(search)
-                end_of_queue_names = line.find('2>') if '2>' in line else len(line)
-                watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')})
-
-        # ses-callbacks isn't used in api (only used in SNS lambda)
-        ignored_queues = {'ses-callbacks'}
-        assert watched_queues == set(QueueNames.all_queues()) | ignored_queues