WIP

bind to notify-redis automatically

app/billing/billing_schemas.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST annual billing schema",
     "type": "object",
     "title": "Create",

app/broadcast_message/broadcast_message_schema.py

@@ -2,7 +2,7 @@ from app.models import BroadcastStatusType
 from app.schema_validation.definitions import uuid
 
 create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST create broadcast_message schema',
     'type': 'object',
     'title': 'Create broadcast_message',
@@ -32,7 +32,7 @@ create_broadcast_message_schema = {
 }
 
 update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST update broadcast_message schema',
     'type': 'object',
     'title': 'Update broadcast_message',
@@ -47,7 +47,7 @@ update_broadcast_message_schema = {
 }
 
 update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST update broadcast_message status schema',
     'type': 'object',
     'title': 'Update broadcast_message',

app/clients/sms/__init__.py

@@ -23,7 +23,6 @@ class SmsClient(Client):
     def init_app(self, current_app, statsd_client):
         self.current_app = current_app
         self.statsd_client = statsd_client
-        self.from_number = self.current_app.config.get('FROM_NUMBER')
 
     def record_outcome(self, success):
         log_message = "Provider request for {} {}".format(
@@ -41,15 +40,6 @@ class SmsClient(Client):
     def send_sms(self, to, content, reference, international, sender):
         start_time = monotonic()
 
-        if sender is None:
-            # temporary log to see if the following ternary is necessary
-            # or if it's safe to remove it - keep for 1-2 weeks
-            self.current_app.logger.warning(
-                f"send_sms called with 'sender' of 'None' for {reference}"
-            )
-
-        sender = self.from_number if sender is None else sender
-
         try:
             response = self.try_send_sms(to, content, reference, international, sender)
             self.record_outcome(True)

app/cloudfoundry_config.py

@@ -1,22 +1,12 @@
-"""
-Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
-on local/aws boxes
-"""
-
 import json
 import os
 
 
 def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
-    set_config_env_vars(vcap_services)
 
-
-def set_config_env_vars(vcap_services):
     # Postgres config
     os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'].replace('postgres',
                                                                                                        'postgresql')
-
-    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
-    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
-    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
+    # Redis config
+    os.environ['REDIS_URL'] = vcap_services['redis'][0]['credentials']['uri']

app/complaint/complaint_schema.py

@@ -1,6 +1,6 @@
 
 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "complaint count request schema",
     "type": "object",
     "title": "Complaint count request",

app/config.py

@@ -185,6 +185,7 @@ class Config(object):
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
     NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
     LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
+    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
     # we only need real email in Live environment (production)
     DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
 
@@ -546,6 +547,21 @@ class CloudFoundryConfig(Config):
     pass
 
 
+# CloudFoundry sandbox
+class Sandbox(CloudFoundryConfig):
+    NOTIFY_EMAIL_DOMAIN = 'notify.works'
+    NOTIFY_ENVIRONMENT = 'sandbox'
+    CSV_UPLOAD_BUCKET_NAME = 'cf-sandbox-notifications-csv-upload'
+    CONTACT_LIST_BUCKET_NAME = 'cf-sandbox-contact-list'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    TEST_LETTERS_BUCKET_NAME = 'cf-sandbox-test-letters'
+    DVLA_RESPONSE_BUCKET_NAME = 'notify.works-ftp'
+    LETTERS_PDF_BUCKET_NAME = 'cf-sandbox-letters-pdf'
+    LETTERS_SCAN_BUCKET_NAME = 'cf-sandbox-letters-scan'
+    INVALID_PDF_BUCKET_NAME = 'cf-sandbox-letters-invalid-pdf'
+    FROM_NUMBER = 'sandbox'
+
+
 configs = {
     'development': Development,
     'test': Test,
@@ -553,4 +569,5 @@ configs = {
     'production': Live,
     'staging': Staging,
     'preview': Preview,
+    'sandbox': Sandbox
 }

app/dao/email_branding_dao.py

@@ -11,10 +11,6 @@ def dao_get_email_branding_by_id(email_branding_id):
     return EmailBranding.query.filter_by(id=email_branding_id).one()
 
 
-def dao_get_email_branding_by_name(email_branding_name):
-    return EmailBranding.query.filter_by(name=email_branding_name).first()
-
-
 @autocommit
 def dao_create_email_branding(email_branding):
     db.session.add(email_branding)

app/dao/fact_billing_dao.py

@@ -779,6 +779,31 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
     return aggregated_totals
 
 
+def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
+    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
+
+    daily_volume_stats = db.session.query(
+        FactBilling.bst_date,
+        FactBilling.provider,
+        func.sum(FactBilling.notifications_sent).label('sms_totals'),
+        func.sum(FactBilling.billable_units).label('sms_fragment_totals'),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('sms_chargeable_units'),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier * FactBilling.rate).label('sms_cost'),
+    ).filter(
+        FactBilling.notification_type == SMS_TYPE,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+    ).group_by(
+        FactBilling.bst_date,
+        FactBilling.provider,
+    ).order_by(
+        FactBilling.bst_date,
+        FactBilling.provider,
+    ).all()
+
+    return daily_volume_stats
+
+
 def fetch_volumes_by_service(start_date, end_date):
     # query to return the volume totals by service aggregated for the date range given
     # start and end dates are inclusive.

app/dao/services_dao.py

@@ -9,7 +9,7 @@ from sqlalchemy.sql.expression import and_, asc, case, func
 from app import db
 from app.dao.dao_utils import VersionOptions, autocommit, version_class
 from app.dao.date_util import get_current_financial_year
-from app.dao.email_branding_dao import dao_get_email_branding_by_name
+from app.dao.email_branding_dao import dao_get_email_branding_by_id
 from app.dao.letter_branding_dao import dao_get_letter_branding_by_name
 from app.dao.organisation_dao import dao_get_organisation_by_email_address
 from app.dao.service_sms_sender_dao import insert_service_sms_sender
@@ -326,7 +326,7 @@ def dao_create_service(
             service.letter_branding = organisation.letter_branding
 
     elif service.organisation_type in NHS_ORGANISATION_TYPES or email_address_is_nhs(user.email_address):
-        service.email_branding = dao_get_email_branding_by_name('NHS')
+        service.email_branding = dao_get_email_branding_by_id(current_app.config['NHS_EMAIL_BRANDING_ID'])
         service.letter_branding = dao_get_letter_branding_by_name('NHS')
 
     if organisation:

app/email_branding/email_branding_schema.py

@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES
 
 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }
 
 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {

app/inbound_sms/inbound_sms_schemas.py

@@ -1,5 +1,5 @@
 get_inbound_sms_for_service_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for parameters allowed when searching for to field=",
     "type": "object",
     "properties": {

app/letter_branding/letter_branding_schema.py

@@ -1,10 +1,10 @@
 post_letter_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for creating or updating a letter brand",
     "type": "object",
     "properties": {
         "name": {"type": ["string", "null"]},
         "filename": {"type": ["string", "null"]},
     },
-    "required": ("name", "filename")
+    "required": ["name", "filename"]
 }

app/letters/letter_schemas.py

@@ -1,5 +1,5 @@
 letter_references = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "list of letter notification references",
     "type": "object",
     "title": "references",

app/notifications/notifications_letter_callback.py

@@ -17,7 +17,7 @@ register_errors(letter_callback_blueprint)
 
 
 dvla_sns_callback_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "sns callback received on s3 update",
     "type": "object",
     "title": "dvla internal sns callback",

app/organisation/organisation_schema.py

@@ -2,7 +2,7 @@ from app.models import INVITED_USER_STATUS_TYPES, ORGANISATION_TYPES
 from app.schema_validation.definitions import uuid
 
 post_create_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST organisation schema",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_organisation_schema = {
 }
 
 post_update_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST organisation schema",
     "type": "object",
     "properties": {
@@ -28,7 +28,7 @@ post_update_organisation_schema = {
 }
 
 post_link_service_to_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST link service to organisation schema",
     "type": "object",
     "properties": {
@@ -39,7 +39,7 @@ post_link_service_to_organisation_schema = {
 
 
 post_create_invited_org_user_status_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST create organisation invite schema",
     "type": "object",
     "properties": {
@@ -52,7 +52,7 @@ post_create_invited_org_user_status_schema = {
 
 
 post_update_invited_org_user_status_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST update organisation invite schema",
     "type": "object",
     "properties": {

app/organisation/rest.py

@@ -22,7 +22,7 @@ from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
 from app.errors import InvalidRequest, register_errors
-from app.models import KEY_TYPE_NORMAL, Organisation
+from app.models import KEY_TYPE_NORMAL, NHS_ORGANISATION_TYPES, Organisation
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
@@ -93,6 +93,9 @@ def create_organisation():
 
     validate(data, post_create_organisation_schema)
 
+    if data["organisation_type"] in NHS_ORGANISATION_TYPES:
+        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
+
     organisation = Organisation(**data)
     dao_create_organisation(organisation)
     return jsonify(organisation.serialize()), 201
@@ -102,6 +105,12 @@ def create_organisation():
 def update_organisation(organisation_id):
     data = request.get_json()
     validate(data, post_update_organisation_schema)
+
+    organisation = dao_get_organisation_by_id(organisation_id)
+
+    if data.get('organisation_type') in NHS_ORGANISATION_TYPES and not organisation.email_branding_id:
+        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
+
     result = dao_update_organisation(organisation_id, **data)
 
     if data.get('agreement_signed') is True:

app/performance_dashboard/performance_dashboard_schema.py

@@ -1,5 +1,5 @@
 performance_dashboard_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Performance dashboard request schema",
     "type": "object",
     "title": "Performance dashboard request",

app/platform_stats/platform_stats_schema.py

@@ -1,5 +1,5 @@
 platform_stats_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "platform stats request schema",
     "type": "object",
     "title": "Platform stats request",

app/platform_stats/rest.py

@@ -5,6 +5,7 @@ from flask import Blueprint, jsonify, request
 from app.dao.date_util import get_financial_year_for_datetime
 from app.dao.fact_billing_dao import (
     fetch_billing_details_for_all_services,
+    fetch_daily_sms_provider_volumes_for_platform,
     fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
@@ -161,6 +162,27 @@ def daily_volumes_report():
     return jsonify(report)
 
 
+@platform_stats_blueprint.route('daily-sms-provider-volumes-report')
+def daily_sms_provider_volumes_report():
+    start_date = validate_date_format(request.args.get('start_date'))
+    end_date = validate_date_format(request.args.get('end_date'))
+
+    daily_volumes = fetch_daily_sms_provider_volumes_for_platform(start_date, end_date)
+    report = []
+
+    for row in daily_volumes:
+        report.append({
+            'day': row.bst_date.isoformat(),
+            'provider': row.provider,
+            'sms_totals': int(row.sms_totals),
+            'sms_fragment_totals': int(row.sms_fragment_totals),
+            'sms_chargeable_units': int(row.sms_chargeable_units),
+            # convert from Decimal to float as it's not json serialisable
+            'sms_cost': float(row.sms_cost),
+        })
+    return jsonify(report)
+
+
 @platform_stats_blueprint.route('volumes-by-service')
 def volumes_by_service_report():
     start_date = validate_date_format(request.args.get('start_date'))

app/service/send_notification.py

@@ -7,7 +7,10 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app import create_random_identifier
 from app.config import QueueNames
-from app.dao.notifications_dao import _update_notification_status
+from app.dao.notifications_dao import (
+    _update_notification_status,
+    get_notification_by_id,
+)
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
 from app.dao.services_dao import dao_fetch_service_by_id
@@ -166,15 +169,20 @@ def send_pdf_letter_notification(service_id, post_data):
         allow_guest_list_recipients=False,
     )
 
+    # notification already exists e.g. if the user clicked send in different tabs
+    if get_notification_by_id(post_data['file_id']):
+        return {'id': str(post_data['file_id'])}
+
     template = get_precompiled_letter_template(service.id)
     file_location = 'service-{}/{}.pdf'.format(service.id, post_data['file_id'])
 
     try:
         letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
     except S3ObjectNotFound as e:
-        current_app.logger.exception('Letter {}.pdf not in transient {} bucket'.format(
+        current_app.logger.warning('Letter {}.pdf not in transient {} bucket'.format(
             post_data['file_id'], current_app.config['TRANSIENT_UPLOADED_LETTERS'])
         )
+
         raise e
 
     # Getting the page count won't raise an error since admin has already checked the PDF is valid

app/service/send_pdf_letter_schema.py

@@ -1,5 +1,5 @@
 send_pdf_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST send uploaded pdf letter",
     "type": "object",
     "title": "Send an uploaded pdf letter",

app/service/service_broadcast_settings_schema.py

@@ -1,5 +1,5 @@
 service_broadcast_settings_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Set a services broadcast settings",
     "type": "object",
     "title": "Set a services broadcast settings",

app/service/service_callback_api_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import https_url, uuid
 
 create_service_callback_api_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service callback/inbound api schema",
     "type": "object",
     "title": "Create service callback/inbound api",
@@ -14,7 +14,7 @@ create_service_callback_api_schema = {
 }
 
 update_service_callback_api_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service callback/inbound api schema",
     "type": "object",
     "title": "Create service callback/inbound api",

app/service/service_contact_list_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 create_service_contact_list_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST create service contact list schema",
     "type": "object",
     "title": "Create service contact list",

app/service/service_data_retention_schema.py

@@ -1,5 +1,5 @@
 add_service_data_retention_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service data retention schema",
     "title": "Add service data retention for notification type api",
     "type": "object",
@@ -12,7 +12,7 @@ add_service_data_retention_request = {
 
 
 update_service_data_retention_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service data retention schema",
     "title": "Update service data retention for notification type api",
     "type": "object",

app/service/service_senders_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 add_service_email_reply_to_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service email reply to address",
     "type": "object",
     "title": "Add new email reply to address for service",
@@ -14,7 +14,7 @@ add_service_email_reply_to_request = {
 
 
 add_service_letter_contact_block_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service letter contact block",
     "type": "object",
     "title": "Add new letter contact block for service",
@@ -27,7 +27,7 @@ add_service_letter_contact_block_request = {
 
 
 add_service_sms_sender_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST add service SMS sender",
     "type": "object",
     "title": "Add new SMS sender for service",

app/template_folder/template_folder_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import nullable_uuid, uuid
 
 post_create_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting template_folder",
     "type": "object",
     "properties": {
@@ -12,7 +12,7 @@ post_create_template_folder_schema = {
 }
 
 post_update_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for updating template_folder",
     "type": "object",
     "properties": {
@@ -23,7 +23,7 @@ post_update_template_folder_schema = {
 }
 
 post_move_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for renaming template_folder",
     "type": "object",
     "properties": {

app/user/users_schema.py

@@ -1,5 +1,5 @@
 post_verify_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for verifying a 2fa code',
     'type': 'object',
     'properties': {
@@ -12,7 +12,7 @@ post_verify_code_schema = {
 
 
 post_verify_webauthn_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for verifying a webauthn login attempt',
     'type': 'object',
     'properties': {
@@ -24,7 +24,7 @@ post_verify_webauthn_schema = {
 
 
 post_send_user_email_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': (
         'POST schema for generating a 2fa email - "to" is required for legacy purposes. '
         '"next" is an optional url to redirect to on sign in'
@@ -43,7 +43,7 @@ post_send_user_email_code_schema = {
 
 
 post_send_user_sms_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for generating a 2fa sms',
     'type': 'object',
     'properties': {

app/v2/inbound_sms/inbound_sms_schemas.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 get_inbound_sms_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for query parameters allowed when getting list of received text messages",
     "type": "object",
     "properties": {
@@ -12,7 +12,7 @@ get_inbound_sms_request = {
 
 
 get_inbound_sms_single_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET inbound sms schema response",
     "type": "object",
     "title": "GET response v2/inbound_sms",
@@ -36,7 +36,7 @@ get_inbound_sms_single_response = {
 }
 
 get_inbound_sms_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET list of inbound sms response schema",
     "type": "object",
     "properties": {

app/v2/notifications/notification_schemas.py

@@ -7,7 +7,7 @@ from app.models import (
 from app.schema_validation.definitions import personalisation, uuid
 
 template = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "template schema",
     "type": "object",
     "title": "notification content",
@@ -20,7 +20,7 @@ template = {
 }
 
 notification_by_id = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET notification response schema",
     "type": "object",
     "title": "response v2/notification",
@@ -32,7 +32,7 @@ notification_by_id = {
 
 
 get_notification_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET notification response schema",
     "type": "object",
     "title": "response v2/notification",
@@ -67,7 +67,7 @@ get_notification_response = {
 }
 
 get_notifications_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for query parameters allowed when getting list of notifications",
     "type": "object",
     "properties": {
@@ -92,7 +92,7 @@ get_notifications_request = {
 }
 
 get_notifications_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET list of notifications response schema",
     "type": "object",
     "properties": {
@@ -126,7 +126,7 @@ get_notifications_response = {
 }
 
 post_sms_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification schema",
     "type": "object",
     "title": "POST v2/notifications/sms",
@@ -143,7 +143,7 @@ post_sms_request = {
 }
 
 sms_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "content schema for SMS notification response schema",
     "type": "object",
     "title": "notification content",
@@ -155,7 +155,7 @@ sms_content = {
 }
 
 post_sms_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification response schema",
     "type": "object",
     "title": "response v2/notifications/sms",
@@ -172,7 +172,7 @@ post_sms_response = {
 
 
 post_email_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST email notification schema",
     "type": "object",
     "title": "POST v2/notifications/email",
@@ -189,7 +189,7 @@ post_email_request = {
 }
 
 email_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Email content for POST email notification",
     "type": "object",
     "title": "notification email content",
@@ -202,7 +202,7 @@ email_content = {
 }
 
 post_email_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST email notification response schema",
     "type": "object",
     "title": "response v2/notifications/email",
@@ -218,7 +218,7 @@ post_email_response = {
 }
 
 post_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST letter notification schema",
     "type": "object",
     "title": "POST v2/notifications/letter",
@@ -232,7 +232,7 @@ post_letter_request = {
 }
 
 post_precompiled_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST precompiled letter notification schema",
     "type": "object",
     "title": "POST v2/notifications/letter",
@@ -246,7 +246,7 @@ post_precompiled_letter_request = {
 }
 
 letter_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Letter content for POST letter notification",
     "type": "object",
     "title": "notification letter content",
@@ -258,7 +258,7 @@ letter_content = {
 }
 
 post_letter_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification response schema",
     "type": "object",
     "title": "response v2/notifications/letter",

app/v2/template/template_schemas.py

@@ -2,7 +2,7 @@ from app.models import TEMPLATE_TYPES
 from app.schema_validation.definitions import personalisation, uuid
 
 get_template_by_id_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for parameters allowed when getting template by id",
     "type": "object",
     "properties": {
@@ -14,7 +14,7 @@ get_template_by_id_request = {
 }
 
 get_template_by_id_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET template by id schema response",
     "type": "object",
     "title": "reponse v2/template",
@@ -42,7 +42,7 @@ get_template_by_id_response = {
 }
 
 post_template_preview_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST template schema",
     "type": "object",
     "title": "POST v2/template/{id}/preview",
@@ -54,7 +54,7 @@ post_template_preview_request = {
 }
 
 post_template_preview_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST template preview schema response",
     "type": "object",
     "title": "reponse v2/template/{id}/preview",

app/v2/templates/templates_schemas.py

@@ -4,7 +4,7 @@ from app.v2.template.template_schemas import (
 )
 
 get_all_template_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "request schema for parameters allowed when getting all templates",
     "type": "object",
     "properties": {
@@ -14,7 +14,7 @@ get_all_template_request = {
 }
 
 get_all_template_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET response schema when getting all templates",
     "type": "object",
     "properties": {

gunicorn_config.py

@@ -2,6 +2,8 @@ import os
 import sys
 import traceback
 import gunicorn
+import eventlet
+import socket
 
 from gds_metrics.gunicorn import child_exit  # noqa
 
@@ -30,3 +32,22 @@ def on_exit(server):
 
 def worker_int(worker):
     worker.log.info("worker: received SIGINT {}".format(worker.pid))
+
+
+def fix_ssl_monkeypatching():
+    """
+    eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently
+    a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However
+    eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not
+    ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type
+    :scream:
+
+    https://github.com/eventlet/eventlet/issues/692
+    """
+    # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again.
+    # eventlet.monkey_patch can be called multiple times without issue
+    eventlet.monkey_patch()
+    eventlet.green.ssl.timeout_exc = socket.timeout
+
+
+fix_ssl_monkeypatching()

manifest.yml.j2

@@ -7,9 +7,9 @@
       'STATSD_HOST': None
     },
     'routes': {
-      'preview': ['api.notify.works'],
-      'staging': ['api.staging-notify.works'],
-      'production': ['api.notifications.service.gov.uk'],
+      'preview': ['api.notify.works', 'notify-api-preview.apps.internal'],
+      'staging': ['api.staging-notify.works', 'notify-api-staging.apps.internal'],
+      'production': ['api.notifications.service.gov.uk', 'notify-api-production.apps.internal'],
     },
     'health-check-type': 'port',
     'health-check-invocation-timeout': 3,
@@ -27,9 +27,9 @@
       'STATSD_HOST': None
     },
     'routes': {
-      'preview': ['api.notify.works/notifications/sms/mmg', 'api.notify.works/notifications/sms/firetext', 'api.notify.works/notifications/sms/reach'],
-      'staging': ['api.staging-notify.works/notifications/sms/mmg', 'api.staging-notify.works/notifications/sms/firetext', 'api.staging-notify.works/notifications/sms/reach'],
-      'production': ['api.notifications.service.gov.uk/notifications/sms/mmg', 'api.notifications.service.gov.uk/notifications/sms/firetext', 'api.notifications.service.gov.uk/notifications/sms/reach'],
+      'preview': ['api.notify.works/notifications/sms/mmg', 'api.notify.works/notifications/sms/firetext', 'api.notify.works/notifications/sms/reach','notify-api-sms-receipts-preview.apps.internal'],
+      'staging': ['api.staging-notify.works/notifications/sms/mmg', 'api.staging-notify.works/notifications/sms/firetext', 'api.staging-notify.works/notifications/sms/reach', 'notify-api-sms-receipts-staging.apps.internal'],
+      'production': ['api.notifications.service.gov.uk/notifications/sms/mmg', 'api.notifications.service.gov.uk/notifications/sms/firetext', 'api.notifications.service.gov.uk/notifications/sms/reach', 'notify-api-sms-receipts-production.apps.internal' ],
     },
     'health-check-type': 'port',
     'health-check-invocation-timeout': 3,
@@ -99,6 +99,7 @@ applications:
 
     services:
       - notify-db
+      - notify-redis
       - logit-ssl-syslog-drain
       {% if CF_APP == 'notify-api' %}
       - notify-prometheus
@@ -107,8 +108,10 @@ applications:
 
     env:
       NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
+      NOTIFY_LOG_PATH: /home/vcap/logs/app.log
       SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
       FLASK_APP: application.py
+      NOTIFY_ENVIRONMENT: {{ environment }}
 
       # Credentials variables
       ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
@@ -142,7 +145,6 @@ applications:
       FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
 
       REDIS_ENABLED: '{{ REDIS_ENABLED }}'
-      REDIS_URL: '{{ REDIS_URL }}'
 
       TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
       TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'

migrations/versions/0368_move_orgs_to_nhs_branding_.py

@@ -0,0 +1,31 @@
+"""
+
+Revision ID: 0368_move_orgs_to_nhs_branding
+Revises: 0367_add_reach
+Create Date: 2022-04-12 18:22:12.069016
+
+"""
+from alembic import op
+
+revision = '0368_move_orgs_to_nhs_branding'
+down_revision = '0367_add_reach'
+
+
+def upgrade():
+    op.execute("""
+        UPDATE
+            organisation
+        SET
+            email_branding_id = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
+        WHERE
+            organisation_type IN ('nhs_central', 'nhs_local', 'nhs_gp')
+        AND
+            email_branding_id IS NULL
+    """)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

requirements.in

@@ -2,27 +2,25 @@
 # with package version changes made in requirements.in
 
 cffi==1.15.0
-celery[sqs]==5.2.3
-Flask-Bcrypt==0.7.1
+celery[sqs]==5.2.6
+Flask-Bcrypt==1.0.1
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
-Flask==2.1.0
+Flask==2.1.1
 click-datetime==0.2
 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
 git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0
 iso8601==1.0.2
 itsdangerous==2.1.2
-jsonschema==3.2.0
+jsonschema[format]==4.4.0
 marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
 psycopg2-binary==2.9.3
 PyJWT==2.3.0
-SQLAlchemy==1.4.32
-strict-rfc3339==0.7
-rfc3987==1.3.8
-cachetools==4.2.1
-beautifulsoup4==4.10.0
+SQLAlchemy==1.4.35
+cachetools==5.0.0
+beautifulsoup4==4.11.1
 lxml==4.8.0
 Werkzeug==2.0.3
 
@@ -31,8 +29,8 @@ notifications-python-client==6.3.0
 # PaaS
 awscli-cwlogs==1.4.6
 
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.4
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
-prometheus-client==0.10.1
+prometheus-client==0.14.1
 gds-metrics==0.2.4

requirements.txt

@@ -8,17 +8,17 @@ alembic==1.7.4
     # via flask-migrate
 amqp==5.0.9
     # via kombu
+arrow==1.2.2
+    # via isoduration
 attrs==21.2.0
     # via jsonschema
 awscli==1.21.4
-    # via
-    #   awscli-cwlogs
-    #   notifications-utils
+    # via awscli-cwlogs
 awscli-cwlogs==1.4.6
     # via -r requirements.in
 bcrypt==3.2.0
     # via flask-bcrypt
-beautifulsoup4==4.10.0
+beautifulsoup4==4.11.1
     # via -r requirements.in
 billiard==3.6.4.0
     # via celery
@@ -33,11 +33,11 @@ botocore==1.22.4
     #   awscli
     #   boto3
     #   s3transfer
-cachetools==4.2.1
+cachetools==5.0.0
     # via
     #   -r requirements.in
     #   notifications-utils
-celery[sqs]==5.2.3
+celery[sqs]==5.2.6
     # via -r requirements.in
 certifi==2021.10.8
     # via
@@ -75,7 +75,7 @@ docutils==0.15.2
     # via awscli
 eventlet==0.33.0
     # via gunicorn
-flask==2.1.0
+flask==2.1.1
     # via
     #   -r requirements.in
     #   flask-bcrypt
@@ -84,7 +84,7 @@ flask==2.1.0
     #   flask-redis
     #   gds-metrics
     #   notifications-utils
-flask-bcrypt==0.7.1
+flask-bcrypt==1.0.1
     # via -r requirements.in
 flask-marshmallow==0.14.0
     # via -r requirements.in
@@ -96,6 +96,8 @@ flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732
     # via
     #   -r requirements.in
     #   flask-migrate
+fqdn==1.5.1
+    # via jsonschema
 gds-metrics==0.2.4
     # via -r requirements.in
 geojson==2.5.0
@@ -107,11 +109,15 @@ greenlet==1.1.2
 gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64
     # via -r requirements.in
 idna==3.3
-    # via requests
+    # via
+    #   jsonschema
+    #   requests
 importlib-metadata==4.11.3
     # via flask
 iso8601==1.0.2
     # via -r requirements.in
+isoduration==20.11.0
+    # via jsonschema
 itsdangerous==2.1.2
     # via
     #   -r requirements.in
@@ -125,7 +131,9 @@ jmespath==0.10.0
     # via
     #   boto3
     #   botocore
-jsonschema==3.2.0
+jsonpointer==2.3
+    # via jsonschema
+jsonschema[format]==4.4.0
     # via -r requirements.in
 kombu==5.2.3
     # via celery
@@ -148,7 +156,7 @@ mistune==0.8.4
     # via notifications-utils
 notifications-python-client==6.3.0
     # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.4
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
@@ -156,7 +164,7 @@ packaging==21.0
     # via bleach
 phonenumbers==8.12.36
     # via notifications-utils
-prometheus-client==0.10.1
+prometheus-client==0.14.1
     # via
     #   -r requirements.in
     #   gds-metrics
@@ -182,6 +190,7 @@ pyrsistent==0.18.0
     # via jsonschema
 python-dateutil==2.8.2
     # via
+    #   arrow
     #   awscli-cwlogs
     #   botocore
 python-json-logger==2.0.2
@@ -202,8 +211,10 @@ requests==2.26.0
     #   govuk-bank-holidays
     #   notifications-python-client
     #   notifications-utils
+rfc3339-validator==0.1.4
+    # via jsonschema
 rfc3987==1.3.8
-    # via -r requirements.in
+    # via jsonschema
 rsa==4.7.2
     # via awscli
 s3transfer==0.5.0
@@ -220,21 +231,21 @@ six==1.16.0
     #   click-repl
     #   eventlet
     #   flask-marshmallow
-    #   jsonschema
     #   python-dateutil
+    #   rfc3339-validator
 smartypants==2.0.1
     # via notifications-utils
 soupsieve==2.2.1
     # via beautifulsoup4
-sqlalchemy==1.4.32
+sqlalchemy==1.4.35
     # via
     #   -r requirements.in
     #   alembic
     #   marshmallow-sqlalchemy
 statsd==3.3.0
     # via notifications-utils
-strict-rfc3339==0.7
-    # via -r requirements.in
+uri-template==1.2.0
+    # via jsonschema
 urllib3==1.26.7
     # via
     #   botocore
@@ -246,6 +257,8 @@ vine==5.0.0
     #   kombu
 wcwidth==0.2.5
     # via prompt-toolkit
+webcolors==1.11.1
+    # via jsonschema
 webencodings==0.5.1
     # via bleach
 werkzeug==2.0.3

requirements_for_test.txt

@@ -1,14 +1,14 @@
 -r requirements.txt
 flake8==4.0.1
-flake8-bugbear==22.1.11
+flake8-bugbear==22.3.23
 isort==5.10.1
-moto==3.0.7
-pytest==7.0.1
+moto==3.1.4
+pytest==7.1.1
 pytest-env==0.6.2
 pytest-mock==3.7.0
 pytest-cov==3.0.0
 pytest-xdist==2.5.0
-freezegun==1.2.0
+freezegun==1.2.1
 requests-mock==1.9.3
 # used for creating manifest file locally
-jinja2-cli[yaml]==0.8.1
+jinja2-cli[yaml]==0.8.2

tests/app/clients/test_sms.py

@@ -24,7 +24,7 @@ def test_send_sms(fake_client, mocker):
         content='content',
         reference='reference',
         international=False,
-        sender=None,
+        sender='testing',
     )
 
     mock_send.assert_called_with(
@@ -45,22 +45,3 @@ def test_send_sms_error(fake_client, mocker):
             international=False,
             sender=None,
         )
-
-
-def test_send_sms_override_configured_shortcode_with_sender(
-    fake_client,
-    mocker
-):
-    mock_send = mocker.patch.object(fake_client, 'try_send_sms')
-
-    fake_client.send_sms(
-        to='to',
-        content='content',
-        reference='reference',
-        international=False,
-        sender='sender'
-    )
-
-    mock_send.assert_called_with(
-        'to', 'content', 'reference', False, 'sender'
-    )

tests/app/complaint/test_complaint_rest.py

@@ -87,5 +87,4 @@ def test_get_complaint_with_invalid_data_returns_400_status_code(client):
     )
 
     assert response.status_code == 400
-    assert response.json['errors'][0]['message'] == 'start_date time data {} does not match format %Y-%m-%d'.format(
-        start_date)
+    assert response.json['errors'][0]['message'] == 'start_date month must be in 1..12'

tests/app/conftest.py

@@ -53,6 +53,7 @@ from app.models import (
 from tests import create_admin_authorization_header
 from tests.app.db import (
     create_api_key,
+    create_email_branding,
     create_inbound_number,
     create_invited_org_user,
     create_job,
@@ -919,6 +920,19 @@ def broadcast_organisation(notify_db_session):
     return org
 
 
+@pytest.fixture
+def nhs_email_branding(notify_db_session):
+    # we wipe email_branding table in test db between the tests, so we have to recreate this branding
+    # that is normally present on all environments and applied through migration
+    nhs_email_branding_id = current_app.config['NHS_EMAIL_BRANDING_ID']
+
+    return create_email_branding(
+        id=nhs_email_branding_id,
+        logo='1ac6f483-3105-4c9e-9017-dd7fb2752c44-nhs-blue_x2.png',
+        name='NHS'
+    )
+
+
 @pytest.fixture
 def restore_provider_details(notify_db, notify_db_session):
     """

tests/app/dao/test_email_branding_dao.py

@@ -1,6 +1,5 @@
 from app.dao.email_branding_dao import (
     dao_get_email_branding_by_id,
-    dao_get_email_branding_by_name,
     dao_get_email_branding_options,
     dao_update_email_branding,
 )
@@ -27,14 +26,6 @@ def test_get_email_branding_by_id_gets_correct_email_branding(notify_db, notify_
     assert email_branding_from_db == email_branding
 
 
-def test_get_email_branding_by_name_gets_correct_email_branding(notify_db, notify_db_session):
-    email_branding = create_email_branding(name="Crystal Gems")
-
-    email_branding_from_db = dao_get_email_branding_by_name("Crystal Gems")
-
-    assert email_branding_from_db == email_branding
-
-
 def test_update_email_branding(notify_db, notify_db_session):
     updated_name = 'new name'
     create_email_branding()

tests/app/dao/test_ft_billing_dao.py

@@ -10,6 +10,7 @@ from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
+    fetch_daily_sms_provider_volumes_for_platform,
     fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
@@ -856,6 +857,84 @@ def test_fetch_daily_volumes_for_platform(
     assert results[1].letter_sheet_totals == 40
 
 
+def test_fetch_daily_sms_provider_volumes_for_platform_groups_values_by_provider(
+    notify_db_session,
+):
+    services = [
+        create_service(service_name='a'),
+        create_service(service_name='b')
+    ]
+    templates = [
+        create_template(services[0]),
+        create_template(services[1])
+    ]
+
+    create_ft_billing('2022-02-01', templates[0], provider='foo', notifications_sent=1, billable_unit=2)
+    create_ft_billing('2022-02-01', templates[1], provider='foo', notifications_sent=4, billable_unit=8)
+
+    create_ft_billing('2022-02-01', templates[0], provider='bar', notifications_sent=16, billable_unit=32)
+    create_ft_billing('2022-02-01', templates[1], provider='bar', notifications_sent=64, billable_unit=128)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 2
+    assert results[0].provider == 'bar'
+    assert results[0].sms_totals == 80
+    assert results[0].sms_fragment_totals == 160
+
+    assert results[1].provider == 'foo'
+    assert results[1].sms_totals == 5
+    assert results[1].sms_fragment_totals == 10
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_calculates_chargeable_units_and_costs(
+    sample_template,
+):
+    create_ft_billing('2022-02-01', sample_template, rate_multiplier=3, rate=1.5, notifications_sent=1, billable_unit=2)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 1
+    assert results[0].sms_totals == 1
+    assert results[0].sms_fragment_totals == 2
+    assert results[0].sms_chargeable_units == 6
+    assert results[0].sms_cost == 9
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_searches_dates_inclusively(sample_template):
+    # too early
+    create_ft_billing('2022-02-02', sample_template)
+
+    # just right
+    create_ft_billing('2022-02-03', sample_template)
+    create_ft_billing('2022-02-04', sample_template)
+    create_ft_billing('2022-02-05', sample_template)
+
+    # too late
+    create_ft_billing('2022-02-06', sample_template)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-03', end_date='2022-02-05')
+
+    assert len(results) == 3
+    assert results[0].bst_date == date(2022, 2, 3)
+    assert results[-1].bst_date == date(2022, 2, 5)
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_only_returns_sms(
+    sample_template,
+    sample_email_template,
+    sample_letter_template
+):
+    create_ft_billing('2022-02-01', sample_template, notifications_sent=1)
+    create_ft_billing('2022-02-01', sample_email_template, notifications_sent=2)
+    create_ft_billing('2022-02-01', sample_letter_template, notifications_sent=4)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 1
+    assert results[0].sms_totals == 1
+
+
 def test_fetch_volumes_by_service(notify_db_session):
     set_up_usage_data(datetime(2022, 2, 1))
 

tests/app/db.py

@@ -502,13 +502,17 @@ def create_service_callback_api(
     return service_callback_api
 
 
-def create_email_branding(colour='blue', logo='test_x2.png', name='test_org_1', text='DisplayName'):
+def create_email_branding(
+    id=None, colour='blue', logo='test_x2.png', name='test_org_1', text='DisplayName'
+):
     data = {
         'colour': colour,
         'logo': logo,
         'name': name,
         'text': text,
     }
+    if id:
+        data['id'] = id
     email_branding = EmailBranding(**data)
     dao_create_email_branding(email_branding)
 
@@ -671,6 +675,7 @@ def create_organisation(
     billing_contact_names=None,
     billing_contact_email_addresses=None,
     billing_reference=None,
+    email_branding_id=None,
 ):
     data = {
         'id': organisation_id,
@@ -681,6 +686,7 @@ def create_organisation(
         'billing_contact_names': billing_contact_names,
         'billing_contact_email_addresses': billing_contact_email_addresses,
         'billing_reference': billing_reference,
+        'email_branding_id': email_branding_id
     }
     organisation = Organisation(**data)
     dao_create_organisation(organisation)

tests/app/organisation/test_rest.py

@@ -2,6 +2,7 @@ import uuid
 from datetime import datetime
 
 import pytest
+from flask import current_app
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -177,14 +178,39 @@ def test_post_create_organisation(admin_request, notify_db_session, crown):
         _expected_status=201
     )
 
-    organisation = Organisation.query.all()
+    organisations = Organisation.query.all()
 
     assert data['name'] == response['name']
     assert data['active'] == response['active']
     assert data['crown'] == response['crown']
     assert data['organisation_type'] == response['organisation_type']
 
-    assert len(organisation) == 1
+    assert len(organisations) == 1
+    # check that for non-nhs orgs, default branding is not set
+    assert organisations[0].email_branding_id is None
+
+
+@pytest.mark.parametrize('org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_create_organisation_sets_default_nhs_branding_for_nhs_orgs(
+    admin_request, notify_db_session, nhs_email_branding, org_type
+):
+    data = {
+        'name': 'test organisation',
+        'active': True,
+        'crown': False,
+        'organisation_type': org_type,
+    }
+
+    admin_request.post(
+        'organisation.create_organisation',
+        _data=data,
+        _expected_status=201
+    )
+
+    organisations = Organisation.query.all()
+
+    assert len(organisations) == 1
+    assert organisations[0].email_branding_id == uuid.UUID(current_app.config['NHS_EMAIL_BRANDING_ID'])
 
 
 def test_post_create_organisation_existing_name_raises_400(admin_request, sample_organisation):
@@ -344,6 +370,64 @@ def test_update_other_organisation_attributes_doesnt_clear_domains(
     ]
 
 
+@pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present(
+    admin_request,
+    nhs_email_branding,
+    notify_db_session,
+    new_org_type
+):
+    org = create_organisation(organisation_type='central')
+    data = {
+        'organisation_type': new_org_type,
+    }
+
+    admin_request.post(
+        'organisation.update_organisation',
+        _data=data,
+        organisation_id=org.id,
+        _expected_status=204
+    )
+
+    organisation = Organisation.query.all()
+
+    assert len(organisation) == 1
+    assert organisation[0].id == org.id
+    assert organisation[0].organisation_type == new_org_type
+    assert organisation[0].email_branding_id == uuid.UUID(current_app.config['NHS_EMAIL_BRANDING_ID'])
+
+
+@pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_update_organisation_to_nhs_type_does_not_update_branding_if_default_branding_set(
+    admin_request,
+    nhs_email_branding,
+    notify_db_session,
+    new_org_type
+):
+    current_branding = create_email_branding(
+        logo='example.png',
+        name='custom branding'
+    )
+    org = create_organisation(organisation_type='central', email_branding_id=current_branding.id)
+    data = {
+        'organisation_type': new_org_type,
+    }
+
+    admin_request.post(
+        'organisation.update_organisation',
+        _data=data,
+        organisation_id=org.id,
+        _expected_status=204
+    )
+
+    organisation = Organisation.query.all()
+
+    assert len(organisation) == 1
+    assert organisation[0].id == org.id
+    assert organisation[0].organisation_type == new_org_type
+    assert organisation[0].email_branding_id == current_branding.id
+
+
 def test_update_organisation_default_branding(
     admin_request,
     notify_db_session,

tests/app/platform_stats/test_rest.py

@@ -9,6 +9,7 @@ from app.platform_stats.rest import (
     validate_date_range_is_within_a_financial_year,
 )
 from tests.app.db import (
+    create_ft_billing,
     create_ft_notification_status,
     create_notification,
     create_service,
@@ -47,8 +48,7 @@ def test_get_platform_stats_validates_the_date(admin_request):
         _expected_status=400
     )
 
-    assert response['errors'][0]['message'] == 'start_date time data {} does not match format %Y-%m-%d'.format(
-        start_date)
+    assert response['errors'][0]['message'] == 'start_date month must be in 1..12'
 
 
 @freeze_time('2018-10-31 14:00')
@@ -238,3 +238,23 @@ def test_volumes_by_service_report(
                            'service_id': str(fixture['service_with_sms_within_allowance'].id),
                            'service_name': fixture['service_with_sms_within_allowance'].name,
                            'sms_chargeable_units': 0, 'sms_notifications': 0}
+
+
+def test_daily_sms_provider_volumes_report(admin_request, sample_template):
+
+    create_ft_billing('2022-03-01', sample_template, provider='foo', rate=1.5, notifications_sent=1, billable_unit=3)
+    resp = admin_request.get(
+        'platform_stats.daily_sms_provider_volumes_report',
+        start_date='2022-03-01',
+        end_date='2022-03-01'
+    )
+
+    assert len(resp) == 1
+    assert resp[0] == {
+        'day': '2022-03-01',
+        'provider': 'foo',
+        'sms_totals': 1,
+        'sms_fragment_totals': 3,
+        'sms_chargeable_units': 3,
+        'sms_cost': 4.5,
+    }

tests/app/public_contracts/__init__.py

@@ -2,7 +2,7 @@ import os
 
 import jsonschema
 from flask import json
-from jsonschema import Draft4Validator
+from jsonschema import Draft7Validator
 
 
 def return_json_from_response(response):
@@ -22,5 +22,5 @@ def validate_v0(json_to_validate, schema_filename):
 
 
 def validate(json_to_validate, schema):
-    validator = Draft4Validator(schema)
+    validator = Draft7Validator(schema)
     validator.validate(json_to_validate, schema)

tests/app/public_contracts/schemas/v0/GET_notification_return_email.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for email notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/GET_notification_return_sms.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/GET_notifications_return.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/POST_notification_return_email.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "POST notification return schema - for email notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/POST_notification_return_sms.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "POST notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/definitions.json

@@ -1,12 +1,8 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Common definitions - usage example: {'$ref': 'definitions.json#/uuid'} (swap quotes for double quotes)",
   "uuid": {
     "type": "string",
     "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
-  },
-  "datetime": {
-    "type": "string",
-    "format": "date-time"
   }
 }

tests/app/public_contracts/schemas/v0/email_notification.json

@@ -14,19 +14,13 @@
       "type": "string",
       "enum": ["email"]
     },
-    "created_at": {"$ref": "definitions.json#/datetime"},
-    "sent_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "created_at": {"type": "string", "format": "date-time"},
+    "sent_at": {"type": ["string", "null"], "format": "date-time"},
     "sent_by": {"oneOf":[
       {"type": "string"},
       {"type": "null"}
     ]},
-    "updated_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "updated_at": {"type": ["string", "null"], "format": "date-time"},
     "status": {
       "type": "string",
       "enum": [

tests/app/public_contracts/schemas/v0/sms_notification.json

@@ -14,19 +14,13 @@
       "type": "string",
       "enum": ["sms"]
     },
-    "created_at": {"$ref": "definitions.json#/datetime"},
-    "sent_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "created_at": {"type": "string", "format": "date-time"},
+    "sent_at": {"type": ["string", "null"], "format": "date-time"},
     "sent_by": {"oneOf":[
       {"type": "string"},
       {"type": "null"}
     ]},
-    "updated_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "updated_at": {"type": ["string", "null"], "format": "date-time"},
     "status": {
       "type": "string",
       "enum": [

tests/app/service/send_notification/test_send_pdf_letter_notification.py

@@ -1,5 +1,3 @@
-import uuid
-
 import pytest
 from freezegun import freeze_time
 from notifications_utils.s3 import S3ObjectNotFound
@@ -11,18 +9,27 @@ from app.v2.errors import BadRequestError, TooManyRequestsError
 from tests.app.db import create_service
 
 
+@pytest.fixture
+def post_data(sample_service_full_permissions, fake_uuid):
+    return {
+        'filename': 'valid.pdf',
+        'created_by': sample_service_full_permissions.users[0].id,
+        'file_id': fake_uuid,
+        'postage': 'second',
+        'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'
+    }
+
+
 @pytest.mark.parametrize('permissions', [
     [EMAIL_TYPE],
     [UPLOAD_LETTERS],
 ])
 def test_send_pdf_letter_notification_raises_error_if_service_does_not_have_permission(
     notify_db_session,
-    fake_uuid,
     permissions,
+    post_data,
 ):
     service = create_service(service_permissions=permissions)
-    post_data = {'filename': 'valid.pdf', 'created_by': fake_uuid, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(BadRequestError):
         send_pdf_letter_notification(service.id, post_data)
@@ -31,23 +38,22 @@ def test_send_pdf_letter_notification_raises_error_if_service_does_not_have_perm
 def test_send_pdf_letter_notification_raises_error_if_service_is_over_daily_message_limit(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
+    post_data,
 ):
     mocker.patch(
         'app.service.send_notification.check_service_over_daily_message_limit',
         side_effect=TooManyRequestsError(10))
-    post_data = {'filename': 'valid.pdf', 'created_by': fake_uuid, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(TooManyRequestsError):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
 
 
 def test_send_pdf_letter_notification_validates_created_by(
-    sample_service_full_permissions, fake_uuid, sample_user
+    sample_service_full_permissions,
+    sample_user,
+    post_data
 ):
-    post_data = {'filename': 'valid.pdf', 'created_by': sample_user.id, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
+    post_data['created_by'] = sample_user.id
 
     with pytest.raises(BadRequestError):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
@@ -56,12 +62,9 @@ def test_send_pdf_letter_notification_validates_created_by(
 def test_send_pdf_letter_notification_raises_error_if_service_in_trial_mode(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
+    post_data,
 ):
     sample_service_full_permissions.restricted = True
-    user = sample_service_full_permissions.users[0]
-    post_data = {'filename': 'valid.pdf', 'created_by': user.id, 'file_id': fake_uuid,
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(BadRequestError) as e:
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
@@ -71,49 +74,54 @@ def test_send_pdf_letter_notification_raises_error_if_service_in_trial_mode(
 def test_send_pdf_letter_notification_raises_error_when_pdf_is_not_in_transient_letter_bucket(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
     notify_user,
+    post_data,
 ):
-    user = sample_service_full_permissions.users[0]
-    post_data = {'filename': 'valid.pdf', 'created_by': user.id, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
     mocker.patch('app.service.send_notification.utils_s3download', side_effect=S3ObjectNotFound({}, ''))
 
     with pytest.raises(S3ObjectNotFound):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
 
 
+def test_send_pdf_letter_notification_does_nothing_if_notification_already_exists(
+    mocker,
+    sample_service_full_permissions,
+    notify_user,
+    sample_notification,
+    post_data,
+):
+    post_data['file_id'] = sample_notification.id
+    mocker.patch('app.service.send_notification.utils_s3download', side_effect=S3ObjectNotFound({}, ''))
+    response = send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
+    assert response['id'] == str(sample_notification.id)
+
+
 @freeze_time("2019-08-02 11:00:00")
 def test_send_pdf_letter_notification_creates_notification_and_moves_letter(
     mocker,
     sample_service_full_permissions,
     notify_user,
+    post_data,
 ):
-    user = sample_service_full_permissions.users[0]
-    filename = 'valid.pdf'
-    file_id = uuid.uuid4()
-    post_data = {'filename': filename, 'created_by': user.id, 'file_id': file_id, 'postage': 'second',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
-
     mocker.patch('app.service.send_notification.utils_s3download')
     mocker.patch('app.service.send_notification.get_page_count', return_value=1)
     s3_mock = mocker.patch('app.service.send_notification.move_uploaded_pdf_to_letters_bucket')
 
     result = send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
+    file_id = post_data['file_id']
 
     notification = get_notification_by_id(file_id)
 
-    assert notification.id == file_id
+    assert str(notification.id) == file_id
     assert notification.api_key_id is None
-    assert notification.client_reference == filename
-    assert notification.created_by_id == user.id
+    assert notification.client_reference == post_data['filename']
+    assert notification.created_by_id == post_data['created_by']
     assert notification.postage == 'second'
     assert notification.notification_type == LETTER_TYPE
     assert notification.billable_units == 1
     assert notification.to == "Bugs Bunny\n123 Main Street\nLooney Town"
 
     assert notification.service_id == sample_service_full_permissions.id
-
     assert result == {'id': str(notification.id)}
 
     s3_mock.assert_called_once_with(

tests/app/test_cloudfoundry_config.py

@@ -3,52 +3,29 @@ import os
 
 import pytest
 
-from app.cloudfoundry_config import (
-    extract_cloudfoundry_config,
-    set_config_env_vars,
-)
+from app.cloudfoundry_config import extract_cloudfoundry_config
 
 
 @pytest.fixture
-def postgres_config():
-    return [
-        {
+def vcap_services():
+    return {
+        'postgres': [{
             'credentials': {
                 'uri': 'postgres uri'
             }
-        }
-    ]
-
-
-@pytest.fixture
-def cloudfoundry_config(postgres_config):
-    return {
-        'postgres': postgres_config,
+        }],
+        'redis': [{
+            'credentials': {
+                'uri': 'redis uri'
+            }
+        }],
         'user-provided': []
     }
 
 
-@pytest.fixture
-def cloudfoundry_environ(os_environ, cloudfoundry_config):
-    os.environ['VCAP_SERVICES'] = json.dumps(cloudfoundry_config)
-    os.environ['VCAP_APPLICATION'] = '{"space_name": "🚀🌌"}'
-
-
-def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_environ):
+def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
+    os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
     extract_cloudfoundry_config()
 
     assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri'
-    assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌'
-    assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log'
-
-
-def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config, cloudfoundry_environ):
-    cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}}
-    cloudfoundry_config['user-provided'].append({
-        'name': 'bar', 'credentials': {'bar': 'bar'}
-    })
-
-    set_config_env_vars(cloudfoundry_config)
-
-    assert 'foo' not in os.environ
-    assert 'bar' not in os.environ
+    assert os.environ['REDIS_URL'] == 'redis uri'

tests/app/v2/broadcast/sample_cap_xml_documents.py

@@ -248,5 +248,5 @@ WINDEMERE = """
 """
 
 LONG_GSM7 = WITH_PLACEHOLDER_FOR_CONTENT.format('a' * 1396)
-LONG_UCS2 = WITH_PLACEHOLDER_FOR_CONTENT.format('ŵ' * 616)
+LONG_UCS2 = WITH_PLACEHOLDER_FOR_CONTENT.format('ŵyl' * 205 + 'a')
 MISSING_AREA_NAMES = re.sub("<areaDesc>.*</areaDesc>", "<areaDesc> </areaDesc>", WAINFLEET)