Fix incorrect chargeable_units column

This was added to migrate away from the vague "billing_units" field,
but without fixing the inconsistent data behind it:

- For emails and letters, "billing_units" was just the number sent.

- For SMS, "billing_units" really was the chargeable_units.

To avoid confusion we need two fields to represent the original mix
of data - this exposes "notifications_sent" in both APIs.

app/billing/billing_schemas.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 create_or_update_free_sms_fragment_limit_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST annual billing schema",
     "type": "object",
     "title": "Create",
@@ -12,30 +12,40 @@ create_or_update_free_sms_fragment_limit_schema = {
 }
 
 
-def serialize_ft_billing_remove_emails(data):
-    results = []
-    billed_notifications = [x for x in data if x.notification_type != 'email']
-    for notification in billed_notifications:
-        json_result = {
-            "month": (datetime.strftime(notification.month, "%B")),
-            "notification_type": notification.notification_type,
-            "billing_units": notification.billable_units,
-            "rate": float(notification.rate),
-            "postage": notification.postage,
+def serialize_ft_billing_remove_emails(rows):
+    return [
+        {
+            "month": (datetime.strftime(row.month, "%B")),
+            "notification_type": row.notification_type,
+            # TEMPORARY: while we migrate to "chargeable_units" in the Admin app
+            "billing_units": row.billable_units,
+            "chargeable_units": row.chargeable_units,
+            "rate": float(row.rate),
+            "postage": row.postage,
+            "cost": float(row.cost),
+            "free_chargeable_units": row.free_chargeable_units,
+            "charged_units": row.charged_units,
+            "notifications_sent": row.notifications_sent,
         }
-        results.append(json_result)
-    return results
+        for row in rows
+        if row.notification_type != 'email'
+    ]
 
 
-def serialize_ft_billing_yearly_totals(data):
-    yearly_totals = []
-    for total in data:
-        json_result = {
-            "notification_type": total.notification_type,
-            "billing_units": total.billable_units,
-            "rate": float(total.rate),
-            "letter_total": float(total.billable_units * total.rate) if total.notification_type == 'letter' else 0
+def serialize_ft_billing_yearly_totals(rows):
+    return [
+        {
+            "notification_type": row.notification_type,
+            # TEMPORARY: while we migrate to "chargeable_units" in the Admin app
+            "billing_units": row.billable_units,
+            "chargeable_units": row.chargeable_units,
+            "rate": float(row.rate),
+            # TEMPORARY: while we migrate to "cost" in the Admin app
+            "letter_total": float(row.billable_units * row.rate) if row.notification_type == 'letter' else 0,
+            "cost": float(row.cost),
+            "free_chargeable_units": row.free_chargeable_units,
+            "charged_units": row.charged_units,
+            "notifications_sent": row.notifications_sent,
         }
-        yearly_totals.append(json_result)
-
-    return yearly_totals
+        for row in rows
+    ]

app/billing/rest.py

@@ -30,7 +30,6 @@ billing_blueprint = Blueprint(
 register_errors(billing_blueprint)
 
 
-@billing_blueprint.route('/ft-monthly-usage')
 @billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
@@ -42,7 +41,6 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     return jsonify(data)
 
 
-@billing_blueprint.route('/ft-yearly-usage-summary')
 @billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:

app/broadcast_message/broadcast_message_schema.py

@@ -2,7 +2,7 @@ from app.models import BroadcastStatusType
 from app.schema_validation.definitions import uuid
 
 create_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST create broadcast_message schema',
     'type': 'object',
     'title': 'Create broadcast_message',
@@ -32,7 +32,7 @@ create_broadcast_message_schema = {
 }
 
 update_broadcast_message_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST update broadcast_message schema',
     'type': 'object',
     'title': 'Update broadcast_message',
@@ -47,7 +47,7 @@ update_broadcast_message_schema = {
 }
 
 update_broadcast_message_status_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST update broadcast_message status schema',
     'type': 'object',
     'title': 'Update broadcast_message',

app/clients/sms/__init__.py

@@ -23,7 +23,6 @@ class SmsClient(Client):
     def init_app(self, current_app, statsd_client):
         self.current_app = current_app
         self.statsd_client = statsd_client
-        self.from_number = self.current_app.config.get('FROM_NUMBER')
 
     def record_outcome(self, success):
         log_message = "Provider request for {} {}".format(
@@ -41,15 +40,6 @@ class SmsClient(Client):
     def send_sms(self, to, content, reference, international, sender):
         start_time = monotonic()
 
-        if sender is None:
-            # temporary log to see if the following ternary is necessary
-            # or if it's safe to remove it - keep for 1-2 weeks
-            self.current_app.logger.warning(
-                f"send_sms called with 'sender' of 'None' for {reference}"
-            )
-
-        sender = self.from_number if sender is None else sender
-
         try:
             response = self.try_send_sms(to, content, reference, international, sender)
             self.record_outcome(True)

app/cloudfoundry_config.py

@@ -1,22 +1,12 @@
-"""
-Extracts cloudfoundry config from its json and populates the environment variables that we would expect to be populated
-on local/aws boxes
-"""
-
 import json
 import os
 
 
 def extract_cloudfoundry_config():
     vcap_services = json.loads(os.environ['VCAP_SERVICES'])
-    set_config_env_vars(vcap_services)
 
-
-def set_config_env_vars(vcap_services):
     # Postgres config
     os.environ['SQLALCHEMY_DATABASE_URI'] = vcap_services['postgres'][0]['credentials']['uri'].replace('postgres',
                                                                                                        'postgresql')
-
-    vcap_application = json.loads(os.environ['VCAP_APPLICATION'])
-    os.environ['NOTIFY_ENVIRONMENT'] = vcap_application['space_name']
-    os.environ['NOTIFY_LOG_PATH'] = '/home/vcap/logs/app.log'
+    # Redis config
+    os.environ['REDIS_URL'] = vcap_services['redis'][0]['credentials']['uri']

app/complaint/complaint_schema.py

@@ -1,6 +1,6 @@
 
 complaint_count_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "complaint count request schema",
     "type": "object",
     "title": "Complaint count request",

app/config.py

@@ -114,7 +114,7 @@ class Config(object):
 
     # URL of redis instance
     REDIS_URL = os.getenv('REDIS_URL')
-    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
+    REDIS_ENABLED = True
     EXPIRE_CACHE_TEN_MINUTES = 600
     EXPIRE_CACHE_EIGHT_DAYS = 8 * 24 * 60 * 60
 
@@ -185,6 +185,7 @@ class Config(object):
     MOU_SIGNED_ON_BEHALF_ON_BEHALF_RECEIPT_TEMPLATE_ID = '522b6657-5ca5-4368-a294-6b527703bd0b'
     NOTIFY_INTERNATIONAL_SMS_SENDER = '07984404008'
     LETTERS_VOLUME_EMAIL_TEMPLATE_ID = '11fad854-fd38-4a7c-bd17-805fb13dfc12'
+    NHS_EMAIL_BRANDING_ID = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
     # we only need real email in Live environment (production)
     DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
 
@@ -403,6 +404,8 @@ class Development(Config):
     DEBUG = True
     SQLALCHEMY_ECHO = False
 
+    REDIS_ENABLED = os.getenv('REDIS_ENABLED') == '1'
+
     CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
     CONTACT_LIST_BUCKET_NAME = 'development-contact-list'
     TEST_LETTERS_BUCKET_NAME = 'development-test-letters'

app/dao/date_util.py

@@ -1,7 +1,7 @@
 from datetime import date, datetime, time, timedelta
 
 import pytz
-from notifications_utils.timezones import convert_bst_to_utc
+from notifications_utils.timezones import convert_bst_to_utc, convert_utc_to_bst
 
 
 def get_months_for_financial_year(year):
@@ -22,6 +22,15 @@ def get_financial_year(year):
     return get_april_fools(year), get_april_fools(year + 1) - timedelta(microseconds=1)
 
 
+def get_financial_year_dates(year):
+    year_start_datetime, year_end_datetime = get_financial_year(year)
+
+    return (
+        convert_utc_to_bst(year_start_datetime).date(),
+        convert_utc_to_bst(year_end_datetime).date()
+    )
+
+
 def get_current_financial_year():
     now = datetime.utcnow()
     current_month = int(now.strftime('%-m'))

app/dao/fact_billing_dao.py

@@ -2,13 +2,13 @@ from datetime import date, datetime, timedelta
 
 from flask import current_app
 from notifications_utils.timezones import convert_utc_to_bst
-from sqlalchemy import Date, Integer, and_, desc, func
+from sqlalchemy import Date, Integer, and_, desc, func, union
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.sql.expression import case, literal
 
 from app import db
 from app.dao.date_util import (
-    get_financial_year,
+    get_financial_year_dates,
     get_financial_year_for_datetime,
 )
 from app.dao.organisation_dao import dao_get_organisation_live_services
@@ -197,111 +197,179 @@ def fetch_letter_line_items_for_all_services(start_date, end_date):
 
 
 def fetch_billing_totals_for_year(service_id, year):
-    year_start_date, year_end_date = get_financial_year(year)
-    """
-      Billing for email: only record the total number of emails.
-      Billing for letters: The billing units is used to fetch the correct rate for the sheet count of the letter.
-      Total cost is notifications_sent * rate.
-      Rate multiplier does not apply to email or letters.
-    """
-    email_and_letters = db.session.query(
-        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.notifications_sent).label("billable_units"),
-        FactBilling.rate.label('rate'),
-        FactBilling.notification_type.label('notification_type')
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date,
-        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
-    ).group_by(
-        FactBilling.rate,
-        FactBilling.notification_type
-    )
-    """
-    Billing for SMS using the billing_units * rate_multiplier. Billing unit of SMS is the fragment count of a message
-    """
-    sms = db.session.query(
-        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
-        FactBilling.rate,
-        FactBilling.notification_type
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date,
-        FactBilling.notification_type == SMS_TYPE
-    ).group_by(
-        FactBilling.rate,
-        FactBilling.notification_type
-    )
-
-    yearly_data = email_and_letters.union_all(sms).order_by(
-        'notification_type',
-        'rate'
+    return db.session.query(
+        union(*[
+            db.session.query(
+                func.sum(query.c.notifications_sent).label("notifications_sent"),
+                # TEMPORARY: while we switch to "chargeable units"
+                func.sum(query.c.billable_units).label("billable_units"),
+                func.sum(query.c.chargeable_units).label("chargeable_units"),
+                query.c.rate.label("rate"),
+                query.c.notification_type.label("notification_type"),
+                func.sum(query.c.cost).label("cost"),
+                func.sum(query.c.free_chargeable_units).label("free_chargeable_units"),
+                func.sum(query.c.charged_units).label("charged_units"),
+            ).group_by(
+                query.c.rate,
+                query.c.notification_type
+            )
+            for query in [
+                query_service_sms_usage_for_year(service_id, year).subquery(),
+                query_service_email_usage_for_year(service_id, year).subquery(),
+                query_service_letter_usage_for_year(service_id, year).subquery(),
+            ]
+        ]).subquery()
+    ).order_by(
+        "notification_type",
+        "rate",
     ).all()
 
-    return yearly_data
-
 
 def fetch_monthly_billing_for_year(service_id, year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    year_start_date = convert_utc_to_bst(year_start_datetime).date()
-    year_end_date = convert_utc_to_bst(year_end_datetime).date()
-
+    _, year_end = get_financial_year_dates(year)
     today = convert_utc_to_bst(datetime.utcnow()).date()
+
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end_date >= today:
+    if year_end >= today:
         data = fetch_billing_data_for_day(process_day=today, service_id=service_id, check_permissions=True)
         for d in data:
             update_fact_billing(data=d, process_day=today)
 
-    email_and_letters = db.session.query(
-        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
-        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.notifications_sent).label("billable_units"),
-        FactBilling.rate.label('rate'),
-        FactBilling.notification_type.label('notification_type'),
-        FactBilling.postage
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date,
-        FactBilling.notification_type.in_([EMAIL_TYPE, LETTER_TYPE])
-    ).group_by(
-        'month',
-        FactBilling.rate,
-        FactBilling.notification_type,
-        FactBilling.postage
-    )
-
-    sms = db.session.query(
-        func.date_trunc('month', FactBilling.bst_date).cast(Date).label("month"),
-        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
-        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label("billable_units"),
-        FactBilling.rate,
-        FactBilling.notification_type,
-        FactBilling.postage
-    ).filter(
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= year_end_date,
-        FactBilling.notification_type == SMS_TYPE
-    ).group_by(
-        'month',
-        FactBilling.rate,
-        FactBilling.notification_type,
-        FactBilling.postage
-    )
-
-    yearly_data = email_and_letters.union_all(sms).order_by(
-        'month',
-        'notification_type',
-        'rate'
+    return db.session.query(
+        union(*[
+            db.session.query(
+                func.date_trunc('month', query.c.bst_date).cast(Date).label("month"),
+                func.sum(query.c.notifications_sent).label("notifications_sent"),
+                # TEMPORARY: while we switch to "chargeable units"
+                func.sum(query.c.billable_units).label("billable_units"),
+                func.sum(query.c.chargeable_units).label("chargeable_units"),
+                query.c.rate.label("rate"),
+                query.c.postage.label("postage"),
+                query.c.notification_type.label("notification_type"),
+                func.sum(query.c.cost).label("cost"),
+                func.sum(query.c.free_chargeable_units).label("free_chargeable_units"),
+                func.sum(query.c.charged_units).label("charged_units"),
+            ).group_by(
+                query.c.rate,
+                query.c.notification_type,
+                query.c.postage,
+                'month',
+            )
+            for query in [
+                query_service_sms_usage_for_year(service_id, year).subquery(),
+                query_service_email_usage_for_year(service_id, year).subquery(),
+                query_service_letter_usage_for_year(service_id, year).subquery(),
+            ]
+        ]).subquery()
+    ).order_by(
+        "month",
+        "notification_type",
+        "rate",
     ).all()
 
-    return yearly_data
+
+def query_service_email_usage_for_year(service_id, year):
+    year_start, year_end = get_financial_year_dates(year)
+
+    return db.session.query(
+        FactBilling.bst_date,
+        FactBilling.postage,  # should always be "none"
+        FactBilling.notifications_sent,
+        # TEMPORARY: while we switch to "chargeable units"
+        FactBilling.notifications_sent.label("billable_units"),
+        FactBilling.billable_units.label("chargeable_units"),
+        FactBilling.rate,
+        FactBilling.notification_type,
+        FactBilling.notifications_sent.label("charged_units"),
+        literal(0).label("free_chargeable_units"),
+        literal(0).label("cost"),
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start,
+        FactBilling.bst_date <= year_end,
+        FactBilling.notification_type == EMAIL_TYPE
+    )
+
+
+def query_service_letter_usage_for_year(service_id, year):
+    year_start, year_end = get_financial_year_dates(year)
+
+    return db.session.query(
+        FactBilling.bst_date,
+        FactBilling.postage,
+        FactBilling.notifications_sent,
+        # TEMPORARY: while we switch to "chargeable units"
+        FactBilling.notifications_sent.label("billable_units"),
+        FactBilling.billable_units.label("chargeable_units"),
+        FactBilling.rate,
+        FactBilling.notification_type,
+        FactBilling.notifications_sent.label("charged_units"),
+        literal(0).label("free_chargeable_units"),
+        (FactBilling.notifications_sent * FactBilling.rate).label("cost"),
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start,
+        FactBilling.bst_date <= year_end,
+        FactBilling.notification_type == LETTER_TYPE
+    )
+
+
+def query_service_sms_usage_for_year(service_id, year):
+    year_start, year_end = get_financial_year_dates(year)
+    chargeable_units = FactBilling.billable_units * FactBilling.rate_multiplier
+
+    # Subquery for the number of chargeable units in all rows preceding this one,
+    # which might be none if this is the first row (hence the "coalesce").
+    cumulative_chargeable_units = func.coalesce(
+        func.sum(chargeable_units).over(
+            order_by=[
+                FactBilling.bst_date,  # order is "ASC" by default
+                FactBilling.rate  # ensures test stability for rows on the same day
+            ],
+            rows=(None, -1)  # ROWS BETWEEN UNBOUNDED PRECEDING AND 1 ROW PRECEDING
+        ),
+        literal(0)
+    )
+
+    # Subquery for how much free allowance we have left before the current row,
+    # so we can work out the cost for this row after taking it into account.
+    cumulative_free_remainder = func.greatest(
+        AnnualBilling.free_sms_fragment_limit - cumulative_chargeable_units,
+        0
+    )
+
+    charged_units = func.greatest(
+        chargeable_units - cumulative_free_remainder,
+        literal(0)
+    ).cast(Integer)  # for some reason the result is a String!
+
+    free_chargeable_units = func.least(
+        cumulative_free_remainder,
+        chargeable_units,
+    ).cast(Integer)  # for some reason the result is a Decimal
+
+    return db.session.query(
+        FactBilling.bst_date,
+        FactBilling.postage,  # should always be "none"
+        FactBilling.notifications_sent,
+        # TEMPORARY: while we switch to "chargeable units"
+        chargeable_units.label("billable_units"),
+        chargeable_units.label("chargeable_units"),
+        FactBilling.rate,
+        FactBilling.notification_type,
+        charged_units.label("charged_units"),
+        free_chargeable_units.label("free_chargeable_units"),
+        (charged_units * FactBilling.rate).label("cost")
+    ).outerjoin(
+        AnnualBilling,
+        AnnualBilling.service_id == service_id
+    ).filter(
+        FactBilling.service_id == service_id,
+        FactBilling.bst_date >= year_start,
+        FactBilling.bst_date <= year_end,
+        FactBilling.notification_type == SMS_TYPE,
+        AnnualBilling.financial_year_start == year,
+    )
 
 
 def delete_billing_data_for_service_for_day(process_day, service_id):
@@ -649,15 +717,12 @@ def fetch_sms_billing_for_organisation(organisation_id, start_date, end_date):
 
 
 def fetch_usage_year_for_organisation(organisation_id, year):
-    year_start_datetime, year_end_datetime = get_financial_year(year)
-
-    year_start_date = convert_utc_to_bst(year_start_datetime).date()
-    year_end_date = convert_utc_to_bst(year_end_datetime).date()
-
+    year_start, year_end = get_financial_year_dates(year)
     today = convert_utc_to_bst(datetime.utcnow()).date()
     services = dao_get_organisation_live_services(organisation_id)
+
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end_date >= today:
+    if year_end >= today:
         for service in services:
             data = fetch_billing_data_for_day(process_day=today, service_id=service.id)
             for d in data:
@@ -677,9 +742,9 @@ def fetch_usage_year_for_organisation(organisation_id, year):
             'emails_sent': 0,
             'active': service.active
         }
-    sms_usages = fetch_sms_billing_for_organisation(organisation_id, year_start_date, year_end_date)
-    letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start_date, year_end_date)
-    email_usages = fetch_email_usage_for_organisation(organisation_id, year_start_date, year_end_date)
+    sms_usages = fetch_sms_billing_for_organisation(organisation_id, year_start, year_end)
+    letter_usages = fetch_letter_costs_for_organisation(organisation_id, year_start, year_end)
+    email_usages = fetch_email_usage_for_organisation(organisation_id, year_start, year_end)
     for usage in sms_usages:
         service_with_usage[str(usage.service_id)] = {
             'service_id': usage.service_id,
@@ -779,6 +844,31 @@ def fetch_daily_volumes_for_platform(start_date, end_date):
     return aggregated_totals
 
 
+def fetch_daily_sms_provider_volumes_for_platform(start_date, end_date):
+    # query to return the total notifications sent per day for each channel. NB start and end dates are inclusive
+
+    daily_volume_stats = db.session.query(
+        FactBilling.bst_date,
+        FactBilling.provider,
+        func.sum(FactBilling.notifications_sent).label('sms_totals'),
+        func.sum(FactBilling.billable_units).label('sms_fragment_totals'),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier).label('sms_chargeable_units'),
+        func.sum(FactBilling.billable_units * FactBilling.rate_multiplier * FactBilling.rate).label('sms_cost'),
+    ).filter(
+        FactBilling.notification_type == SMS_TYPE,
+        FactBilling.bst_date >= start_date,
+        FactBilling.bst_date <= end_date,
+    ).group_by(
+        FactBilling.bst_date,
+        FactBilling.provider,
+    ).order_by(
+        FactBilling.bst_date,
+        FactBilling.provider,
+    ).all()
+
+    return daily_volume_stats
+
+
 def fetch_volumes_by_service(start_date, end_date):
     # query to return the volume totals by service aggregated for the date range given
     # start and end dates are inclusive.

app/email_branding/email_branding_schema.py

@@ -1,7 +1,7 @@
 from app.models import BRANDING_TYPES
 
 post_create_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_email_branding_schema = {
 }
 
 post_update_email_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting email_branding",
     "type": "object",
     "properties": {

app/inbound_sms/inbound_sms_schemas.py

@@ -1,5 +1,5 @@
 get_inbound_sms_for_service_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for parameters allowed when searching for to field=",
     "type": "object",
     "properties": {

app/letter_branding/letter_branding_schema.py

@@ -1,10 +1,10 @@
 post_letter_branding_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for creating or updating a letter brand",
     "type": "object",
     "properties": {
         "name": {"type": ["string", "null"]},
         "filename": {"type": ["string", "null"]},
     },
-    "required": ("name", "filename")
+    "required": ["name", "filename"]
 }

app/letters/letter_schemas.py

@@ -1,5 +1,5 @@
 letter_references = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "list of letter notification references",
     "type": "object",
     "title": "references",

app/notifications/notifications_letter_callback.py

@@ -17,7 +17,7 @@ register_errors(letter_callback_blueprint)
 
 
 dvla_sns_callback_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "sns callback received on s3 update",
     "type": "object",
     "title": "dvla internal sns callback",

app/organisation/organisation_schema.py

@@ -2,7 +2,7 @@ from app.models import INVITED_USER_STATUS_TYPES, ORGANISATION_TYPES
 from app.schema_validation.definitions import uuid
 
 post_create_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST organisation schema",
     "type": "object",
     "properties": {
@@ -15,7 +15,7 @@ post_create_organisation_schema = {
 }
 
 post_update_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST organisation schema",
     "type": "object",
     "properties": {
@@ -28,7 +28,7 @@ post_update_organisation_schema = {
 }
 
 post_link_service_to_organisation_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST link service to organisation schema",
     "type": "object",
     "properties": {
@@ -39,7 +39,7 @@ post_link_service_to_organisation_schema = {
 
 
 post_create_invited_org_user_status_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST create organisation invite schema",
     "type": "object",
     "properties": {
@@ -52,7 +52,7 @@ post_create_invited_org_user_status_schema = {
 
 
 post_update_invited_org_user_status_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST update organisation invite schema",
     "type": "object",
     "properties": {

app/organisation/rest.py

@@ -22,7 +22,7 @@ from app.dao.services_dao import dao_fetch_service_by_id
 from app.dao.templates_dao import dao_get_template_by_id
 from app.dao.users_dao import get_user_by_id
 from app.errors import InvalidRequest, register_errors
-from app.models import KEY_TYPE_NORMAL, Organisation
+from app.models import KEY_TYPE_NORMAL, NHS_ORGANISATION_TYPES, Organisation
 from app.notifications.process_notifications import (
     persist_notification,
     send_notification_to_queue,
@@ -93,6 +93,9 @@ def create_organisation():
 
     validate(data, post_create_organisation_schema)
 
+    if data["organisation_type"] in NHS_ORGANISATION_TYPES:
+        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
+
     organisation = Organisation(**data)
     dao_create_organisation(organisation)
     return jsonify(organisation.serialize()), 201
@@ -102,6 +105,12 @@ def create_organisation():
 def update_organisation(organisation_id):
     data = request.get_json()
     validate(data, post_update_organisation_schema)
+
+    organisation = dao_get_organisation_by_id(organisation_id)
+
+    if data.get('organisation_type') in NHS_ORGANISATION_TYPES and not organisation.email_branding_id:
+        data["email_branding_id"] = current_app.config['NHS_EMAIL_BRANDING_ID']
+
     result = dao_update_organisation(organisation_id, **data)
 
     if data.get('agreement_signed') is True:

app/performance_dashboard/performance_dashboard_schema.py

@@ -1,5 +1,5 @@
 performance_dashboard_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Performance dashboard request schema",
     "type": "object",
     "title": "Performance dashboard request",

app/platform_stats/platform_stats_schema.py

@@ -1,5 +1,5 @@
 platform_stats_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "platform stats request schema",
     "type": "object",
     "title": "Platform stats request",

app/platform_stats/rest.py

@@ -5,6 +5,7 @@ from flask import Blueprint, jsonify, request
 from app.dao.date_util import get_financial_year_for_datetime
 from app.dao.fact_billing_dao import (
     fetch_billing_details_for_all_services,
+    fetch_daily_sms_provider_volumes_for_platform,
     fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
@@ -161,6 +162,27 @@ def daily_volumes_report():
     return jsonify(report)
 
 
+@platform_stats_blueprint.route('daily-sms-provider-volumes-report')
+def daily_sms_provider_volumes_report():
+    start_date = validate_date_format(request.args.get('start_date'))
+    end_date = validate_date_format(request.args.get('end_date'))
+
+    daily_volumes = fetch_daily_sms_provider_volumes_for_platform(start_date, end_date)
+    report = []
+
+    for row in daily_volumes:
+        report.append({
+            'day': row.bst_date.isoformat(),
+            'provider': row.provider,
+            'sms_totals': int(row.sms_totals),
+            'sms_fragment_totals': int(row.sms_fragment_totals),
+            'sms_chargeable_units': int(row.sms_chargeable_units),
+            # convert from Decimal to float as it's not json serialisable
+            'sms_cost': float(row.sms_cost),
+        })
+    return jsonify(report)
+
+
 @platform_stats_blueprint.route('volumes-by-service')
 def volumes_by_service_report():
     start_date = validate_date_format(request.args.get('start_date'))

app/service/send_notification.py

@@ -7,7 +7,10 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app import create_random_identifier
 from app.config import QueueNames
-from app.dao.notifications_dao import _update_notification_status
+from app.dao.notifications_dao import (
+    _update_notification_status,
+    get_notification_by_id,
+)
 from app.dao.service_email_reply_to_dao import dao_get_reply_to_by_id
 from app.dao.service_sms_sender_dao import dao_get_service_sms_senders_by_id
 from app.dao.services_dao import dao_fetch_service_by_id
@@ -166,15 +169,20 @@ def send_pdf_letter_notification(service_id, post_data):
         allow_guest_list_recipients=False,
     )
 
+    # notification already exists e.g. if the user clicked send in different tabs
+    if get_notification_by_id(post_data['file_id']):
+        return {'id': str(post_data['file_id'])}
+
     template = get_precompiled_letter_template(service.id)
     file_location = 'service-{}/{}.pdf'.format(service.id, post_data['file_id'])
 
     try:
         letter = utils_s3download(current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
     except S3ObjectNotFound as e:
-        current_app.logger.exception('Letter {}.pdf not in transient {} bucket'.format(
+        current_app.logger.warning('Letter {}.pdf not in transient {} bucket'.format(
             post_data['file_id'], current_app.config['TRANSIENT_UPLOADED_LETTERS'])
         )
+
         raise e
 
     # Getting the page count won't raise an error since admin has already checked the PDF is valid

app/service/send_pdf_letter_schema.py

@@ -1,5 +1,5 @@
 send_pdf_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST send uploaded pdf letter",
     "type": "object",
     "title": "Send an uploaded pdf letter",

app/service/service_broadcast_settings_schema.py

@@ -1,5 +1,5 @@
 service_broadcast_settings_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Set a services broadcast settings",
     "type": "object",
     "title": "Set a services broadcast settings",

app/service/service_callback_api_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import https_url, uuid
 
 create_service_callback_api_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service callback/inbound api schema",
     "type": "object",
     "title": "Create service callback/inbound api",
@@ -14,7 +14,7 @@ create_service_callback_api_schema = {
 }
 
 update_service_callback_api_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service callback/inbound api schema",
     "type": "object",
     "title": "Create service callback/inbound api",

app/service/service_contact_list_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 create_service_contact_list_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST create service contact list schema",
     "type": "object",
     "title": "Create service contact list",

app/service/service_data_retention_schema.py

@@ -1,5 +1,5 @@
 add_service_data_retention_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service data retention schema",
     "title": "Add service data retention for notification type api",
     "type": "object",
@@ -12,7 +12,7 @@ add_service_data_retention_request = {
 
 
 update_service_data_retention_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service data retention schema",
     "title": "Update service data retention for notification type api",
     "type": "object",

app/service/service_senders_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 add_service_email_reply_to_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service email reply to address",
     "type": "object",
     "title": "Add new email reply to address for service",
@@ -14,7 +14,7 @@ add_service_email_reply_to_request = {
 
 
 add_service_letter_contact_block_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST service letter contact block",
     "type": "object",
     "title": "Add new letter contact block for service",
@@ -27,7 +27,7 @@ add_service_letter_contact_block_request = {
 
 
 add_service_sms_sender_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST add service SMS sender",
     "type": "object",
     "title": "Add new SMS sender for service",

app/template_folder/template_folder_schema.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import nullable_uuid, uuid
 
 post_create_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for getting template_folder",
     "type": "object",
     "properties": {
@@ -12,7 +12,7 @@ post_create_template_folder_schema = {
 }
 
 post_update_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for updating template_folder",
     "type": "object",
     "properties": {
@@ -23,7 +23,7 @@ post_update_template_folder_schema = {
 }
 
 post_move_template_folder_schema = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST schema for renaming template_folder",
     "type": "object",
     "properties": {

app/user/users_schema.py

@@ -1,5 +1,5 @@
 post_verify_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for verifying a 2fa code',
     'type': 'object',
     'properties': {
@@ -12,7 +12,7 @@ post_verify_code_schema = {
 
 
 post_verify_webauthn_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for verifying a webauthn login attempt',
     'type': 'object',
     'properties': {
@@ -24,7 +24,7 @@ post_verify_webauthn_schema = {
 
 
 post_send_user_email_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': (
         'POST schema for generating a 2fa email - "to" is required for legacy purposes. '
         '"next" is an optional url to redirect to on sign in'
@@ -43,7 +43,7 @@ post_send_user_email_code_schema = {
 
 
 post_send_user_sms_code_schema = {
-    '$schema': 'http://json-schema.org/draft-04/schema#',
+    '$schema': 'http://json-schema.org/draft-07/schema#',
     'description': 'POST schema for generating a 2fa sms',
     'type': 'object',
     'properties': {

app/v2/inbound_sms/inbound_sms_schemas.py

@@ -1,7 +1,7 @@
 from app.schema_validation.definitions import uuid
 
 get_inbound_sms_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for query parameters allowed when getting list of received text messages",
     "type": "object",
     "properties": {
@@ -12,7 +12,7 @@ get_inbound_sms_request = {
 
 
 get_inbound_sms_single_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET inbound sms schema response",
     "type": "object",
     "title": "GET response v2/inbound_sms",
@@ -36,7 +36,7 @@ get_inbound_sms_single_response = {
 }
 
 get_inbound_sms_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET list of inbound sms response schema",
     "type": "object",
     "properties": {

app/v2/notifications/notification_schemas.py

@@ -7,7 +7,7 @@ from app.models import (
 from app.schema_validation.definitions import personalisation, uuid
 
 template = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "template schema",
     "type": "object",
     "title": "notification content",
@@ -20,7 +20,7 @@ template = {
 }
 
 notification_by_id = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET notification response schema",
     "type": "object",
     "title": "response v2/notification",
@@ -32,7 +32,7 @@ notification_by_id = {
 
 
 get_notification_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET notification response schema",
     "type": "object",
     "title": "response v2/notification",
@@ -67,7 +67,7 @@ get_notification_response = {
 }
 
 get_notifications_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for query parameters allowed when getting list of notifications",
     "type": "object",
     "properties": {
@@ -92,7 +92,7 @@ get_notifications_request = {
 }
 
 get_notifications_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET list of notifications response schema",
     "type": "object",
     "properties": {
@@ -126,7 +126,7 @@ get_notifications_response = {
 }
 
 post_sms_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification schema",
     "type": "object",
     "title": "POST v2/notifications/sms",
@@ -143,7 +143,7 @@ post_sms_request = {
 }
 
 sms_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "content schema for SMS notification response schema",
     "type": "object",
     "title": "notification content",
@@ -155,7 +155,7 @@ sms_content = {
 }
 
 post_sms_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification response schema",
     "type": "object",
     "title": "response v2/notifications/sms",
@@ -172,7 +172,7 @@ post_sms_response = {
 
 
 post_email_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST email notification schema",
     "type": "object",
     "title": "POST v2/notifications/email",
@@ -189,7 +189,7 @@ post_email_request = {
 }
 
 email_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Email content for POST email notification",
     "type": "object",
     "title": "notification email content",
@@ -202,7 +202,7 @@ email_content = {
 }
 
 post_email_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST email notification response schema",
     "type": "object",
     "title": "response v2/notifications/email",
@@ -218,7 +218,7 @@ post_email_response = {
 }
 
 post_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST letter notification schema",
     "type": "object",
     "title": "POST v2/notifications/letter",
@@ -232,7 +232,7 @@ post_letter_request = {
 }
 
 post_precompiled_letter_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST precompiled letter notification schema",
     "type": "object",
     "title": "POST v2/notifications/letter",
@@ -246,7 +246,7 @@ post_precompiled_letter_request = {
 }
 
 letter_content = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "Letter content for POST letter notification",
     "type": "object",
     "title": "notification letter content",
@@ -258,7 +258,7 @@ letter_content = {
 }
 
 post_letter_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST sms notification response schema",
     "type": "object",
     "title": "response v2/notifications/letter",

app/v2/template/template_schemas.py

@@ -2,7 +2,7 @@ from app.models import TEMPLATE_TYPES
 from app.schema_validation.definitions import personalisation, uuid
 
 get_template_by_id_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "schema for parameters allowed when getting template by id",
     "type": "object",
     "properties": {
@@ -14,7 +14,7 @@ get_template_by_id_request = {
 }
 
 get_template_by_id_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET template by id schema response",
     "type": "object",
     "title": "reponse v2/template",
@@ -42,7 +42,7 @@ get_template_by_id_response = {
 }
 
 post_template_preview_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST template schema",
     "type": "object",
     "title": "POST v2/template/{id}/preview",
@@ -54,7 +54,7 @@ post_template_preview_request = {
 }
 
 post_template_preview_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "POST template preview schema response",
     "type": "object",
     "title": "reponse v2/template/{id}/preview",

app/v2/templates/templates_schemas.py

@@ -4,7 +4,7 @@ from app.v2.template.template_schemas import (
 )
 
 get_all_template_request = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "request schema for parameters allowed when getting all templates",
     "type": "object",
     "properties": {
@@ -14,7 +14,7 @@ get_all_template_request = {
 }
 
 get_all_template_response = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$schema": "http://json-schema.org/draft-07/schema#",
     "description": "GET response schema when getting all templates",
     "type": "object",
     "properties": {

gunicorn_config.py

@@ -2,6 +2,8 @@ import os
 import sys
 import traceback
 import gunicorn
+import eventlet
+import socket
 
 from gds_metrics.gunicorn import child_exit  # noqa
 
@@ -30,3 +32,22 @@ def on_exit(server):
 
 def worker_int(worker):
     worker.log.info("worker: received SIGINT {}".format(worker.pid))
+
+
+def fix_ssl_monkeypatching():
+    """
+    eventlet works by monkey-patching core IO libraries (such as ssl) to be non-blocking. However, there's currently
+    a bug: In the normal socket library it may throw a timeout error as a `socket.timeout` exception. However
+    eventlet.green.ssl's patch raises an ssl.SSLError('timed out',) instead. redispy handles socket.timeout but not
+    ssl.SSLError, so we solve this by monkey patching the monkey patching code to raise the correct exception type
+    :scream:
+
+    https://github.com/eventlet/eventlet/issues/692
+    """
+    # this has probably already been called somewhere in gunicorn internals, however, to be sure, we invoke it again.
+    # eventlet.monkey_patch can be called multiple times without issue
+    eventlet.monkey_patch()
+    eventlet.green.ssl.timeout_exc = socket.timeout
+
+
+fix_ssl_monkeypatching()

manifest.yml.j2

@@ -7,9 +7,9 @@
       'STATSD_HOST': None
     },
     'routes': {
-      'preview': ['api.notify.works'],
-      'staging': ['api.staging-notify.works'],
-      'production': ['api.notifications.service.gov.uk'],
+      'preview': ['api.notify.works', 'notify-api-preview.apps.internal'],
+      'staging': ['api.staging-notify.works', 'notify-api-staging.apps.internal'],
+      'production': ['api.notifications.service.gov.uk', 'notify-api-production.apps.internal'],
     },
     'health-check-type': 'port',
     'health-check-invocation-timeout': 3,
@@ -27,9 +27,9 @@
       'STATSD_HOST': None
     },
     'routes': {
-      'preview': ['api.notify.works/notifications/sms/mmg', 'api.notify.works/notifications/sms/firetext', 'api.notify.works/notifications/sms/reach'],
-      'staging': ['api.staging-notify.works/notifications/sms/mmg', 'api.staging-notify.works/notifications/sms/firetext', 'api.staging-notify.works/notifications/sms/reach'],
-      'production': ['api.notifications.service.gov.uk/notifications/sms/mmg', 'api.notifications.service.gov.uk/notifications/sms/firetext', 'api.notifications.service.gov.uk/notifications/sms/reach'],
+      'preview': ['api.notify.works/notifications/sms/mmg', 'api.notify.works/notifications/sms/firetext', 'api.notify.works/notifications/sms/reach','notify-api-sms-receipts-preview.apps.internal'],
+      'staging': ['api.staging-notify.works/notifications/sms/mmg', 'api.staging-notify.works/notifications/sms/firetext', 'api.staging-notify.works/notifications/sms/reach', 'notify-api-sms-receipts-staging.apps.internal'],
+      'production': ['api.notifications.service.gov.uk/notifications/sms/mmg', 'api.notifications.service.gov.uk/notifications/sms/firetext', 'api.notifications.service.gov.uk/notifications/sms/reach', 'notify-api-sms-receipts-production.apps.internal' ],
     },
     'health-check-type': 'port',
     'health-check-invocation-timeout': 3,
@@ -99,6 +99,7 @@ applications:
 
     services:
       - notify-db
+      - notify-redis
       - logit-ssl-syslog-drain
       {% if CF_APP == 'notify-api' %}
       - notify-prometheus
@@ -107,8 +108,10 @@ applications:
 
     env:
       NOTIFY_APP_NAME: {{ app.get('NOTIFY_APP_NAME', CF_APP.replace('notify-', '')) }}
+      NOTIFY_LOG_PATH: /home/vcap/logs/app.log
       SQLALCHEMY_POOL_SIZE: {{ app.get('sqlalchemy_pool_size', 1) }}
       FLASK_APP: application.py
+      NOTIFY_ENVIRONMENT: {{ environment }}
 
       # Credentials variables
       ADMIN_BASE_URL: '{{ ADMIN_BASE_URL }}'
@@ -119,6 +122,7 @@ applications:
       ROUTE_SECRET_KEY_1: '{{ ROUTE_SECRET_KEY_1 }}'
       ROUTE_SECRET_KEY_2: '{{ ROUTE_SECRET_KEY_2 }}'
       CRONITOR_KEYS: '{{ CRONITOR_KEYS | tojson }}'
+      METRICS_BASIC_AUTH_TOKEN: {{ METRICS_BASIC_AUTH_TOKEN }}
 
       HIGH_VOLUME_SERVICE: '{{ HIGH_VOLUME_SERVICE | tojson }}'
 
@@ -141,9 +145,6 @@ applications:
       FIRETEXT_INTERNATIONAL_API_KEY: '{{ FIRETEXT_INTERNATIONAL_API_KEY }}'
       FIRETEXT_INBOUND_SMS_AUTH: '{{ FIRETEXT_INBOUND_SMS_AUTH | tojson }}'
 
-      REDIS_ENABLED: '{{ REDIS_ENABLED }}'
-      REDIS_URL: '{{ REDIS_URL }}'
-
       TEMPLATE_PREVIEW_API_HOST: '{{ TEMPLATE_PREVIEW_API_HOST }}'
       TEMPLATE_PREVIEW_API_KEY: '{{ TEMPLATE_PREVIEW_API_KEY }}'
 

migrations/versions/0368_move_orgs_to_nhs_branding_.py

@@ -0,0 +1,31 @@
+"""
+
+Revision ID: 0368_move_orgs_to_nhs_branding
+Revises: 0367_add_reach
+Create Date: 2022-04-12 18:22:12.069016
+
+"""
+from alembic import op
+
+revision = '0368_move_orgs_to_nhs_branding'
+down_revision = '0367_add_reach'
+
+
+def upgrade():
+    op.execute("""
+        UPDATE
+            organisation
+        SET
+            email_branding_id = 'a7dc4e56-660b-4db7-8cff-12c37b12b5ea'
+        WHERE
+            organisation_type IN ('nhs_central', 'nhs_local', 'nhs_gp')
+        AND
+            email_branding_id IS NULL
+    """)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###

migrations/versions/0369_update_sms_rates.py

@@ -0,0 +1,24 @@
+"""
+
+Revision ID: 0369_update_sms_rates
+Revises: 0368_move_orgs_to_nhs_branding
+Create Date: 2022-04-26 09:39:45.260951
+
+"""
+import uuid
+
+from alembic import op
+
+revision = '0369_update_sms_rates'
+down_revision = '0368_move_orgs_to_nhs_branding'
+
+
+def upgrade():
+    op.execute(
+        "INSERT INTO rates(id, valid_from, rate, notification_type) "
+        f"VALUES('{uuid.uuid4()}', '2022-04-30 23:00:00', 0.0172, 'sms')"
+    )
+
+
+def downgrade():
+    pass

pytest.ini

@@ -5,5 +5,6 @@ env =
     MMG_API_KEY=mmg-secret-key
     FIRETEXT_API_KEY=Firetext
     NOTIFICATION_QUEUE_PREFIX=testing
+    REDIS_ENABLED=0
 addopts = -p no:warnings
 xfail_strict = true

requirements.in

@@ -2,27 +2,25 @@
 # with package version changes made in requirements.in
 
 cffi==1.15.0
-celery[sqs]==5.2.3
-Flask-Bcrypt==0.7.1
+celery[sqs]==5.2.6
+Flask-Bcrypt==1.0.1
 flask-marshmallow==0.14.0
 Flask-Migrate==3.1.0
 git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732dd1b975a56ab06a46bd1a20a21e682262#egg=Flask-SQLAlchemy==2.3.2.dev20190108
-Flask==2.1.0
+Flask==2.1.1
 click-datetime==0.2
 # Should be pinned until a new gunicorn release greater than 20.1.0 comes out. (Due to eventlet v0.33 compatibility issues)
 git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64#egg=gunicorn[eventlet]==20.1.0
 iso8601==1.0.2
 itsdangerous==2.1.2
-jsonschema==3.2.0
+jsonschema[format]==4.4.0
 marshmallow-sqlalchemy==0.23.1 # pyup: <0.24.0 # marshmallow v3 throws errors
 marshmallow==2.21.0 # pyup: <3 # v3 throws errors
 psycopg2-binary==2.9.3
 PyJWT==2.3.0
-SQLAlchemy==1.4.32
-strict-rfc3339==0.7
-rfc3987==1.3.8
-cachetools==4.2.1
-beautifulsoup4==4.10.0
+SQLAlchemy==1.4.35
+cachetools==5.0.0
+beautifulsoup4==4.11.1
 lxml==4.8.0
 Werkzeug==2.0.3
 
@@ -31,8 +29,8 @@ notifications-python-client==6.3.0
 # PaaS
 awscli-cwlogs==1.4.6
 
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.4
 
 # gds-metrics requires prometheseus 0.2.0, override that requirement as 0.7.1 brings significant performance gains
-prometheus-client==0.10.1
-gds-metrics==0.2.4
+prometheus-client==0.14.1
+git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72

requirements.txt

@@ -8,17 +8,17 @@ alembic==1.7.4
     # via flask-migrate
 amqp==5.0.9
     # via kombu
+arrow==1.2.2
+    # via isoduration
 attrs==21.2.0
     # via jsonschema
 awscli==1.21.4
-    # via
-    #   awscli-cwlogs
-    #   notifications-utils
+    # via awscli-cwlogs
 awscli-cwlogs==1.4.6
     # via -r requirements.in
 bcrypt==3.2.0
     # via flask-bcrypt
-beautifulsoup4==4.10.0
+beautifulsoup4==4.11.1
     # via -r requirements.in
 billiard==3.6.4.0
     # via celery
@@ -33,11 +33,11 @@ botocore==1.22.4
     #   awscli
     #   boto3
     #   s3transfer
-cachetools==4.2.1
+cachetools==5.0.0
     # via
     #   -r requirements.in
     #   notifications-utils
-celery[sqs]==5.2.3
+celery[sqs]==5.2.6
     # via -r requirements.in
 certifi==2021.10.8
     # via
@@ -75,7 +75,7 @@ docutils==0.15.2
     # via awscli
 eventlet==0.33.0
     # via gunicorn
-flask==2.1.0
+flask==2.1.1
     # via
     #   -r requirements.in
     #   flask-bcrypt
@@ -84,7 +84,7 @@ flask==2.1.0
     #   flask-redis
     #   gds-metrics
     #   notifications-utils
-flask-bcrypt==0.7.1
+flask-bcrypt==1.0.1
     # via -r requirements.in
 flask-marshmallow==0.14.0
     # via -r requirements.in
@@ -96,22 +96,30 @@ flask-sqlalchemy @ git+https://github.com/mitsuhiko/flask-sqlalchemy.git@500e732
     # via
     #   -r requirements.in
     #   flask-migrate
-gds-metrics==0.2.4
+fqdn==1.5.1
+    # via jsonschema
+gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72
     # via -r requirements.in
 geojson==2.5.0
     # via notifications-utils
 govuk-bank-holidays==0.10
     # via notifications-utils
 greenlet==1.1.2
-    # via eventlet
+    # via
+    #   eventlet
+    #   sqlalchemy
 gunicorn @ git+https://github.com/benoitc/gunicorn.git@1299ea9e967a61ae2edebe191082fd169b864c64
     # via -r requirements.in
 idna==3.3
-    # via requests
+    # via
+    #   jsonschema
+    #   requests
 importlib-metadata==4.11.3
     # via flask
 iso8601==1.0.2
     # via -r requirements.in
+isoduration==20.11.0
+    # via jsonschema
 itsdangerous==2.1.2
     # via
     #   -r requirements.in
@@ -125,7 +133,9 @@ jmespath==0.10.0
     # via
     #   boto3
     #   botocore
-jsonschema==3.2.0
+jsonpointer==2.3
+    # via jsonschema
+jsonschema[format]==4.4.0
     # via -r requirements.in
 kombu==5.2.3
     # via celery
@@ -148,7 +158,7 @@ mistune==0.8.4
     # via notifications-utils
 notifications-python-client==6.3.0
     # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@53.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@55.1.4
     # via -r requirements.in
 orderedset==2.0.3
     # via notifications-utils
@@ -156,7 +166,7 @@ packaging==21.0
     # via bleach
 phonenumbers==8.12.36
     # via notifications-utils
-prometheus-client==0.10.1
+prometheus-client==0.14.1
     # via
     #   -r requirements.in
     #   gds-metrics
@@ -182,6 +192,7 @@ pyrsistent==0.18.0
     # via jsonschema
 python-dateutil==2.8.2
     # via
+    #   arrow
     #   awscli-cwlogs
     #   botocore
 python-json-logger==2.0.2
@@ -202,8 +213,10 @@ requests==2.26.0
     #   govuk-bank-holidays
     #   notifications-python-client
     #   notifications-utils
+rfc3339-validator==0.1.4
+    # via jsonschema
 rfc3987==1.3.8
-    # via -r requirements.in
+    # via jsonschema
 rsa==4.7.2
     # via awscli
 s3transfer==0.5.0
@@ -220,21 +233,21 @@ six==1.16.0
     #   click-repl
     #   eventlet
     #   flask-marshmallow
-    #   jsonschema
     #   python-dateutil
+    #   rfc3339-validator
 smartypants==2.0.1
     # via notifications-utils
 soupsieve==2.2.1
     # via beautifulsoup4
-sqlalchemy==1.4.32
+sqlalchemy==1.4.35
     # via
     #   -r requirements.in
     #   alembic
     #   marshmallow-sqlalchemy
 statsd==3.3.0
     # via notifications-utils
-strict-rfc3339==0.7
-    # via -r requirements.in
+uri-template==1.2.0
+    # via jsonschema
 urllib3==1.26.7
     # via
     #   botocore
@@ -246,6 +259,8 @@ vine==5.0.0
     #   kombu
 wcwidth==0.2.5
     # via prompt-toolkit
+webcolors==1.11.1
+    # via jsonschema
 webencodings==0.5.1
     # via bleach
 werkzeug==2.0.3

requirements_for_test.txt

@@ -1,14 +1,14 @@
 -r requirements.txt
 flake8==4.0.1
-flake8-bugbear==22.1.11
+flake8-bugbear==22.3.23
 isort==5.10.1
-moto==3.0.7
-pytest==7.0.1
+moto==3.1.4
+pytest==7.1.1
 pytest-env==0.6.2
 pytest-mock==3.7.0
 pytest-cov==3.0.0
 pytest-xdist==2.5.0
-freezegun==1.2.0
+freezegun==1.2.1
 requests-mock==1.9.3
 # used for creating manifest file locally
-jinja2-cli[yaml]==0.8.1
+jinja2-cli[yaml]==0.8.2

tests/app/billing/test_billing.py

@@ -1,411 +0,0 @@
-import json
-from calendar import monthrange
-from datetime import datetime, timedelta
-
-import pytest
-from freezegun import freeze_time
-
-from app.billing.rest import update_free_sms_fragment_limit_data
-from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year
-from app.dao.date_util import (
-    get_current_financial_year_start_year,
-    get_month_start_and_end_date_in_utc,
-)
-from app.models import FactBilling
-from tests import create_admin_authorization_header
-from tests.app.db import (
-    create_annual_billing,
-    create_ft_billing,
-    create_notification,
-    create_rate,
-    create_service,
-    create_template,
-)
-
-APR_2016_MONTH_START = datetime(2016, 3, 31, 23, 00, 00)
-APR_2016_MONTH_END = datetime(2016, 4, 30, 22, 59, 59, 99999)
-
-IN_MAY_2016 = datetime(2016, 5, 10, 23, 00, 00)
-IN_JUN_2016 = datetime(2016, 6, 3, 23, 00, 00)
-
-
-def test_create_update_free_sms_fragment_limit_invalid_schema(client, sample_service):
-
-    response = client.post('service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),
-                           data={},
-                           headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-    json_resp = json.loads(response.get_data(as_text=True))
-
-    assert response.status_code == 400
-    assert 'JSON' in json_resp['message']
-
-
-def test_create_free_sms_fragment_limit_current_year_updates_future_years(admin_request, sample_service):
-    current_year = get_current_financial_year_start_year()
-    future_billing = create_annual_billing(sample_service.id, 1, current_year + 1)
-
-    admin_request.post(
-        'billing.create_or_update_free_sms_fragment_limit',
-        service_id=sample_service.id,
-        _data={'free_sms_fragment_limit': 9999},
-        _expected_status=201
-    )
-
-    current_billing = dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year)
-    assert future_billing.free_sms_fragment_limit == 9999
-    assert current_billing.financial_year_start == current_year
-    assert current_billing.free_sms_fragment_limit == 9999
-
-
-@pytest.mark.parametrize('update_existing', [True, False])
-def test_create_or_update_free_sms_fragment_limit_past_year_doenst_update_other_years(
-    admin_request,
-    sample_service,
-    update_existing
-):
-    current_year = get_current_financial_year_start_year()
-    create_annual_billing(sample_service.id, 1, current_year)
-    if update_existing:
-        create_annual_billing(sample_service.id, 1, current_year - 1)
-
-    data = {'financial_year_start': current_year - 1, 'free_sms_fragment_limit': 9999}
-    admin_request.post(
-        'billing.create_or_update_free_sms_fragment_limit',
-        service_id=sample_service.id,
-        _data=data,
-        _expected_status=201)
-
-    assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year - 1).free_sms_fragment_limit == 9999
-    assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year).free_sms_fragment_limit == 1
-
-
-def test_create_free_sms_fragment_limit_updates_existing_year(admin_request, sample_service):
-    current_year = get_current_financial_year_start_year()
-    annual_billing = create_annual_billing(sample_service.id, 1, current_year)
-
-    admin_request.post(
-        'billing.create_or_update_free_sms_fragment_limit',
-        service_id=sample_service.id,
-        _data={'financial_year_start': current_year, 'free_sms_fragment_limit': 2},
-        _expected_status=201)
-
-    assert annual_billing.free_sms_fragment_limit == 2
-
-
-@freeze_time('2021-04-02 13:00')
-def test_get_free_sms_fragment_limit(
-    client, sample_service
-):
-    create_annual_billing(service_id=sample_service.id, free_sms_fragment_limit=11000, financial_year_start=2021)
-    response_get = client.get(
-        'service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),
-        headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-
-    json_resp = json.loads(response_get.get_data(as_text=True))
-    assert response_get.status_code == 200
-    assert json_resp['financial_year_start'] == 2021
-    assert json_resp['free_sms_fragment_limit'] == 11000
-
-
-@freeze_time('2021-04-02 13:00')
-def test_get_free_sms_fragment_limit_current_year_creates_new_row_if_annual_billing_is_missing(
-    client, sample_service
-):
-    response_get = client.get(
-        'service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),
-        headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-
-    json_resp = json.loads(response_get.get_data(as_text=True))
-    assert response_get.status_code == 200
-    assert json_resp['financial_year_start'] == 2021
-    assert json_resp['free_sms_fragment_limit'] == 10000  # based on other organisation type
-
-
-def test_update_free_sms_fragment_limit_data(client, sample_service):
-    current_year = get_current_financial_year_start_year()
-    create_annual_billing(sample_service.id, free_sms_fragment_limit=250000, financial_year_start=current_year - 1)
-
-    update_free_sms_fragment_limit_data(sample_service.id, 9999, current_year)
-
-    annual_billing = dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year)
-    assert annual_billing.free_sms_fragment_limit == 9999
-
-
-@freeze_time('2018-04-21 14:00')
-def test_get_yearly_usage_by_monthly_from_ft_billing_populates_deltas(client, notify_db_session):
-    service = create_service()
-    sms_template = create_template(service=service, template_type="sms")
-    create_rate(start_date=datetime.utcnow() - timedelta(days=1), value=0.158, notification_type='sms')
-
-    create_notification(template=sms_template, status='delivered')
-
-    assert FactBilling.query.count() == 0
-
-    response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id),
-                          headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-
-    assert response.status_code == 200
-    assert len(json.loads(response.get_data(as_text=True))) == 1
-    fact_billing = FactBilling.query.all()
-    assert len(fact_billing) == 1
-    assert fact_billing[0].notification_type == 'sms'
-
-
-def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
-    service = create_service()
-    sms_template = create_template(service=service, template_type="sms")
-    email_template = create_template(service=service, template_type="email")
-    letter_template = create_template(service=service, template_type="letter")
-    for month in range(1, 13):
-        mon = str(month).zfill(2)
-        for day in range(1, monthrange(2016, month)[1] + 1):
-            d = str(day).zfill(2)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=sms_template,
-                              billable_unit=1,
-                              rate=0.162)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=email_template,
-                              rate=0)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=letter_template,
-                              billable_unit=1,
-                              rate=0.33,
-                              postage='second')
-
-    response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
-                          headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-
-    json_resp = json.loads(response.get_data(as_text=True))
-    ft_letters = [x for x in json_resp if x['notification_type'] == 'letter']
-    ft_sms = [x for x in json_resp if x['notification_type'] == 'sms']
-    ft_email = [x for x in json_resp if x['notification_type'] == 'email']
-    keys = [x.keys() for x in ft_sms][0]
-    expected_sms_april = {"month": "April",
-                          "notification_type": "sms",
-                          "billing_units": 30,
-                          "rate": 0.162,
-                          "postage": "none"
-                          }
-    expected_letter_april = {"month": "April",
-                             "notification_type": "letter",
-                             "billing_units": 30,
-                             "rate": 0.33,
-                             "postage": "second"
-                             }
-
-    for k in keys:
-        assert ft_sms[0][k] == expected_sms_april[k]
-        assert ft_letters[0][k] == expected_letter_april[k]
-    assert len(ft_email) == 0
-
-
-def set_up_yearly_data():
-    service = create_service()
-    sms_template = create_template(service=service, template_type="sms")
-    email_template = create_template(service=service, template_type="email")
-    letter_template = create_template(service=service, template_type="letter")
-    for month in range(1, 13):
-        mon = str(month).zfill(2)
-        for day in range(1, monthrange(2016, month)[1] + 1):
-            d = str(day).zfill(2)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=sms_template,
-                              rate=0.0162)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=sms_template,
-                              rate_multiplier=2,
-                              rate=0.0162)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=email_template,
-                              billable_unit=0,
-                              rate=0)
-            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
-                              template=letter_template,
-                              rate=0.33,
-                              postage='second')
-        start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1))
-    return service
-
-
-def test_get_yearly_billing_usage_summary_from_ft_billing_returns_400_if_missing_year(client, sample_service):
-    response = client.get(
-        '/service/{}/billing/ft-yearly-usage-summary'.format(sample_service.id),
-        headers=[create_admin_authorization_header()]
-    )
-    assert response.status_code == 400
-    assert json.loads(response.get_data(as_text=True)) == {
-        'message': 'No valid year provided', 'result': 'error'
-    }
-
-
-def test_get_yearly_billing_usage_summary_from_ft_billing_returns_empty_list_if_no_billing_data(
-        client, sample_service
-):
-    response = client.get(
-        '/service/{}/billing/ft-yearly-usage-summary?year=2016'.format(sample_service.id),
-        headers=[create_admin_authorization_header()]
-    )
-    assert response.status_code == 200
-    assert json.loads(response.get_data(as_text=True)) == []
-
-
-def test_get_yearly_billing_usage_summary_from_ft_billing(client, notify_db_session):
-    service = set_up_yearly_data()
-
-    response = client.get('/service/{}/billing/ft-yearly-usage-summary?year=2016'.format(service.id),
-                          headers=[create_admin_authorization_header()]
-                          )
-    assert response.status_code == 200
-    json_response = json.loads(response.get_data(as_text=True))
-    assert len(json_response) == 3
-    assert json_response[0]['notification_type'] == 'email'
-    assert json_response[0]['billing_units'] == 275
-    assert json_response[0]['rate'] == 0
-    assert json_response[0]['letter_total'] == 0
-    assert json_response[1]['notification_type'] == 'letter'
-    assert json_response[1]['billing_units'] == 275
-    assert json_response[1]['rate'] == 0.33
-    assert json_response[1]['letter_total'] == 90.75
-    assert json_response[2]['notification_type'] == 'sms'
-    assert json_response[2]['billing_units'] == 825
-    assert json_response[2]['rate'] == 0.0162
-    assert json_response[2]['letter_total'] == 0
-
-
-def test_get_yearly_usage_by_monthly_from_ft_billing_all_cases(client, notify_db_session):
-    service = set_up_data_for_all_cases()
-    response = client.get('service/{}/billing/ft-monthly-usage?year=2018'.format(service.id),
-                          headers=[('Content-Type', 'application/json'), create_admin_authorization_header()])
-
-    assert response.status_code == 200
-    json_response = json.loads(response.get_data(as_text=True))
-    assert len(json_response) == 5
-    assert json_response[0]['month'] == 'May'
-    assert json_response[0]['notification_type'] == 'letter'
-    assert json_response[0]['rate'] == 0.33
-    assert json_response[0]['billing_units'] == 1
-    assert json_response[0]['postage'] == 'second'
-
-    assert json_response[1]['month'] == 'May'
-    assert json_response[1]['notification_type'] == 'letter'
-    assert json_response[1]['rate'] == 0.36
-    assert json_response[1]['billing_units'] == 1
-    assert json_response[1]['postage'] == 'second'
-
-    assert json_response[2]['month'] == 'May'
-    assert json_response[2]['notification_type'] == 'letter'
-    assert json_response[2]['rate'] == 0.39
-    assert json_response[2]['billing_units'] == 1
-    assert json_response[2]['postage'] == 'first'
-
-    assert json_response[3]['month'] == 'May'
-    assert json_response[3]['notification_type'] == 'sms'
-    assert json_response[3]['rate'] == 0.0150
-    assert json_response[3]['billing_units'] == 4
-    assert json_response[3]['postage'] == 'none'
-
-    assert json_response[4]['month'] == 'May'
-    assert json_response[4]['notification_type'] == 'sms'
-    assert json_response[4]['rate'] == 0.162
-    assert json_response[4]['billing_units'] == 5
-    assert json_response[4]['postage'] == 'none'
-
-
-def test_get_yearly_billing_usage_summary_from_ft_billing_all_cases(client, notify_db_session):
-    service = set_up_data_for_all_cases()
-    response = client.get('/service/{}/billing/ft-yearly-usage-summary?year=2018'.format(service.id),
-                          headers=[create_admin_authorization_header()])
-    assert response.status_code == 200
-    json_response = json.loads(response.get_data(as_text=True))
-
-    assert len(json_response) == 6
-    assert json_response[0]["notification_type"] == 'email'
-    assert json_response[0]["billing_units"] == 1
-    assert json_response[0]["rate"] == 0
-    assert json_response[0]["letter_total"] == 0
-
-    assert json_response[1]["notification_type"] == 'letter'
-    assert json_response[1]["billing_units"] == 1
-    assert json_response[1]["rate"] == 0.33
-    assert json_response[1]["letter_total"] == 0.33
-
-    assert json_response[2]["notification_type"] == 'letter'
-    assert json_response[2]["billing_units"] == 1
-    assert json_response[2]["rate"] == 0.36
-    assert json_response[2]["letter_total"] == 0.36
-
-    assert json_response[3]["notification_type"] == 'letter'
-    assert json_response[3]["billing_units"] == 1
-    assert json_response[3]["rate"] == 0.39
-    assert json_response[3]["letter_total"] == 0.39
-
-    assert json_response[4]["notification_type"] == 'sms'
-    assert json_response[4]["billing_units"] == 4
-    assert json_response[4]["rate"] == 0.0150
-    assert json_response[4]["letter_total"] == 0
-
-    assert json_response[5]["notification_type"] == 'sms'
-    assert json_response[5]["billing_units"] == 5
-    assert json_response[5]["rate"] == 0.162
-    assert json_response[5]["letter_total"] == 0
-
-
-def set_up_data_for_all_cases():
-    service = create_service()
-    sms_template = create_template(service=service, template_type="sms")
-    email_template = create_template(service=service, template_type="email")
-    letter_template = create_template(service=service, template_type="letter")
-    create_ft_billing(bst_date='2018-05-16',
-                      template=sms_template,
-                      rate_multiplier=1,
-                      international=False,
-                      rate=0.162,
-                      billable_unit=1,
-                      notifications_sent=1)
-    create_ft_billing(bst_date='2018-05-17',
-                      template=sms_template,
-                      rate_multiplier=2,
-                      international=False,
-                      rate=0.162,
-                      billable_unit=2,
-                      notifications_sent=1)
-    create_ft_billing(bst_date='2018-05-16',
-                      template=sms_template,
-                      rate_multiplier=2,
-                      international=False,
-                      rate=0.0150,
-                      billable_unit=2,
-                      notifications_sent=1)
-    create_ft_billing(bst_date='2018-05-16',
-                      template=email_template,
-                      rate_multiplier=1,
-                      international=False,
-                      rate=0,
-                      billable_unit=0,
-                      notifications_sent=1)
-    create_ft_billing(bst_date='2018-05-16',
-                      template=letter_template,
-                      rate_multiplier=1,
-                      international=False,
-                      rate=0.33,
-                      billable_unit=1,
-                      notifications_sent=1,
-                      postage='second')
-    create_ft_billing(bst_date='2018-05-17',
-                      template=letter_template,
-                      rate_multiplier=1,
-                      international=False,
-                      rate=0.36,
-                      billable_unit=2,
-                      notifications_sent=1,
-                      postage='second')
-    create_ft_billing(bst_date='2018-05-18',
-                      template=letter_template,
-                      rate_multiplier=1,
-                      international=False,
-                      rate=0.39,
-                      billable_unit=3,
-                      notifications_sent=1,
-                      postage='first')
-    return service

tests/app/billing/test_rest.py

@@ -0,0 +1,288 @@
+from calendar import monthrange
+from datetime import datetime
+
+import pytest
+from freezegun import freeze_time
+
+from app.billing.rest import update_free_sms_fragment_limit_data
+from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year
+from app.dao.date_util import (
+    get_current_financial_year_start_year,
+    get_month_start_and_end_date_in_utc,
+)
+from tests.app.db import (
+    create_annual_billing,
+    create_ft_billing,
+    create_service,
+    create_template,
+)
+
+APR_2016_MONTH_START = datetime(2016, 3, 31, 23, 00, 00)
+APR_2016_MONTH_END = datetime(2016, 4, 30, 22, 59, 59, 99999)
+
+IN_MAY_2016 = datetime(2016, 5, 10, 23, 00, 00)
+IN_JUN_2016 = datetime(2016, 6, 3, 23, 00, 00)
+
+
+def test_create_update_free_sms_fragment_limit_invalid_schema(admin_request, sample_service):
+    json_response = admin_request.post(
+        'billing.create_or_update_free_sms_fragment_limit',
+        service_id=sample_service.id,
+        _data={},
+        _expected_status=400
+    )
+
+    assert 'errors' in json_response
+
+
+def test_create_free_sms_fragment_limit_current_year_updates_future_years(admin_request, sample_service):
+    current_year = get_current_financial_year_start_year()
+    future_billing = create_annual_billing(sample_service.id, 1, current_year + 1)
+
+    admin_request.post(
+        'billing.create_or_update_free_sms_fragment_limit',
+        service_id=sample_service.id,
+        _data={'free_sms_fragment_limit': 9999},
+        _expected_status=201
+    )
+
+    current_billing = dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year)
+    assert future_billing.free_sms_fragment_limit == 9999
+    assert current_billing.financial_year_start == current_year
+    assert current_billing.free_sms_fragment_limit == 9999
+
+
+@pytest.mark.parametrize('update_existing', [True, False])
+def test_create_or_update_free_sms_fragment_limit_past_year_doenst_update_other_years(
+    admin_request,
+    sample_service,
+    update_existing
+):
+    current_year = get_current_financial_year_start_year()
+    create_annual_billing(sample_service.id, 1, current_year)
+    if update_existing:
+        create_annual_billing(sample_service.id, 1, current_year - 1)
+
+    data = {'financial_year_start': current_year - 1, 'free_sms_fragment_limit': 9999}
+    admin_request.post(
+        'billing.create_or_update_free_sms_fragment_limit',
+        service_id=sample_service.id,
+        _data=data,
+        _expected_status=201)
+
+    assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year - 1).free_sms_fragment_limit == 9999
+    assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year).free_sms_fragment_limit == 1
+
+
+def test_create_free_sms_fragment_limit_updates_existing_year(admin_request, sample_service):
+    current_year = get_current_financial_year_start_year()
+    annual_billing = create_annual_billing(sample_service.id, 1, current_year)
+
+    admin_request.post(
+        'billing.create_or_update_free_sms_fragment_limit',
+        service_id=sample_service.id,
+        _data={'financial_year_start': current_year, 'free_sms_fragment_limit': 2},
+        _expected_status=201)
+
+    assert annual_billing.free_sms_fragment_limit == 2
+
+
+@freeze_time('2021-04-02 13:00')
+def test_get_free_sms_fragment_limit(
+    admin_request, sample_service
+):
+    create_annual_billing(service_id=sample_service.id, free_sms_fragment_limit=11000, financial_year_start=2021)
+
+    json_response = admin_request.get(
+        'billing.get_free_sms_fragment_limit',
+        service_id=sample_service.id
+    )
+
+    assert json_response['financial_year_start'] == 2021
+    assert json_response['free_sms_fragment_limit'] == 11000
+
+
+@freeze_time('2021-04-02 13:00')
+def test_get_free_sms_fragment_limit_current_year_creates_new_row_if_annual_billing_is_missing(
+    admin_request, sample_service
+):
+    json_response = admin_request.get(
+        'billing.get_free_sms_fragment_limit',
+        service_id=sample_service.id
+    )
+
+    assert json_response['financial_year_start'] == 2021
+    assert json_response['free_sms_fragment_limit'] == 10000  # based on other organisation type
+
+
+def test_update_free_sms_fragment_limit_data(client, sample_service):
+    current_year = get_current_financial_year_start_year()
+    create_annual_billing(sample_service.id, free_sms_fragment_limit=250000, financial_year_start=current_year - 1)
+
+    update_free_sms_fragment_limit_data(sample_service.id, 9999, current_year)
+
+    annual_billing = dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year)
+    assert annual_billing.free_sms_fragment_limit == 9999
+
+
+def set_up_monthly_data():
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    email_template = create_template(service=service, template_type="email")
+    letter_template = create_template(service=service, template_type="letter")
+
+    for month in range(1, 13):
+        mon = str(month).zfill(2)
+        for day in range(1, monthrange(2016, month)[1] + 1):
+            d = str(day).zfill(2)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=sms_template,
+                              billable_unit=1,
+                              rate=0.162)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=email_template,
+                              rate=0)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=letter_template,
+                              billable_unit=1,
+                              rate=0.33,
+                              postage='second')
+
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=4, financial_year_start=2016)
+    return service
+
+
+def test_get_yearly_usage_by_monthly_from_ft_billing(admin_request, notify_db_session):
+    service = set_up_monthly_data()
+
+    json_response = admin_request.get(
+        'billing.get_yearly_usage_by_monthly_from_ft_billing',
+        service_id=service.id,
+        year=2016
+    )
+
+    assert len(json_response) == 18
+
+    email_rows = [row for row in json_response if row['notification_type'] == 'email']
+    assert len(email_rows) == 0
+
+    letter_row = next(x for x in json_response if x['notification_type'] == 'letter')
+    sms_row = next(x for x in json_response if x['notification_type'] == 'sms')
+
+    assert letter_row["month"] == "April"
+    assert letter_row["notification_type"] == "letter"
+    assert letter_row["billing_units"] == 30
+    assert letter_row["chargeable_units"] == 30
+    assert letter_row["rate"] == 0.33
+    assert letter_row["postage"] == "second"
+    assert letter_row["cost"] == 9.9
+    assert letter_row["free_chargeable_units"] == 0
+    assert letter_row["charged_units"] == 30
+    assert letter_row["notifications_sent"] == 30
+
+    assert sms_row["month"] == "April"
+    assert sms_row["notification_type"] == "sms"
+    assert sms_row["billing_units"] == 30
+    assert sms_row["chargeable_units"] == 30
+    assert sms_row["rate"] == 0.162
+    assert sms_row["postage"] == "none"
+    # free allowance is 4, so (30 - 4) * 0.162
+    assert sms_row["cost"] == 4.212
+    assert sms_row["free_chargeable_units"] == 4
+    assert sms_row["charged_units"] == 26
+    assert sms_row["notifications_sent"] == 30
+
+
+def set_up_yearly_data():
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    email_template = create_template(service=service, template_type="email")
+    letter_template = create_template(service=service, template_type="letter")
+
+    for month in range(1, 13):
+        mon = str(month).zfill(2)
+        for day in range(1, monthrange(2016, month)[1] + 1):
+            d = str(day).zfill(2)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=sms_template,
+                              rate=0.0162)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=sms_template,
+                              rate_multiplier=2,
+                              rate=0.0162)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=email_template,
+                              billable_unit=0,
+                              rate=0)
+            create_ft_billing(bst_date='2016-{}-{}'.format(mon, d),
+                              template=letter_template,
+                              rate=0.33,
+                              postage='second')
+        start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1))
+
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=4, financial_year_start=2016)
+    return service
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_returns_400_if_missing_year(admin_request, sample_service):
+    json_response = admin_request.get(
+        'billing.get_yearly_billing_usage_summary_from_ft_billing',
+        service_id=sample_service.id,
+        _expected_status=400
+    )
+    assert json_response == {
+        'message': 'No valid year provided', 'result': 'error'
+    }
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing_returns_empty_list_if_no_billing_data(
+    admin_request, sample_service
+):
+    json_response = admin_request.get(
+        'billing.get_yearly_billing_usage_summary_from_ft_billing',
+        service_id=sample_service.id,
+        year=2016
+    )
+    assert json_response == []
+
+
+def test_get_yearly_billing_usage_summary_from_ft_billing(admin_request, notify_db_session):
+    service = set_up_yearly_data()
+
+    json_response = admin_request.get(
+        'billing.get_yearly_billing_usage_summary_from_ft_billing',
+        service_id=service.id,
+        year=2016
+    )
+
+    assert len(json_response) == 3
+
+    assert json_response[0]['notification_type'] == 'email'
+    assert json_response[0]['billing_units'] == 275
+    assert json_response[0]['chargeable_units'] == 275
+    assert json_response[0]['rate'] == 0
+    assert json_response[0]['letter_total'] == 0
+    assert json_response[0]['cost'] == 0
+    assert json_response[0]['free_chargeable_units'] == 0
+    assert json_response[0]['charged_units'] == 275
+    assert json_response[0]['notifications_sent'] == 275
+
+    assert json_response[1]['notification_type'] == 'letter'
+    assert json_response[1]['billing_units'] == 275
+    assert json_response[1]['chargeable_units'] == 275
+    assert json_response[1]['rate'] == 0.33
+    assert json_response[1]['letter_total'] == 90.75
+    assert json_response[1]['cost'] == 90.75
+    assert json_response[1]['free_chargeable_units'] == 0
+    assert json_response[1]['charged_units'] == 275
+    assert json_response[1]['notifications_sent'] == 275
+
+    assert json_response[2]['notification_type'] == 'sms'
+    assert json_response[2]['billing_units'] == 825
+    assert json_response[2]['chargeable_units'] == 825
+    assert json_response[2]['rate'] == 0.0162
+    assert json_response[2]['letter_total'] == 0
+    assert json_response[2]['cost'] == 13.3002
+    assert json_response[2]['free_chargeable_units'] == 4
+    assert json_response[2]['charged_units'] == 821
+    assert json_response[2]['notifications_sent'] == 825

tests/app/clients/test_sms.py

@@ -24,7 +24,7 @@ def test_send_sms(fake_client, mocker):
         content='content',
         reference='reference',
         international=False,
-        sender=None,
+        sender='testing',
     )
 
     mock_send.assert_called_with(
@@ -45,22 +45,3 @@ def test_send_sms_error(fake_client, mocker):
             international=False,
             sender=None,
         )
-
-
-def test_send_sms_override_configured_shortcode_with_sender(
-    fake_client,
-    mocker
-):
-    mock_send = mocker.patch.object(fake_client, 'try_send_sms')
-
-    fake_client.send_sms(
-        to='to',
-        content='content',
-        reference='reference',
-        international=False,
-        sender='sender'
-    )
-
-    mock_send.assert_called_with(
-        'to', 'content', 'reference', False, 'sender'
-    )

tests/app/complaint/test_complaint_rest.py

@@ -87,5 +87,4 @@ def test_get_complaint_with_invalid_data_returns_400_status_code(client):
     )
 
     assert response.status_code == 400
-    assert response.json['errors'][0]['message'] == 'start_date time data {} does not match format %Y-%m-%d'.format(
-        start_date)
+    assert response.json['errors'][0]['message'] == 'start_date month must be in 1..12'

tests/app/conftest.py

@@ -53,6 +53,7 @@ from app.models import (
 from tests import create_admin_authorization_header
 from tests.app.db import (
     create_api_key,
+    create_email_branding,
     create_inbound_number,
     create_invited_org_user,
     create_job,
@@ -919,6 +920,19 @@ def broadcast_organisation(notify_db_session):
     return org
 
 
+@pytest.fixture
+def nhs_email_branding(notify_db_session):
+    # we wipe email_branding table in test db between the tests, so we have to recreate this branding
+    # that is normally present on all environments and applied through migration
+    nhs_email_branding_id = current_app.config['NHS_EMAIL_BRANDING_ID']
+
+    return create_email_branding(
+        id=nhs_email_branding_id,
+        logo='1ac6f483-3105-4c9e-9017-dd7fb2752c44-nhs-blue_x2.png',
+        name='NHS'
+    )
+
+
 @pytest.fixture
 def restore_provider_details(notify_db, notify_db_session):
     """

tests/app/dao/test_ft_billing_dao.py

@@ -10,6 +10,7 @@ from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
+    fetch_daily_sms_provider_volumes_for_platform,
     fetch_daily_volumes_for_platform,
     fetch_letter_costs_and_totals_for_all_services,
     fetch_letter_line_items_for_all_services,
@@ -44,8 +45,16 @@ def set_up_yearly_data():
     email_template = create_template(service=service, template_type="email")
     letter_template = create_template(service=service, template_type="letter")
 
-    start_date = date(2016, 3, 31)
-    end_date = date(2017, 4, 2)
+    # use different rates for adjacent financial years to make sure the query
+    # doesn't accidentally bleed over into them
+    for dt in (date(2016, 3, 31), date(2017, 4, 1)):
+        create_ft_billing(bst_date=dt, template=sms_template, rate=0.163)
+        create_ft_billing(bst_date=dt, template=email_template, rate=0)
+        create_ft_billing(bst_date=dt, template=letter_template, rate=0.34, postage='second')
+        create_ft_billing(bst_date=dt, template=letter_template, rate=0.31, postage='second')
+
+    start_date = date(2016, 4, 1)
+    end_date = date(2017, 4, 1)
 
     for n in range((end_date - start_date).days):
         dt = start_date + timedelta(days=n)
@@ -54,6 +63,20 @@ def set_up_yearly_data():
         create_ft_billing(bst_date=dt, template=email_template, rate=0)
         create_ft_billing(bst_date=dt, template=letter_template, rate=0.33, postage='second')
         create_ft_billing(bst_date=dt, template=letter_template, rate=0.30, postage='second')
+
+    return service
+
+
+def set_up_yearly_data_variable_rates():
+    service = create_service()
+    sms_template = create_template(service=service, template_type="sms")
+    letter_template = create_template(service=service, template_type="letter")
+
+    create_ft_billing(bst_date='2018-05-16', template=sms_template, rate=0.162)
+    create_ft_billing(bst_date='2018-05-17', template=sms_template, rate_multiplier=2, rate=0.162, billable_unit=2)
+    create_ft_billing(bst_date='2018-05-16', template=sms_template, rate_multiplier=2, rate=0.0150, billable_unit=2)
+    create_ft_billing(bst_date='2018-05-16', template=letter_template, rate=0.33, postage='second')
+    create_ft_billing(bst_date='2018-05-17', template=letter_template, rate=0.36, billable_unit=2, postage='second')
     return service
 
 
@@ -393,107 +416,229 @@ def test_get_rate_for_letters_when_page_count_is_zero(notify_db_session):
 
 
 def test_fetch_monthly_billing_for_year(notify_db_session):
-    service = create_service()
-    template = create_template(service=service, template_type="sms")
-    for i in range(1, 31):
-        create_ft_billing(bst_date='2018-06-{}'.format(i),
-                          template=template,
-                          rate_multiplier=2,
-                          rate=0.162)
-    for i in range(1, 32):
-        create_ft_billing(bst_date='2018-07-{}'.format(i),
-                          template=template,
-                          rate=0.158)
+    service = set_up_yearly_data()
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=10, financial_year_start=2016)
+    results = fetch_monthly_billing_for_year(service.id, 2016)
 
-    results = fetch_monthly_billing_for_year(service_id=service.id, year=2018)
+    assert len(results) == 48
 
-    assert len(results) == 2
-    assert str(results[0].month) == "2018-06-01"
+    assert str(results[0].month) == "2016-04-01"
+    assert results[0].notification_type == 'email'
     assert results[0].notifications_sent == 30
-    assert results[0].billable_units == Decimal('60')
-    assert results[0].rate == Decimal('0.162')
-    assert results[0].notification_type == 'sms'
-    assert results[0].postage == 'none'
+    assert results[0].billable_units == 30
+    assert results[0].chargeable_units == 30
+    assert results[0].rate == Decimal('0')
+    assert results[0].cost == Decimal('0')
+    assert results[0].free_chargeable_units == 0
+    assert results[0].charged_units == 30
 
-    assert str(results[1].month) == "2018-07-01"
-    assert results[1].notifications_sent == 31
-    assert results[1].billable_units == Decimal('31')
-    assert results[1].rate == Decimal('0.158')
-    assert results[1].notification_type == 'sms'
-    assert results[1].postage == 'none'
+    assert str(results[1].month) == "2016-04-01"
+    assert results[1].notification_type == 'letter'
+    assert results[1].notifications_sent == 30
+    assert results[1].billable_units == 30
+    assert results[1].chargeable_units == 30
+    assert results[1].rate == Decimal('0.30')
+    assert results[1].cost == Decimal('9')
+    assert results[1].free_chargeable_units == 0
+    assert results[1].charged_units == 30
+
+    assert str(results[1].month) == "2016-04-01"
+    assert results[2].notification_type == 'letter'
+    assert results[2].notifications_sent == 30
+    assert results[2].billable_units == 30
+    assert results[2].chargeable_units == 30
+    assert results[2].rate == Decimal('0.33')
+    assert results[2].cost == Decimal('9.9')
+    assert results[2].free_chargeable_units == 0
+    assert results[2].charged_units == 30
+
+    assert str(results[3].month) == "2016-04-01"
+    assert results[3].notification_type == 'sms'
+    assert results[3].notifications_sent == 30
+    assert results[3].billable_units == 30
+    assert results[3].chargeable_units == 30
+    assert results[3].rate == Decimal('0.162')
+    # free allowance is 10, so (30 - 10) * 0.162
+    assert results[3].cost == Decimal('3.24')
+    assert results[3].free_chargeable_units == 10
+    assert results[3].charged_units == 20
+
+    assert str(results[4].month) == "2016-05-01"
+    assert str(results[47].month) == "2017-03-01"
+
+
+def test_fetch_monthly_billing_for_year_variable_rates(notify_db_session):
+    service = set_up_yearly_data_variable_rates()
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=6, financial_year_start=2018)
+    results = fetch_monthly_billing_for_year(service.id, 2018)
+
+    # Test data is only for the month of May
+    assert len(results) == 4
+
+    assert str(results[0].month) == "2018-05-01"
+    assert results[0].notification_type == 'letter'
+    assert results[0].notifications_sent == 1
+    assert results[0].billable_units == 1
+    assert results[0].chargeable_units == 1
+    assert results[0].rate == Decimal('0.33')
+    assert results[0].cost == Decimal('0.33')
+    assert results[0].free_chargeable_units == 0
+    assert results[0].charged_units == 1
+
+    assert str(results[1].month) == "2018-05-01"
+    assert results[1].notification_type == 'letter'
+    assert results[1].notifications_sent == 1
+    assert results[1].billable_units == 1
+    assert results[1].chargeable_units == 2
+    assert results[1].rate == Decimal('0.36')
+    assert results[1].cost == Decimal('0.36')
+    assert results[1].free_chargeable_units == 0
+    assert results[1].charged_units == 1
+
+    assert str(results[2].month) == "2018-05-01"
+    assert results[2].notification_type == 'sms'
+    assert results[2].notifications_sent == 1
+    assert results[2].billable_units == 4
+    assert results[2].chargeable_units == 4
+    assert results[2].rate == Decimal('0.015')
+    # 4 free units sent on the 16th, 0 on the 17th
+    assert results[2].cost == Decimal('0')
+    assert results[2].free_chargeable_units == 4
+    assert results[2].charged_units == 0
+
+    assert str(results[3].month) == "2018-05-01"
+    assert results[3].notification_type == 'sms'
+    assert results[3].notifications_sent == 2
+    assert results[3].billable_units == 5
+    assert results[3].chargeable_units == 5
+    assert results[3].rate == Decimal('0.162')
+    # 1 free unit on the 16th, 1 on the 17th (+ 3 paid)
+    assert results[3].cost == Decimal('0.486')
+    assert results[3].free_chargeable_units == 2
+    assert results[3].charged_units == 3
 
 
 @freeze_time('2018-08-01 13:30:00')
 def test_fetch_monthly_billing_for_year_adds_data_for_today(notify_db_session):
     service = create_service()
-    template = create_template(service=service, template_type="email")
+    template = create_template(service=service, template_type="sms")
+
+    create_rate(start_date=datetime.utcnow() - timedelta(days=1), value=0.158, notification_type='sms')
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=1000, financial_year_start=2018)
+
     for i in range(1, 32):
         create_ft_billing(bst_date='2018-07-{}'.format(i), template=template)
+
     create_notification(template=template, status='delivered')
 
     assert db.session.query(FactBilling.bst_date).count() == 31
-    results = fetch_monthly_billing_for_year(service_id=service.id,
-                                             year=2018)
+    results = fetch_monthly_billing_for_year(service_id=service.id, year=2018)
+
     assert db.session.query(FactBilling.bst_date).count() == 32
     assert len(results) == 2
 
 
-def test_fetch_monthly_billing_for_year_return_financial_year(notify_db_session):
-    service = set_up_yearly_data()
-
-    results = fetch_monthly_billing_for_year(service.id, 2016)
-    # returns 3 rows, per month, returns financial year april to end of march
-    # Orders by Month
-
-    assert len(results) == 48
-    assert str(results[0].month) == "2016-04-01"
-    assert results[0].notification_type == 'email'
-    assert results[0].notifications_sent == 30
-    assert results[0].billable_units == 30
-    assert results[0].rate == Decimal('0')
-    assert str(results[1].month) == "2016-04-01"
-    assert results[1].notification_type == 'letter'
-    assert results[1].notifications_sent == 30
-    assert results[1].billable_units == 30
-    assert results[1].rate == Decimal('0.30')
-    assert str(results[1].month) == "2016-04-01"
-    assert results[2].notification_type == 'letter'
-    assert results[2].notifications_sent == 30
-    assert results[2].billable_units == 30
-    assert results[2].rate == Decimal('0.33')
-    assert str(results[3].month) == "2016-04-01"
-    assert results[3].notification_type == 'sms'
-    assert results[3].notifications_sent == 30
-    assert results[3].billable_units == 30
-    assert results[3].rate == Decimal('0.162')
-    assert str(results[4].month) == "2016-05-01"
-    assert str(results[47].month) == "2017-03-01"
-
-
 def test_fetch_billing_totals_for_year(notify_db_session):
     service = set_up_yearly_data()
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=1000, financial_year_start=2016)
     results = fetch_billing_totals_for_year(service_id=service.id, year=2016)
 
     assert len(results) == 4
     assert results[0].notification_type == 'email'
     assert results[0].notifications_sent == 365
     assert results[0].billable_units == 365
+    assert results[0].chargeable_units == 365
     assert results[0].rate == Decimal('0')
+    assert results[0].cost == Decimal('0')
+    assert results[0].free_chargeable_units == 0
+    assert results[0].charged_units == 365
 
     assert results[1].notification_type == 'letter'
     assert results[1].notifications_sent == 365
     assert results[1].billable_units == 365
+    assert results[1].chargeable_units == 365
     assert results[1].rate == Decimal('0.3')
+    assert results[1].cost == Decimal('109.5')
+    assert results[1].free_chargeable_units == 0
+    assert results[1].charged_units == 365
 
     assert results[2].notification_type == 'letter'
     assert results[2].notifications_sent == 365
     assert results[2].billable_units == 365
+    assert results[2].chargeable_units == 365
     assert results[2].rate == Decimal('0.33')
+    assert results[2].cost == Decimal('120.45')
+    assert results[2].free_chargeable_units == 0
+    assert results[2].charged_units == 365
 
     assert results[3].notification_type == 'sms'
     assert results[3].notifications_sent == 365
     assert results[3].billable_units == 365
+    assert results[3].chargeable_units == 365
     assert results[3].rate == Decimal('0.162')
+    assert results[3].cost == Decimal('0')
+    assert results[3].free_chargeable_units == 365
+    assert results[3].charged_units == 0
+
+
+def test_fetch_billing_totals_for_year_uses_current_annual_billing(notify_db_session):
+    service = set_up_yearly_data()
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=400, financial_year_start=2015)
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=0, financial_year_start=2016)
+
+    result = next(
+        result for result in
+        fetch_billing_totals_for_year(service_id=service.id, year=2016)
+        if result.notification_type == 'sms'
+    )
+
+    assert result.chargeable_units == 365
+    assert result.cost > 0
+
+
+def test_fetch_billing_totals_for_year_variable_rates(notify_db_session):
+    service = set_up_yearly_data_variable_rates()
+    create_annual_billing(service_id=service.id, free_sms_fragment_limit=6, financial_year_start=2018)
+    results = fetch_billing_totals_for_year(service_id=service.id, year=2018)
+
+    assert len(results) == 4
+    assert results[0].notification_type == 'letter'
+    assert results[0].notifications_sent == 1
+    assert results[0].billable_units == 1
+    assert results[0].chargeable_units == 1
+    assert results[0].rate == Decimal('0.33')
+    assert results[0].cost == Decimal('0.33')
+    assert results[0].free_chargeable_units == 0
+    assert results[0].charged_units == 1
+
+    assert results[1].notification_type == 'letter'
+    assert results[1].notifications_sent == 1
+    assert results[1].billable_units == 1
+    assert results[1].chargeable_units == 2
+    assert results[1].rate == Decimal('0.36')
+    assert results[1].cost == Decimal('0.36')
+    assert results[1].free_chargeable_units == 0
+    assert results[1].charged_units == 1
+
+    assert results[2].notification_type == 'sms'
+    assert results[2].notifications_sent == 1
+    assert results[2].billable_units == 4
+    assert results[2].chargeable_units == 4
+    assert results[2].rate == Decimal('0.015')
+    # 4 units sent on the 16th, 0 on the 17th
+    assert results[2].cost == Decimal('0')
+    assert results[2].free_chargeable_units == 4
+    assert results[2].charged_units == 0
+
+    assert results[3].notification_type == 'sms'
+    assert results[3].notifications_sent == 2
+    assert results[3].billable_units == 5
+    assert results[3].chargeable_units == 5
+    assert results[3].rate == Decimal('0.162')
+    # 1 free unit on the 16th, 1 on the 17th (+ 3 paid)
+    assert results[3].cost == Decimal('0.486')  # (5 - 2) * 0.162
+    assert results[3].free_chargeable_units == 2
+    assert results[3].charged_units == 3
 
 
 def test_delete_billing_data(notify_db_session):
@@ -856,6 +1001,84 @@ def test_fetch_daily_volumes_for_platform(
     assert results[1].letter_sheet_totals == 40
 
 
+def test_fetch_daily_sms_provider_volumes_for_platform_groups_values_by_provider(
+    notify_db_session,
+):
+    services = [
+        create_service(service_name='a'),
+        create_service(service_name='b')
+    ]
+    templates = [
+        create_template(services[0]),
+        create_template(services[1])
+    ]
+
+    create_ft_billing('2022-02-01', templates[0], provider='foo', notifications_sent=1, billable_unit=2)
+    create_ft_billing('2022-02-01', templates[1], provider='foo', notifications_sent=4, billable_unit=8)
+
+    create_ft_billing('2022-02-01', templates[0], provider='bar', notifications_sent=16, billable_unit=32)
+    create_ft_billing('2022-02-01', templates[1], provider='bar', notifications_sent=64, billable_unit=128)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 2
+    assert results[0].provider == 'bar'
+    assert results[0].sms_totals == 80
+    assert results[0].sms_fragment_totals == 160
+
+    assert results[1].provider == 'foo'
+    assert results[1].sms_totals == 5
+    assert results[1].sms_fragment_totals == 10
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_calculates_chargeable_units_and_costs(
+    sample_template,
+):
+    create_ft_billing('2022-02-01', sample_template, rate_multiplier=3, rate=1.5, notifications_sent=1, billable_unit=2)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 1
+    assert results[0].sms_totals == 1
+    assert results[0].sms_fragment_totals == 2
+    assert results[0].sms_chargeable_units == 6
+    assert results[0].sms_cost == 9
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_searches_dates_inclusively(sample_template):
+    # too early
+    create_ft_billing('2022-02-02', sample_template)
+
+    # just right
+    create_ft_billing('2022-02-03', sample_template)
+    create_ft_billing('2022-02-04', sample_template)
+    create_ft_billing('2022-02-05', sample_template)
+
+    # too late
+    create_ft_billing('2022-02-06', sample_template)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-03', end_date='2022-02-05')
+
+    assert len(results) == 3
+    assert results[0].bst_date == date(2022, 2, 3)
+    assert results[-1].bst_date == date(2022, 2, 5)
+
+
+def test_fetch_daily_sms_provider_volumes_for_platform_for_platform_only_returns_sms(
+    sample_template,
+    sample_email_template,
+    sample_letter_template
+):
+    create_ft_billing('2022-02-01', sample_template, notifications_sent=1)
+    create_ft_billing('2022-02-01', sample_email_template, notifications_sent=2)
+    create_ft_billing('2022-02-01', sample_letter_template, notifications_sent=4)
+
+    results = fetch_daily_sms_provider_volumes_for_platform(start_date='2022-02-01', end_date='2022-02-01')
+
+    assert len(results) == 1
+    assert results[0].sms_totals == 1
+
+
 def test_fetch_volumes_by_service(notify_db_session):
     set_up_usage_data(datetime(2022, 2, 1))
 

tests/app/db.py

@@ -502,13 +502,17 @@ def create_service_callback_api(
     return service_callback_api
 
 
-def create_email_branding(colour='blue', logo='test_x2.png', name='test_org_1', text='DisplayName'):
+def create_email_branding(
+    id=None, colour='blue', logo='test_x2.png', name='test_org_1', text='DisplayName'
+):
     data = {
         'colour': colour,
         'logo': logo,
         'name': name,
         'text': text,
     }
+    if id:
+        data['id'] = id
     email_branding = EmailBranding(**data)
     dao_create_email_branding(email_branding)
 
@@ -671,6 +675,7 @@ def create_organisation(
     billing_contact_names=None,
     billing_contact_email_addresses=None,
     billing_reference=None,
+    email_branding_id=None,
 ):
     data = {
         'id': organisation_id,
@@ -681,6 +686,7 @@ def create_organisation(
         'billing_contact_names': billing_contact_names,
         'billing_contact_email_addresses': billing_contact_email_addresses,
         'billing_reference': billing_reference,
+        'email_branding_id': email_branding_id
     }
     organisation = Organisation(**data)
     dao_create_organisation(organisation)

tests/app/organisation/test_rest.py

@@ -2,6 +2,7 @@ import uuid
 from datetime import datetime
 
 import pytest
+from flask import current_app
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -177,14 +178,39 @@ def test_post_create_organisation(admin_request, notify_db_session, crown):
         _expected_status=201
     )
 
-    organisation = Organisation.query.all()
+    organisations = Organisation.query.all()
 
     assert data['name'] == response['name']
     assert data['active'] == response['active']
     assert data['crown'] == response['crown']
     assert data['organisation_type'] == response['organisation_type']
 
-    assert len(organisation) == 1
+    assert len(organisations) == 1
+    # check that for non-nhs orgs, default branding is not set
+    assert organisations[0].email_branding_id is None
+
+
+@pytest.mark.parametrize('org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_create_organisation_sets_default_nhs_branding_for_nhs_orgs(
+    admin_request, notify_db_session, nhs_email_branding, org_type
+):
+    data = {
+        'name': 'test organisation',
+        'active': True,
+        'crown': False,
+        'organisation_type': org_type,
+    }
+
+    admin_request.post(
+        'organisation.create_organisation',
+        _data=data,
+        _expected_status=201
+    )
+
+    organisations = Organisation.query.all()
+
+    assert len(organisations) == 1
+    assert organisations[0].email_branding_id == uuid.UUID(current_app.config['NHS_EMAIL_BRANDING_ID'])
 
 
 def test_post_create_organisation_existing_name_raises_400(admin_request, sample_organisation):
@@ -344,6 +370,64 @@ def test_update_other_organisation_attributes_doesnt_clear_domains(
     ]
 
 
+@pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_update_organisation_to_nhs_type_updates_branding_if_none_present(
+    admin_request,
+    nhs_email_branding,
+    notify_db_session,
+    new_org_type
+):
+    org = create_organisation(organisation_type='central')
+    data = {
+        'organisation_type': new_org_type,
+    }
+
+    admin_request.post(
+        'organisation.update_organisation',
+        _data=data,
+        organisation_id=org.id,
+        _expected_status=204
+    )
+
+    organisation = Organisation.query.all()
+
+    assert len(organisation) == 1
+    assert organisation[0].id == org.id
+    assert organisation[0].organisation_type == new_org_type
+    assert organisation[0].email_branding_id == uuid.UUID(current_app.config['NHS_EMAIL_BRANDING_ID'])
+
+
+@pytest.mark.parametrize('new_org_type', ["nhs_central", "nhs_local", "nhs_gp"])
+def test_post_update_organisation_to_nhs_type_does_not_update_branding_if_default_branding_set(
+    admin_request,
+    nhs_email_branding,
+    notify_db_session,
+    new_org_type
+):
+    current_branding = create_email_branding(
+        logo='example.png',
+        name='custom branding'
+    )
+    org = create_organisation(organisation_type='central', email_branding_id=current_branding.id)
+    data = {
+        'organisation_type': new_org_type,
+    }
+
+    admin_request.post(
+        'organisation.update_organisation',
+        _data=data,
+        organisation_id=org.id,
+        _expected_status=204
+    )
+
+    organisation = Organisation.query.all()
+
+    assert len(organisation) == 1
+    assert organisation[0].id == org.id
+    assert organisation[0].organisation_type == new_org_type
+    assert organisation[0].email_branding_id == current_branding.id
+
+
 def test_update_organisation_default_branding(
     admin_request,
     notify_db_session,

tests/app/platform_stats/test_rest.py

@@ -9,6 +9,7 @@ from app.platform_stats.rest import (
     validate_date_range_is_within_a_financial_year,
 )
 from tests.app.db import (
+    create_ft_billing,
     create_ft_notification_status,
     create_notification,
     create_service,
@@ -47,8 +48,7 @@ def test_get_platform_stats_validates_the_date(admin_request):
         _expected_status=400
     )
 
-    assert response['errors'][0]['message'] == 'start_date time data {} does not match format %Y-%m-%d'.format(
-        start_date)
+    assert response['errors'][0]['message'] == 'start_date month must be in 1..12'
 
 
 @freeze_time('2018-10-31 14:00')
@@ -238,3 +238,23 @@ def test_volumes_by_service_report(
                            'service_id': str(fixture['service_with_sms_within_allowance'].id),
                            'service_name': fixture['service_with_sms_within_allowance'].name,
                            'sms_chargeable_units': 0, 'sms_notifications': 0}
+
+
+def test_daily_sms_provider_volumes_report(admin_request, sample_template):
+
+    create_ft_billing('2022-03-01', sample_template, provider='foo', rate=1.5, notifications_sent=1, billable_unit=3)
+    resp = admin_request.get(
+        'platform_stats.daily_sms_provider_volumes_report',
+        start_date='2022-03-01',
+        end_date='2022-03-01'
+    )
+
+    assert len(resp) == 1
+    assert resp[0] == {
+        'day': '2022-03-01',
+        'provider': 'foo',
+        'sms_totals': 1,
+        'sms_fragment_totals': 3,
+        'sms_chargeable_units': 3,
+        'sms_cost': 4.5,
+    }

tests/app/public_contracts/__init__.py

@@ -2,7 +2,7 @@ import os
 
 import jsonschema
 from flask import json
-from jsonschema import Draft4Validator
+from jsonschema import Draft7Validator
 
 
 def return_json_from_response(response):
@@ -22,5 +22,5 @@ def validate_v0(json_to_validate, schema_filename):
 
 
 def validate(json_to_validate, schema):
-    validator = Draft4Validator(schema)
+    validator = Draft7Validator(schema)
     validator.validate(json_to_validate, schema)

tests/app/public_contracts/schemas/v0/GET_notification_return_email.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for email notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/GET_notification_return_sms.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/GET_notifications_return.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "GET notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/POST_notification_return_email.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "POST notification return schema - for email notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/POST_notification_return_sms.json

@@ -1,5 +1,5 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "POST notification return schema - for sms notifications",
   "type" : "object",
   "properties": {

tests/app/public_contracts/schemas/v0/definitions.json

@@ -1,12 +1,8 @@
 {
-  "$schema": "http://json-schema.org/draft-04/schema#",
+  "$schema": "http://json-schema.org/draft-07/schema#",
   "description": "Common definitions - usage example: {'$ref': 'definitions.json#/uuid'} (swap quotes for double quotes)",
   "uuid": {
     "type": "string",
     "pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
-  },
-  "datetime": {
-    "type": "string",
-    "format": "date-time"
   }
 }

tests/app/public_contracts/schemas/v0/email_notification.json

@@ -14,19 +14,13 @@
       "type": "string",
       "enum": ["email"]
     },
-    "created_at": {"$ref": "definitions.json#/datetime"},
-    "sent_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "created_at": {"type": "string", "format": "date-time"},
+    "sent_at": {"type": ["string", "null"], "format": "date-time"},
     "sent_by": {"oneOf":[
       {"type": "string"},
       {"type": "null"}
     ]},
-    "updated_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "updated_at": {"type": ["string", "null"], "format": "date-time"},
     "status": {
       "type": "string",
       "enum": [

tests/app/public_contracts/schemas/v0/sms_notification.json

@@ -14,19 +14,13 @@
       "type": "string",
       "enum": ["sms"]
     },
-    "created_at": {"$ref": "definitions.json#/datetime"},
-    "sent_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "created_at": {"type": "string", "format": "date-time"},
+    "sent_at": {"type": ["string", "null"], "format": "date-time"},
     "sent_by": {"oneOf":[
       {"type": "string"},
       {"type": "null"}
     ]},
-    "updated_at": {"oneOf":[
-      {"$ref": "definitions.json#/datetime"},
-      {"type": "null"}
-    ]},
+    "updated_at": {"type": ["string", "null"], "format": "date-time"},
     "status": {
       "type": "string",
       "enum": [

tests/app/service/send_notification/test_send_pdf_letter_notification.py

@@ -1,5 +1,3 @@
-import uuid
-
 import pytest
 from freezegun import freeze_time
 from notifications_utils.s3 import S3ObjectNotFound
@@ -11,18 +9,27 @@ from app.v2.errors import BadRequestError, TooManyRequestsError
 from tests.app.db import create_service
 
 
+@pytest.fixture
+def post_data(sample_service_full_permissions, fake_uuid):
+    return {
+        'filename': 'valid.pdf',
+        'created_by': sample_service_full_permissions.users[0].id,
+        'file_id': fake_uuid,
+        'postage': 'second',
+        'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'
+    }
+
+
 @pytest.mark.parametrize('permissions', [
     [EMAIL_TYPE],
     [UPLOAD_LETTERS],
 ])
 def test_send_pdf_letter_notification_raises_error_if_service_does_not_have_permission(
     notify_db_session,
-    fake_uuid,
     permissions,
+    post_data,
 ):
     service = create_service(service_permissions=permissions)
-    post_data = {'filename': 'valid.pdf', 'created_by': fake_uuid, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(BadRequestError):
         send_pdf_letter_notification(service.id, post_data)
@@ -31,23 +38,22 @@ def test_send_pdf_letter_notification_raises_error_if_service_does_not_have_perm
 def test_send_pdf_letter_notification_raises_error_if_service_is_over_daily_message_limit(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
+    post_data,
 ):
     mocker.patch(
         'app.service.send_notification.check_service_over_daily_message_limit',
         side_effect=TooManyRequestsError(10))
-    post_data = {'filename': 'valid.pdf', 'created_by': fake_uuid, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(TooManyRequestsError):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
 
 
 def test_send_pdf_letter_notification_validates_created_by(
-    sample_service_full_permissions, fake_uuid, sample_user
+    sample_service_full_permissions,
+    sample_user,
+    post_data
 ):
-    post_data = {'filename': 'valid.pdf', 'created_by': sample_user.id, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
+    post_data['created_by'] = sample_user.id
 
     with pytest.raises(BadRequestError):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
@@ -56,12 +62,9 @@ def test_send_pdf_letter_notification_validates_created_by(
 def test_send_pdf_letter_notification_raises_error_if_service_in_trial_mode(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
+    post_data,
 ):
     sample_service_full_permissions.restricted = True
-    user = sample_service_full_permissions.users[0]
-    post_data = {'filename': 'valid.pdf', 'created_by': user.id, 'file_id': fake_uuid,
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
 
     with pytest.raises(BadRequestError) as e:
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
@@ -71,49 +74,54 @@ def test_send_pdf_letter_notification_raises_error_if_service_in_trial_mode(
 def test_send_pdf_letter_notification_raises_error_when_pdf_is_not_in_transient_letter_bucket(
     mocker,
     sample_service_full_permissions,
-    fake_uuid,
     notify_user,
+    post_data,
 ):
-    user = sample_service_full_permissions.users[0]
-    post_data = {'filename': 'valid.pdf', 'created_by': user.id, 'file_id': fake_uuid, 'postage': 'first',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
     mocker.patch('app.service.send_notification.utils_s3download', side_effect=S3ObjectNotFound({}, ''))
 
     with pytest.raises(S3ObjectNotFound):
         send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
 
 
+def test_send_pdf_letter_notification_does_nothing_if_notification_already_exists(
+    mocker,
+    sample_service_full_permissions,
+    notify_user,
+    sample_notification,
+    post_data,
+):
+    post_data['file_id'] = sample_notification.id
+    mocker.patch('app.service.send_notification.utils_s3download', side_effect=S3ObjectNotFound({}, ''))
+    response = send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
+    assert response['id'] == str(sample_notification.id)
+
+
 @freeze_time("2019-08-02 11:00:00")
 def test_send_pdf_letter_notification_creates_notification_and_moves_letter(
     mocker,
     sample_service_full_permissions,
     notify_user,
+    post_data,
 ):
-    user = sample_service_full_permissions.users[0]
-    filename = 'valid.pdf'
-    file_id = uuid.uuid4()
-    post_data = {'filename': filename, 'created_by': user.id, 'file_id': file_id, 'postage': 'second',
-                 'recipient_address': 'Bugs%20Bunny%0A123%20Main%20Street%0ALooney%20Town'}
-
     mocker.patch('app.service.send_notification.utils_s3download')
     mocker.patch('app.service.send_notification.get_page_count', return_value=1)
     s3_mock = mocker.patch('app.service.send_notification.move_uploaded_pdf_to_letters_bucket')
 
     result = send_pdf_letter_notification(sample_service_full_permissions.id, post_data)
+    file_id = post_data['file_id']
 
     notification = get_notification_by_id(file_id)
 
-    assert notification.id == file_id
+    assert str(notification.id) == file_id
     assert notification.api_key_id is None
-    assert notification.client_reference == filename
-    assert notification.created_by_id == user.id
+    assert notification.client_reference == post_data['filename']
+    assert notification.created_by_id == post_data['created_by']
     assert notification.postage == 'second'
     assert notification.notification_type == LETTER_TYPE
     assert notification.billable_units == 1
     assert notification.to == "Bugs Bunny\n123 Main Street\nLooney Town"
 
     assert notification.service_id == sample_service_full_permissions.id
-
     assert result == {'id': str(notification.id)}
 
     s3_mock.assert_called_once_with(

tests/app/test_cloudfoundry_config.py

@@ -3,52 +3,29 @@ import os
 
 import pytest
 
-from app.cloudfoundry_config import (
-    extract_cloudfoundry_config,
-    set_config_env_vars,
-)
+from app.cloudfoundry_config import extract_cloudfoundry_config
 
 
 @pytest.fixture
-def postgres_config():
-    return [
-        {
+def vcap_services():
+    return {
+        'postgres': [{
             'credentials': {
                 'uri': 'postgres uri'
             }
-        }
-    ]
-
-
-@pytest.fixture
-def cloudfoundry_config(postgres_config):
-    return {
-        'postgres': postgres_config,
+        }],
+        'redis': [{
+            'credentials': {
+                'uri': 'redis uri'
+            }
+        }],
         'user-provided': []
     }
 
 
-@pytest.fixture
-def cloudfoundry_environ(os_environ, cloudfoundry_config):
-    os.environ['VCAP_SERVICES'] = json.dumps(cloudfoundry_config)
-    os.environ['VCAP_APPLICATION'] = '{"space_name": "🚀🌌"}'
-
-
-def test_extract_cloudfoundry_config_populates_other_vars(cloudfoundry_environ):
+def test_extract_cloudfoundry_config_populates_other_vars(os_environ, vcap_services):
+    os.environ['VCAP_SERVICES'] = json.dumps(vcap_services)
     extract_cloudfoundry_config()
 
     assert os.environ['SQLALCHEMY_DATABASE_URI'] == 'postgresql uri'
-    assert os.environ['NOTIFY_ENVIRONMENT'] == '🚀🌌'
-    assert os.environ['NOTIFY_LOG_PATH'] == '/home/vcap/logs/app.log'
-
-
-def test_set_config_env_vars_ignores_unknown_configs(cloudfoundry_config, cloudfoundry_environ):
-    cloudfoundry_config['foo'] = {'credentials': {'foo': 'foo'}}
-    cloudfoundry_config['user-provided'].append({
-        'name': 'bar', 'credentials': {'bar': 'bar'}
-    })
-
-    set_config_env_vars(cloudfoundry_config)
-
-    assert 'foo' not in os.environ
-    assert 'bar' not in os.environ
+    assert os.environ['REDIS_URL'] == 'redis uri'

tests/app/v2/broadcast/sample_cap_xml_documents.py

@@ -248,5 +248,5 @@ WINDEMERE = """
 """
 
 LONG_GSM7 = WITH_PLACEHOLDER_FOR_CONTENT.format('a' * 1396)
-LONG_UCS2 = WITH_PLACEHOLDER_FOR_CONTENT.format('ŵ' * 616)
+LONG_UCS2 = WITH_PLACEHOLDER_FOR_CONTENT.format('ŵyl' * 205 + 'a')
 MISSING_AREA_NAMES = re.sub("<areaDesc>.*</areaDesc>", "<areaDesc> </areaDesc>", WAINFLEET)