The free sms allowances are changing for the financial year starting

April 1 2021.

In this PR there is a command to set annual_billing for all active
services with the the new defaults.

The new method `set_default_free_allowance_for_service` will also be
called in a PR to follow that will set a services free allowance to the
default if the organisation for the service is changed.

app/commands.py

@@ -12,6 +12,7 @@ from flask import current_app, json
 from notifications_utils.recipients import RecipientCSV
 from notifications_utils.statsd_decorators import statsd
 from notifications_utils.template import SMSMessageTemplate
+from sqlalchemy import and_
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 
@@ -29,6 +30,7 @@ from app.celery.tasks import process_row, record_daily_sorted_counts
 from app.config import QueueNames
 from app.dao.annual_billing_dao import (
     dao_create_or_update_annual_billing_for_year,
+    set_default_free_allowance_for_service,
 )
 from app.dao.fact_billing_dao import (
     delete_billing_data_for_service_for_day,
@@ -67,6 +69,7 @@ from app.models import (
     NOTIFICATION_CREATED,
     PROVIDERS,
     SMS_TYPE,
+    AnnualBilling,
     Domain,
     EmailBranding,
     LetterBranding,
@@ -234,36 +237,6 @@ def fix_notification_statuses_not_in_sync():
         result = db.session.execute(subq_hist).fetchall()
 
 
-@notify_command(name='populate-annual-billing')
-@click.option('-y', '--year', required=True, type=int,
-              help="""The year to populate the annual billing data for, i.e. 2019""")
-def populate_annual_billing(year):
-    """
-    add annual_billing for given year.
-    """
-    sql = """
-        Select id from services where active = true
-        except
-        select service_id
-        from annual_billing
-        where financial_year_start = :year
-    """
-    services_without_annual_billing = db.session.execute(sql, {"year": year})
-    for row in services_without_annual_billing:
-        latest_annual_billing = """
-            Select free_sms_fragment_limit
-            from annual_billing
-            where service_id = :service_id
-            order by financial_year_start desc limit 1
-        """
-        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
-        free_allowance = [x[0]for x in free_allowance_rows]
-        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
-        dao_create_or_update_annual_billing_for_year(service_id=row.id,
-                                                     free_sms_fragment_limit=free_allowance[0],
-                                                     financial_year_start=int(year))
-
-
 @notify_command(name='list-routes')
 def list_routes():
     """List URLs of all application routes."""
@@ -877,3 +850,67 @@ def process_row_from_job(job_id, job_row_number):
             notification_id = process_row(row, template, job, job.service)
             current_app.logger.info("Process row {} for job {} created notification_id: {}".format(
                 job_row_number, job_id, notification_id))
+
+
+@notify_command(name='populate-annual-billing-with-the-previous-years-allowance')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2019""")
+def populate_annual_billing_with_the_previous_years_allowance(year):
+    """
+    add annual_billing for given year.
+    """
+    sql = """
+        Select id from services where active = true
+        except
+        select service_id
+        from annual_billing
+        where financial_year_start = :year
+    """
+    services_without_annual_billing = db.session.execute(sql, {"year": year})
+    for row in services_without_annual_billing:
+        latest_annual_billing = """
+            Select free_sms_fragment_limit
+            from annual_billing
+            where service_id = :service_id
+            order by financial_year_start desc limit 1
+        """
+        free_allowance_rows = db.session.execute(latest_annual_billing, {"service_id": row.id})
+        free_allowance = [x[0]for x in free_allowance_rows]
+        print("create free limit of {} for service: {}".format(free_allowance[0], row.id))
+        dao_create_or_update_annual_billing_for_year(service_id=row.id,
+                                                     free_sms_fragment_limit=free_allowance[0],
+                                                     financial_year_start=int(year))
+
+
+@notify_command(name='populate-annual-billing-with-defaults')
+@click.option('-y', '--year', required=True, type=int,
+              help="""The year to populate the annual billing data for, i.e. 2021""")
+@click.option('-m', '--missing-services-only', default=False, type=bool,
+              help="""If true then only populate services missing from annual billing for the year.
+                      If false populate the default values for all active services.""")
+def populate_annual_billing_with_defaults(year, missing_services_only):
+    """
+    Add or update annual billing with free allowance defaults for all active services.
+    DEFAULT_FREE_SMS_FRAGMENT_LIMITS is the new free allowances for the financial year starting 2021.
+
+    If missing_services_only is true then only add rows for services that do not have annual billing for that year yet.
+    This is useful to prevent overriding any services that have a free allowance that is not the default.
+
+    If missing_services_only is false then add or update annual billing for all active services.
+    This is useful to ensure all services start the new year with the correct annual billing.
+    """
+    if missing_services_only:
+        active_services = Service.query.filter(
+            Service.active
+        ).outerjoin(
+            AnnualBilling, and_(Service.id == AnnualBilling.service_id, AnnualBilling.financial_year_start == year)
+        ).filter(
+            AnnualBilling.id == None  # noqa
+        ).all()
+    else:
+        active_services = Service.query.filter(
+            Service.active
+        ).all()
+
+    for service in active_services:
+        set_default_free_allowance_for_service(service, year)

app/dao/annual_billing_dao.py

@@ -1,3 +1,5 @@
+from flask import current_app
+
 from app import db
 from app.dao.dao_utils import transactional
 from app.dao.date_util import get_current_financial_year_start_year
@@ -49,3 +51,54 @@ def dao_get_all_free_sms_fragment_limit(service_id):
     return AnnualBilling.query.filter_by(
         service_id=service_id,
     ).order_by(AnnualBilling.financial_year_start).all()
+
+
+def set_default_free_allowance_for_service(service, year_start=None):
+    default_free_sms_fragment_limits = {
+        'central': {
+            2020: 250_000,
+            2021: 150_000,
+        },
+        'local': {
+            2020: 25_000,
+            2021: 25_000,
+        },
+        'nhs_central': {
+            2020: 250_000,
+            2021: 150_000,
+        },
+        'nhs_local': {
+            2020: 25_000,
+            2021: 25_000,
+        },
+        'nhs_gp': {
+            2020: 25_000,
+            2021: 10_000,
+        },
+        'emergency_service': {
+            2020: 25_000,
+            2021: 25_000,
+        },
+        'school_or_college': {
+            2020: 25_000,
+            2021: 10_000,
+        },
+        'other': {
+            2020: 25_000,
+            2021: 10_000,
+        },
+    }
+    if not year_start:
+        year_start = get_current_financial_year_start_year()
+    if service.organisation_type:
+        free_allowance = default_free_sms_fragment_limits[service.organisation_type][year_start]
+    else:
+        current_app.logger.info(f"no organisation type for service {service.id}. Using other default of "
+                                f"{default_free_sms_fragment_limits['other'][year_start]}")
+        free_allowance = default_free_sms_fragment_limits['other'][year_start]
+
+    dao_create_or_update_annual_billing_for_year(
+        service.id,
+        free_allowance,
+        year_start
+    )

tests/app/dao/test_annual_billing_dao.py

@@ -1,11 +1,15 @@
+import pytest
+from freezegun import freeze_time
 
 from app.dao.annual_billing_dao import (
     dao_create_or_update_annual_billing_for_year,
     dao_get_free_sms_fragment_limit_for_year,
     dao_update_annual_billing_for_future_years,
+    set_default_free_allowance_for_service,
 )
 from app.dao.date_util import get_current_financial_year_start_year
-from tests.app.db import create_annual_billing
+from app.models import AnnualBilling
+from tests.app.db import create_annual_billing, create_service_with_organisation
 
 
 def test_dao_update_free_sms_fragment_limit(notify_db_session, sample_service):
@@ -40,3 +44,48 @@ def test_dao_update_annual_billing_for_future_years(notify_db_session, sample_se
     assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year) is None
     assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year + 1).free_sms_fragment_limit == 9999
     assert dao_get_free_sms_fragment_limit_for_year(sample_service.id, current_year + 2).free_sms_fragment_limit == 9999
+
+
+@pytest.mark.parametrize('org_type, year, expected_default',
+                         [('central', 2021, 150000),
+                          ('local', 2021, 25000),
+                          ('nhs_central', 2021, 150000),
+                          ('nhs_local', 2021, 25000),
+                          ('nhs_gp', 2021, 10000),
+                          ('emergency_service', 2021, 25000),
+                          ('school_or_college', 2021, 10000),
+                          ('other', 2021, 10000),
+                          (None, 2021, 10000),
+                          ('central', 2020, 250000),
+                          ('local', 2020, 25000),
+                          ('nhs_central', 2020, 250000),
+                          ('nhs_local', 2020, 25000),
+                          ('nhs_gp', 2020, 25000),
+                          ('emergency_service', 2020, 25000),
+                          ('school_or_college', 2020, 25000),
+                          ('other', 2020, 25000),
+                          (None, 2020, 25000),
+                          ])
+def test_set_default_free_allowance_for_service(notify_db_session, org_type, year, expected_default):
+
+    service = create_service_with_organisation(org_type=org_type)
+
+    set_default_free_allowance_for_service(service=service, year_start=year)
+
+    annual_billing = AnnualBilling.query.all()
+
+    assert len(annual_billing) == 1
+    assert annual_billing[0].service_id == service.id
+    assert annual_billing[0].free_sms_fragment_limit == expected_default
+
+
+@freeze_time('2021-03-29 14:02:00')
+def test_set_default_free_allowance_for_service_using_correct_year(sample_service, mocker):
+    mock_dao = mocker.patch('app.dao.annual_billing_dao.dao_create_or_update_annual_billing_for_year')
+    set_default_free_allowance_for_service(service=sample_service, year_start=None)
+
+    mock_dao.assert_called_once_with(
+        sample_service.id,
+        25000,
+        2020
+    )

tests/app/db.py

@@ -199,6 +199,14 @@ def create_service_with_defined_sms_sender(
     return service
 
 
+def create_service_with_organisation(org_type):
+    service = create_service(service_name=f'{org_type} service')
+    org = create_organisation(name=f'{org_type} org', organisation_type=org_type)
+    dao_add_service_to_organisation(service=service, organisation_id=org.id)
+
+    return service
+
+
 def create_template(
         service,
         template_type=SMS_TYPE,