From 812f4d20dd6af12b7ea684a7bf9d863356c33292 Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Wed, 18 Jul 2018 17:03:16 +0100
Subject: [PATCH 01/13] Send complaints on to service callback APIs using an
 async task

---
 app/celery/scheduled_tasks.py                 |  4 +-
 app/celery/service_callback_tasks.py          | 98 ++++++++++++++-----
 app/dao/service_callback_api_dao.py           |  9 +-
 .../notifications_ses_callback.py             | 28 ++++--
 app/notifications/process_client_response.py  |  4 +-
 tests/app/celery/test_scheduled_tasks.py      |  4 +-
 .../app/celery/test_service_callback_tasks.py | 76 +++++++++++---
 .../test_notifications_ses_callback.py        | 31 +++++-
 .../test_process_client_response.py           |  4 +-
 9 files changed, 199 insertions(+), 59 deletions(-)

diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py
index 53b57aff5..e949c9326 100644
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -16,7 +16,7 @@ from app import performance_platform_client, zendesk_client
 from app.aws import s3
 from app.celery.service_callback_tasks import (
     send_delivery_status_to_service,
-    create_encrypted_callback_data,
+    create_delivery_status_callback_data,
 )
 from app.celery.tasks import process_job
 from app.config import QueueNames, TaskNames
@@ -212,7 +212,7 @@ def timeout_notifications():
         # queue callback task only if the service_callback_api exists
         service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
         if service_callback_api:
-            encrypted_notification = create_encrypted_callback_data(notification, service_callback_api)
+            encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
             send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
                                                         queue=QueueNames.CALLBACKS)
 
diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py
index ebf71d1f7..bf436b883 100644
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -17,44 +17,78 @@ from app.config import QueueNames
 
 @notify_celery.task(bind=True, name="send-delivery-status", max_retries=5, default_retry_delay=300)
 @statsd(namespace="tasks")
-def send_delivery_status_to_service(self, notification_id,
-                                    encrypted_status_update
-                                    ):
+def send_delivery_status_to_service(
+    self, notification_id, encrypted_status_update
+):
+    status_update = encryption.decrypt(encrypted_status_update)
+
+    data = {
+        "notification_id": str(notification_id),
+        "reference": status_update['notification_client_reference'],
+        "to": status_update['notification_to'],
+        "status": status_update['notification_status'],
+        "created_at": status_update['notification_created_at'],
+        "completed_at": status_update['notification_updated_at'],
+        "sent_at": status_update['notification_sent_at'],
+        "notification_type": status_update['notification_type']
+    }
+    _send_data_to_service_callback_api(
+        self,
+        data,
+        status_update['service_callback_api_url'],
+        status_update['service_callback_api_bearer_token'],
+        'send_delivery_status_to_service'
+    )
+
+
+@notify_celery.task(bind=True, name="send-complaint", max_retries=5, default_retry_delay=300)
+@statsd(namespace="tasks")
+def send_complaint_to_service(self, complaint_data):
+    complaint = encryption.decrypt(complaint_data)
+
+    data = {
+        "notification_id": complaint['notification_id'],
+        "complaint_id": complaint['complaint_id'],
+        "reference": complaint['reference'],
+        "to": complaint['to'],
+        "complaint_date": complaint['complaint_date']
+    }
+
+    _send_data_to_service_callback_api(
+        self,
+        data,
+        complaint['service_callback_api_url'],
+        complaint['service_callback_api_bearer_token'],
+        'send_complaint_to_service'
+    )
+
+
+def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name):
+    notification_id = data["notification_id"]
     try:
-        status_update = encryption.decrypt(encrypted_status_update)
-
-        data = {
-            "id": str(notification_id),
-            "reference": status_update['notification_client_reference'],
-            "to": status_update['notification_to'],
-            "status": status_update['notification_status'],
-            "created_at": status_update['notification_created_at'],
-            "completed_at": status_update['notification_updated_at'],
-            "sent_at": status_update['notification_sent_at'],
-            "notification_type": status_update['notification_type']
-        }
-
         response = request(
             method="POST",
-            url=status_update['service_callback_api_url'],
+            url=service_callback_url,
             data=json.dumps(data),
             headers={
                 'Content-Type': 'application/json',
-                'Authorization': 'Bearer {}'.format(status_update['service_callback_api_bearer_token'])
+                'Authorization': 'Bearer {}'.format(token)
             },
             timeout=60
         )
-        current_app.logger.info('send_delivery_status_to_service sending {} to {}, response {}'.format(
+        current_app.logger.info('{} sending {} to {}, response {}'.format(
+            function_name,
             notification_id,
-            status_update['service_callback_api_url'],
+            service_callback_url,
             response.status_code
         ))
         response.raise_for_status()
     except RequestException as e:
         current_app.logger.warning(
-            "send_delivery_status_to_service request failed for notification_id: {} and url: {}. exc: {}".format(
+            "{} request failed for notification_id: {} and url: {}. exc: {}".format(
+                function_name,
                 notification_id,
-                status_update['service_callback_api_url'],
+                service_callback_url,
                 e
             )
         )
@@ -63,12 +97,12 @@ def send_delivery_status_to_service(self, notification_id,
                 self.retry(queue=QueueNames.RETRY)
             except self.MaxRetriesExceededError:
                 current_app.logger.exception(
-                    """Retry: send_delivery_status_to_service has retried the max num of times
-                     for notification: {}""".format(notification_id)
+                    """Retry: {} has retried the max num of times
+                     for notification: {}""".format(function_name, notification_id)
                 )
 
 
-def create_encrypted_callback_data(notification, service_callback_api):
+def create_delivery_status_callback_data(notification, service_callback_api):
     from app import DATETIME_FORMAT, encryption
     data = {
         "notification_id": str(notification.id),
@@ -84,3 +118,17 @@ def create_encrypted_callback_data(notification, service_callback_api):
         "service_callback_api_bearer_token": service_callback_api.bearer_token,
     }
     return encryption.encrypt(data)
+
+
+def create_complaint_callback_data(complaint, notification, service_callback_api, recipient):
+    from app import DATETIME_FORMAT, encryption
+    data = {
+        "complaint_id": str(complaint.id),
+        "notification_id": str(notification.id),
+        "reference": notification.client_reference,
+        "to": recipient,
+        "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
+        "service_callback_api_url": service_callback_api.url,
+        "service_callback_api_bearer_token": service_callback_api.bearer_token,
+    }
+    return encryption.encrypt(data)
diff --git a/app/dao/service_callback_api_dao.py b/app/dao/service_callback_api_dao.py
index 26a0478a3..ea23d6306 100644
--- a/app/dao/service_callback_api_dao.py
+++ b/app/dao/service_callback_api_dao.py
@@ -4,7 +4,7 @@ from app import db, create_uuid
 from app.dao.dao_utils import transactional, version_class
 from app.models import ServiceCallbackApi
 
-from app.models import DELIVERY_STATUS_CALLBACK_TYPE
+from app.models import DELIVERY_STATUS_CALLBACK_TYPE, COMPLAINT_CALLBACK_TYPE
 
 
 @transactional
@@ -39,6 +39,13 @@ def get_service_delivery_status_callback_api_for_service(service_id):
     ).first()
 
 
+def get_service_complaint_callback_api_for_service(service_id):
+    return ServiceCallbackApi.query.filter_by(
+        service_id=service_id,
+        callback_type=COMPLAINT_CALLBACK_TYPE
+    ).first()
+
+
 @transactional
 def delete_service_callback_api(service_callback_api):
     db.session.delete(service_callback_api)
diff --git a/app/notifications/notifications_ses_callback.py b/app/notifications/notifications_ses_callback.py
index acd0d605e..e90cfeced 100644
--- a/app/notifications/notifications_ses_callback.py
+++ b/app/notifications/notifications_ses_callback.py
@@ -12,12 +12,16 @@ from app.dao import (
 )
 from app.dao.complaint_dao import save_complaint
 from app.dao.notifications_dao import dao_get_notification_history_by_reference
-from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
+from app.dao.service_callback_api_dao import (
+    get_service_delivery_status_callback_api_for_service, get_service_complaint_callback_api_for_service
+)
 from app.models import Complaint
 from app.notifications.process_client_response import validate_callback_data
 from app.celery.service_callback_tasks import (
     send_delivery_status_to_service,
-    create_encrypted_callback_data,
+    send_complaint_to_service,
+    create_delivery_status_callback_data,
+    create_complaint_callback_data
 )
 from app.config import QueueNames
 
@@ -38,7 +42,8 @@ def process_ses_response(ses_request):
         if notification_type == 'Bounce':
             notification_type = determine_notification_bounce_type(notification_type, ses_message)
         elif notification_type == 'Complaint':
-            handle_complaint(ses_message)
+            complaint, notification, recipient = handle_complaint(ses_message)
+            _check_and_queue_complaint_callback_task(complaint, notification, recipient)
             return
 
         try:
@@ -105,7 +110,7 @@ def determine_notification_bounce_type(notification_type, ses_message):
 
 
 def handle_complaint(ses_message):
-    remove_emails_from_complaint(ses_message)
+    recipient_email = remove_emails_from_complaint(ses_message)[0]
     current_app.logger.info("Complaint from SES: \n{}".format(ses_message))
     try:
         reference = ses_message['mail']['messageId']
@@ -123,6 +128,7 @@ def handle_complaint(ses_message):
         complaint_date=ses_complaint.get('timestamp', None) if ses_complaint else None
     )
     save_complaint(complaint)
+    return complaint, notification, recipient_email
 
 
 def remove_mail_headers(dict_to_edit):
@@ -141,13 +147,21 @@ def remove_emails_from_bounce(bounce_dict):
 def remove_emails_from_complaint(complaint_dict):
     remove_mail_headers(complaint_dict)
     complaint_dict['complaint'].pop('complainedRecipients')
-    complaint_dict['mail'].pop('destination')
+    return complaint_dict['mail'].pop('destination')
 
 
 def _check_and_queue_callback_task(notification):
     # queue callback task only if the service_callback_api exists
     service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
     if service_callback_api:
-        encrypted_notification = create_encrypted_callback_data(notification, service_callback_api)
-        send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
+        notification_data = create_delivery_status_callback_data(notification, service_callback_api)
+        send_delivery_status_to_service.apply_async([str(notification.id), notification_data],
                                                     queue=QueueNames.CALLBACKS)
+
+
+def _check_and_queue_complaint_callback_task(complaint, notification, recipient):
+    # queue callback task only if the service_callback_api exists
+    service_callback_api = get_service_complaint_callback_api_for_service(service_id=notification.service_id)
+    if service_callback_api:
+        complaint_data = create_complaint_callback_data(complaint, notification, service_callback_api, recipient)
+        send_complaint_to_service.apply_async([complaint_data], queue=QueueNames.CALLBACKS)
diff --git a/app/notifications/process_client_response.py b/app/notifications/process_client_response.py
index 2cfe861a6..ce2f023e5 100644
--- a/app/notifications/process_client_response.py
+++ b/app/notifications/process_client_response.py
@@ -10,7 +10,7 @@ from app.clients.sms.firetext import get_firetext_responses
 from app.clients.sms.mmg import get_mmg_responses
 from app.celery.service_callback_tasks import (
     send_delivery_status_to_service,
-    create_encrypted_callback_data,
+    create_delivery_status_callback_data,
 )
 from app.config import QueueNames
 from app.dao.notifications_dao import dao_update_notification
@@ -98,7 +98,7 @@ def _process_for_status(notification_status, client_name, provider_reference):
     service_callback_api = get_service_delivery_status_callback_api_for_service(service_id=notification.service_id)
 
     if service_callback_api:
-        encrypted_notification = create_encrypted_callback_data(notification, service_callback_api)
+        encrypted_notification = create_delivery_status_callback_data(notification, service_callback_api)
         send_delivery_status_to_service.apply_async([str(notification.id), encrypted_notification],
                                                     queue=QueueNames.CALLBACKS)
 
diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py
index 573aac282..0b192972d 100644
--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -60,7 +60,7 @@ from app.models import (
     SMS_TYPE
 )
 from app.utils import get_london_midnight_in_utc
-from app.celery.service_callback_tasks import create_encrypted_callback_data
+from app.celery.service_callback_tasks import create_delivery_status_callback_data
 from app.v2.errors import JobIncompleteError
 from tests.app.db import (
     create_notification, create_service, create_template, create_job, create_rate,
@@ -224,7 +224,7 @@ def test_timeout_notifications_sends_status_update_to_service(client, sample_tem
             seconds=current_app.config.get('SENDING_NOTIFICATIONS_TIMEOUT_PERIOD') + 10))
     timeout_notifications()
 
-    encrypted_data = create_encrypted_callback_data(notification, callback_api)
+    encrypted_data = create_delivery_status_callback_data(notification, callback_api)
     mocked.assert_called_once_with([str(notification.id), encrypted_data], queue=QueueNames.CALLBACKS)
 
 
diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py
index 4fbc858b1..543c90cf2 100644
--- a/tests/app/celery/test_service_callback_tasks.py
+++ b/tests/app/celery/test_service_callback_tasks.py
@@ -3,10 +3,12 @@ from datetime import datetime
 
 import pytest
 import requests_mock
+from freezegun import freeze_time
 
 from app import (DATETIME_FORMAT, encryption)
-from app.celery.service_callback_tasks import send_delivery_status_to_service
+from app.celery.service_callback_tasks import send_delivery_status_to_service, send_complaint_to_service
 from tests.app.db import (
+    create_complaint,
     create_notification,
     create_service_callback_api,
     create_service,
@@ -19,7 +21,7 @@ from tests.app.db import (
 def test_send_delivery_status_to_service_post_https_request_to_service_with_encrypted_data(
         notify_db_session, notification_type):
 
-    callback_api, template = _set_up_test_data(notification_type)
+    callback_api, template = _set_up_test_data(notification_type, "delivery_status")
     datestr = datetime(2017, 6, 20)
 
     notification = create_notification(template=template,
@@ -28,7 +30,7 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr
                                        sent_at=datestr,
                                        status='sent'
                                        )
-    encrypted_status_update = _set_up_encrypted_data(callback_api, notification)
+    encrypted_status_update = _set_up_data_for_status_update(callback_api, notification)
     with requests_mock.Mocker() as request_mock:
         request_mock.post(callback_api.url,
                           json={},
@@ -36,7 +38,7 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr
         send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update)
 
     mock_data = {
-        "id": str(notification.id),
+        "notification_id": str(notification.id),
         "reference": notification.client_reference,
         "to": notification.to,
         "status": notification.status,
@@ -54,12 +56,42 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr
     assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
 
 
+def test_send_complaint_to_service_posts_https_request_to_service_with_encrypted_data(notify_db_session):
+    with freeze_time('2001-01-01T12:00:00'):
+        callback_api, template = _set_up_test_data('email', "complaint")
+
+        notification = create_notification(template=template)
+        complaint = create_complaint(service=template.service, notification=notification)
+        complaint_data = _set_up_data_for_complaint(callback_api, complaint, notification)
+        with requests_mock.Mocker() as request_mock:
+            request_mock.post(callback_api.url,
+                              json={},
+                              status_code=200)
+            send_complaint_to_service(complaint_data)
+
+        mock_data = {
+            "notification_id": str(notification.id),
+            "complaint_id": str(complaint.id),
+            "reference": notification.client_reference,
+            "to": notification.to,
+            "complaint_date": datetime.utcnow().strftime(
+                DATETIME_FORMAT),
+        }
+
+        assert request_mock.call_count == 1
+        assert request_mock.request_history[0].url == callback_api.url
+        assert request_mock.request_history[0].method == 'POST'
+        assert request_mock.request_history[0].text == json.dumps(mock_data)
+        assert request_mock.request_history[0].headers["Content-type"] == "application/json"
+        assert request_mock.request_history[0].headers["Authorization"] == "Bearer {}".format(callback_api.bearer_token)
+
+
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_retries_if_request_returns_500_with_encrypted_data(
+def test__send_data_to_service_callback_api_retries_if_request_returns_500_with_encrypted_data(
         notify_db_session, mocker, notification_type
 ):
-    callback_api, template = _set_up_test_data(notification_type)
+    callback_api, template = _set_up_test_data(notification_type, "delivery_status")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,
@@ -67,7 +99,7 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500_with_enc
                                        sent_at=datestr,
                                        status='sent'
                                        )
-    encrypted_data = _set_up_encrypted_data(callback_api, notification)
+    encrypted_data = _set_up_data_for_status_update(callback_api, notification)
     mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
     with requests_mock.Mocker() as request_mock:
         request_mock.post(callback_api.url,
@@ -81,12 +113,12 @@ def test_send_delivery_status_to_service_retries_if_request_returns_500_with_enc
 
 @pytest.mark.parametrize("notification_type",
                          ["email", "letter", "sms"])
-def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404_with_encrypted_data(
+def test__send_data_to_service_callback_api_does_not_retry_if_request_returns_404_with_encrypted_data(
         notify_db_session,
         mocker,
         notification_type
 ):
-    callback_api, template = _set_up_test_data(notification_type)
+    callback_api, template = _set_up_test_data(notification_type, "delivery_status")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,
@@ -94,7 +126,7 @@ def test_send_delivery_status_to_service_does_not_retries_if_request_returns_404
                                        sent_at=datestr,
                                        status='sent'
                                        )
-    encrypted_data = _set_up_encrypted_data(callback_api, notification)
+    encrypted_data = _set_up_data_for_status_update(callback_api, notification)
     mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
     with requests_mock.Mocker() as request_mock:
         request_mock.post(callback_api.url,
@@ -109,7 +141,7 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none(
         notify_db_session,
         mocker
 ):
-    callback_api, template = _set_up_test_data('email')
+    callback_api, template = _set_up_test_data('email', "delivery_status")
     datestr = datetime(2017, 6, 20)
     notification = create_notification(template=template,
                                        created_at=datestr,
@@ -117,7 +149,7 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none(
                                        sent_at=None,
                                        status='technical-failure'
                                        )
-    encrypted_data = _set_up_encrypted_data(callback_api, notification)
+    encrypted_data = _set_up_data_for_status_update(callback_api, notification)
     mocked = mocker.patch('app.celery.service_callback_tasks.send_delivery_status_to_service.retry')
     with requests_mock.Mocker() as request_mock:
         request_mock.post(callback_api.url,
@@ -128,15 +160,15 @@ def test_send_delivery_status_to_service_succeeds_if_sent_at_is_none(
     assert mocked.call_count == 0
 
 
-def _set_up_test_data(notification_type):
+def _set_up_test_data(notification_type, callback_type):
     service = create_service(restricted=True)
     template = create_template(service=service, template_type=notification_type, subject='Hello')
     callback_api = create_service_callback_api(service=service, url="https://some.service.gov.uk/",
-                                               bearer_token="something_unique")
+                                               bearer_token="something_unique", callback_type=callback_type)
     return callback_api, template
 
 
-def _set_up_encrypted_data(callback_api, notification):
+def _set_up_data_for_status_update(callback_api, notification):
     data = {
         "notification_id": str(notification.id),
         "notification_client_reference": notification.client_reference,
@@ -152,3 +184,17 @@ def _set_up_encrypted_data(callback_api, notification):
     }
     encrypted_status_update = encryption.encrypt(data)
     return encrypted_status_update
+
+
+def _set_up_data_for_complaint(callback_api, complaint, notification):
+    data = {
+        "complaint_id": str(complaint.id),
+        "notification_id": str(notification.id),
+        "reference": notification.client_reference,
+        "to": notification.to,
+        "complaint_date": complaint.complaint_date.strftime(DATETIME_FORMAT),
+        "service_callback_api_url": callback_api.url,
+        "service_callback_api_bearer_token": callback_api.bearer_token,
+    }
+    obscured_status_update = encryption.encrypt(data)
+    return obscured_status_update
diff --git a/tests/app/notifications/test_notifications_ses_callback.py b/tests/app/notifications/test_notifications_ses_callback.py
index 701c1a21c..72166b9e8 100644
--- a/tests/app/notifications/test_notifications_ses_callback.py
+++ b/tests/app/notifications/test_notifications_ses_callback.py
@@ -5,7 +5,7 @@ from flask import json
 from freezegun import freeze_time
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import statsd_client
+from app import statsd_client, encryption
 from app.dao.notifications_dao import get_notification_by_id
 from app.models import Notification, Complaint
 from app.notifications.notifications_ses_callback import (
@@ -13,7 +13,7 @@ from app.notifications.notifications_ses_callback import (
     handle_complaint
 )
 from app.celery.research_mode_tasks import ses_hard_bounce_callback, ses_soft_bounce_callback, ses_notification_callback
-from app.celery.service_callback_tasks import create_encrypted_callback_data
+from app.celery.service_callback_tasks import create_delivery_status_callback_data
 
 from tests.app.conftest import sample_notification as create_sample_notification
 from tests.app.db import (
@@ -55,7 +55,7 @@ def test_ses_callback_should_update_notification_status(
         )
         statsd_client.incr.assert_any_call("callback.ses.delivered")
         updated_notification = Notification.query.get(notification.id)
-        encrypted_data = create_encrypted_callback_data(updated_notification, callback_api)
+        encrypted_data = create_delivery_status_callback_data(updated_notification, callback_api)
         send_mock.assert_called_once_with([str(notification.id), encrypted_data], queue="service-callbacks")
 
 
@@ -220,3 +220,28 @@ def test_process_ses_results_in_complaint_save_complaint_with_null_complaint_typ
     assert len(complaints) == 1
     assert complaints[0].notification_id == notification.id
     assert not complaints[0].complaint_type
+
+
+def test_ses_callback_should_send_on_complaint_to_user_callback_api(sample_email_template, mocker):
+    send_mock = mocker.patch(
+        'app.celery.service_callback_tasks.send_complaint_to_service.apply_async'
+    )
+    create_service_callback_api(
+        service=sample_email_template.service, url="https://original_url.com", callback_type="complaint"
+    )
+
+    notification = create_notification(template=sample_email_template, reference='ref1')
+    response = ses_complaint_callback()
+    errors = process_ses_response(response)
+    assert errors is None
+
+    assert send_mock.call_count == 1
+    assert encryption.decrypt(send_mock.call_args[0][0][0]) == {
+        'complaint_date': '2018-06-05T13:59:58.000000Z',
+        'complaint_id': str(Complaint.query.one().id),
+        'notification_id': str(notification.id),
+        'reference': None,
+        'service_callback_api_bearer_token': 'some_super_secret',
+        'service_callback_api_url': 'https://original_url.com',
+        'to': 'recipient1@example.com'
+    }
diff --git a/tests/app/notifications/test_process_client_response.py b/tests/app/notifications/test_process_client_response.py
index 1ea18cb69..6a22a2b03 100644
--- a/tests/app/notifications/test_process_client_response.py
+++ b/tests/app/notifications/test_process_client_response.py
@@ -7,7 +7,7 @@ from app.notifications.process_client_response import (
     validate_callback_data,
     process_sms_client_response
 )
-from app.celery.service_callback_tasks import create_encrypted_callback_data
+from app.celery.service_callback_tasks import create_delivery_status_callback_data
 from tests.app.db import create_service_callback_api
 
 
@@ -64,7 +64,7 @@ def test_outcome_statistics_called_for_successful_callback(sample_notification,
     success, error = process_sms_client_response(status='3', provider_reference=reference, client_name='MMG')
     assert success == "MMG callback succeeded. reference {} updated".format(str(reference))
     assert error is None
-    encrypted_data = create_encrypted_callback_data(sample_notification, callback_api)
+    encrypted_data = create_delivery_status_callback_data(sample_notification, callback_api)
     send_mock.assert_called_once_with([str(sample_notification.id), encrypted_data],
                                       queue="service-callbacks")
 

From 3048c05850eb7a6d42327acaab646525a8e11247 Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Fri, 20 Jul 2018 11:22:38 +0100
Subject: [PATCH 02/13] Revert notification_id name change in delivery_status
 service_callback_task data

---
 app/celery/service_callback_tasks.py            | 4 ++--
 tests/app/celery/test_service_callback_tasks.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/app/celery/service_callback_tasks.py b/app/celery/service_callback_tasks.py
index bf436b883..258ba9f9b 100644
--- a/app/celery/service_callback_tasks.py
+++ b/app/celery/service_callback_tasks.py
@@ -23,7 +23,7 @@ def send_delivery_status_to_service(
     status_update = encryption.decrypt(encrypted_status_update)
 
     data = {
-        "notification_id": str(notification_id),
+        "id": str(notification_id),
         "reference": status_update['notification_client_reference'],
         "to": status_update['notification_to'],
         "status": status_update['notification_status'],
@@ -64,7 +64,7 @@ def send_complaint_to_service(self, complaint_data):
 
 
 def _send_data_to_service_callback_api(self, data, service_callback_url, token, function_name):
-    notification_id = data["notification_id"]
+    notification_id = (data["notification_id"] if "notification_id" in data else data["id"])
     try:
         response = request(
             method="POST",
diff --git a/tests/app/celery/test_service_callback_tasks.py b/tests/app/celery/test_service_callback_tasks.py
index 543c90cf2..b17451722 100644
--- a/tests/app/celery/test_service_callback_tasks.py
+++ b/tests/app/celery/test_service_callback_tasks.py
@@ -38,7 +38,7 @@ def test_send_delivery_status_to_service_post_https_request_to_service_with_encr
         send_delivery_status_to_service(notification.id, encrypted_status_update=encrypted_status_update)
 
     mock_data = {
-        "notification_id": str(notification.id),
+        "id": str(notification.id),
         "reference": notification.client_reference,
         "to": notification.to,
         "status": notification.status,

From 4fc004b00a3255a7b9c3024278c4dd526795d03f Mon Sep 17 00:00:00 2001
From: Rebecca Law <rebecca.law@digital.cabinet-office.gov.uk>
Date: Tue, 24 Jul 2018 16:28:30 +0100
Subject: [PATCH 03/13] Increase the number of days we calculate billing from 3
 to 10 days. Log exception if the billing counts for letters are different in
 the dvla response file than what we collected.

---
 app/celery/reporting_tasks.py             | 2 +-
 app/celery/tasks.py                       | 6 ++++--
 tests/app/celery/test_ftp_update_tasks.py | 2 +-
 tests/app/celery/test_reporting_tasks.py  | 8 ++++----
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py
index 2b90eb623..eb2741be1 100644
--- a/app/celery/reporting_tasks.py
+++ b/app/celery/reporting_tasks.py
@@ -21,7 +21,7 @@ def create_nightly_billing(day_start=None):
     else:
         # When calling the task its a string in the format of "YYYY-MM-DD"
         day_start = datetime.strptime(day_start, "%Y-%m-%d")
-    for i in range(0, 3):
+    for i in range(0, 10):
         process_day = day_start - timedelta(days=i)
 
         transit_data = fetch_billing_data_for_day(process_day=process_day)
diff --git a/app/celery/tasks.py b/app/celery/tasks.py
index dd6d88495..3c2089942 100644
--- a/app/celery/tasks.py
+++ b/app/celery/tasks.py
@@ -491,8 +491,10 @@ def check_billable_units(notification_update):
     if int(notification_update.page_count) != notification.billable_units:
         msg = 'Notification with id {} had {} billable_units but a page count of {}'.format(
             notification.id, notification.billable_units, notification_update.page_count)
-
-        current_app.logger.error(msg)
+        try:
+            raise DVLAException(msg)
+        except DVLAException:
+            current_app.logger.exception(msg)
 
 
 @notify_celery.task(bind=True, name="send-inbound-sms", max_retries=5, default_retry_delay=300)
diff --git a/tests/app/celery/test_ftp_update_tasks.py b/tests/app/celery/test_ftp_update_tasks.py
index de118f5dd..0da3eb541 100644
--- a/tests/app/celery/test_ftp_update_tasks.py
+++ b/tests/app/celery/test_ftp_update_tasks.py
@@ -308,7 +308,7 @@ def test_check_billable_units_when_billable_units_does_not_match_page_count(
     mocker,
     notification_update
 ):
-    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.error')
+    mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
 
     notification = create_notification(sample_letter_template, reference='REFERENCE_ABC', billable_units=3)
 
diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py
index ecd4002f1..a4a21afd7 100644
--- a/tests/app/celery/test_reporting_tasks.py
+++ b/tests/app/celery/test_reporting_tasks.py
@@ -287,7 +287,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta(
     mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     # create records from 11th to 15th
-    for i in range(0, 5):
+    for i in range(0, 11):
         sample_notification(
             notify_db,
             notify_db_session,
@@ -302,7 +302,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta(
         )
 
     notification = Notification.query.order_by(Notification.created_at).all()
-    assert datetime.date(notification[0].created_at) == date(2018, 1, 11)
+    assert datetime.date(notification[0].created_at) == date(2018, 1, 5)
 
     records = FactBilling.query.all()
     assert len(records) == 0
@@ -310,8 +310,8 @@ def test_create_nightly_billing_consolidate_from_3_days_delta(
     create_nightly_billing()
     records = FactBilling.query.order_by(FactBilling.bst_date).all()
 
-    assert len(records) == 3
-    assert records[0].bst_date == date(2018, 1, 12)
+    assert len(records) == 10
+    assert records[0].bst_date == date(2018, 1, 5)
     assert records[-1].bst_date == date(2018, 1, 14)
 
 

From a81206091538584e8ada3e4817786e5fce346b36 Mon Sep 17 00:00:00 2001
From: Rebecca Law <rebecca.law@digital.cabinet-office.gov.uk>
Date: Wed, 25 Jul 2018 14:12:13 +0100
Subject: [PATCH 04/13] The unique constraint on SeviceCallbackApi was on
 service_id only. Now that we have 2 types of api callbacks the constraint to
 be on service_id + callback_type.

---
 app/models.py                                 |  6 ++-
 .../app/dao/test_service_callback_api_dao.py  | 43 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/app/models.py b/app/models.py
index 216c3bc16..bff4a664a 100644
--- a/app/models.py
+++ b/app/models.py
@@ -597,7 +597,7 @@ class ServiceInboundApi(db.Model, Versioned):
 class ServiceCallbackApi(db.Model, Versioned):
     __tablename__ = 'service_callback_api'
     id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False, unique=True)
+    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
     service = db.relationship('Service', backref='service_callback_api')
     url = db.Column(db.String(), nullable=False)
     callback_type = db.Column(db.String(), db.ForeignKey('service_callback_type.name'), nullable=True)
@@ -607,6 +607,10 @@ class ServiceCallbackApi(db.Model, Versioned):
     updated_by = db.relationship('User')
     updated_by_id = db.Column(UUID(as_uuid=True), db.ForeignKey('users.id'), index=True, nullable=False)
 
+    __table_args__ = (
+        UniqueConstraint('service_id', 'callback_type', name='uix_service_callback_type'),
+    )
+
     @property
     def bearer_token(self):
         if self._bearer_token:
diff --git a/tests/app/dao/test_service_callback_api_dao.py b/tests/app/dao/test_service_callback_api_dao.py
index 16d4ae95d..3b05e2fd1 100644
--- a/tests/app/dao/test_service_callback_api_dao.py
+++ b/tests/app/dao/test_service_callback_api_dao.py
@@ -56,6 +56,49 @@ def test_save_service_callback_api_fails_if_service_does_not_exist(notify_db, no
         save_service_callback_api(service_callback_api)
 
 
+def test_update_service_callback_api_unique_constraint(sample_service):
+    service_callback_api = ServiceCallbackApi(
+        service_id=sample_service.id,
+        url="https://some_service/callback_endpoint",
+        bearer_token="some_unique_string",
+        updated_by_id=sample_service.users[0].id,
+        callback_type='delivery_status'
+    )
+    save_service_callback_api(service_callback_api)
+    another = ServiceCallbackApi(
+        service_id=sample_service.id,
+        url="https://some_service/another_callback_endpoint",
+        bearer_token="different_string",
+        updated_by_id=sample_service.users[0].id,
+        callback_type='delivery_status'
+    )
+    with pytest.raises(expected_exception=SQLAlchemyError):
+        save_service_callback_api(another)
+
+
+def test_update_service_callback_can_add_two_api_of_different_types(sample_service):
+    delivery_status = ServiceCallbackApi(
+        service_id=sample_service.id,
+        url="https://some_service/callback_endpoint",
+        bearer_token="some_unique_string",
+        updated_by_id=sample_service.users[0].id,
+        callback_type='delivery_status'
+    )
+    save_service_callback_api(delivery_status)
+    complaint = ServiceCallbackApi(
+        service_id=sample_service.id,
+        url="https://some_service/another_callback_endpoint",
+        bearer_token="different_string",
+        updated_by_id=sample_service.users[0].id,
+        callback_type='complaint'
+    )
+    save_service_callback_api(complaint)
+    results = ServiceCallbackApi.query.order_by(ServiceCallbackApi.callback_type).all()
+    assert len(results) == 2
+    assert results[0].serialize() == complaint.serialize()
+    assert results[1].serialize() == delivery_status.serialize()
+
+
 def test_update_service_callback_api(sample_service):
     service_callback_api = ServiceCallbackApi(
         service_id=sample_service.id,

From 2b0ec9353eed43ef9c2a8f51bd450d5ee97868f0 Mon Sep 17 00:00:00 2001
From: Rebecca Law <rebecca.law@digital.cabinet-office.gov.uk>
Date: Wed, 25 Jul 2018 14:16:36 +0100
Subject: [PATCH 05/13] Added missing migration file

---
 migrations/versions/0208_fix_unique_index.py | 23 ++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 migrations/versions/0208_fix_unique_index.py

diff --git a/migrations/versions/0208_fix_unique_index.py b/migrations/versions/0208_fix_unique_index.py
new file mode 100644
index 000000000..52ac87c23
--- /dev/null
+++ b/migrations/versions/0208_fix_unique_index.py
@@ -0,0 +1,23 @@
+"""
+
+Revision ID: 0208_fix_unique_index
+Revises: 0207_set_callback_history_type
+Create Date: 2018-07-25 13:55:24.941794
+
+"""
+from alembic import op
+
+revision = '84c3b6eb16b3'
+down_revision = '0207_set_callback_history_type'
+
+
+def upgrade():
+    op.create_unique_constraint('uix_service_callback_type', 'service_callback_api', ['service_id', 'callback_type'])
+    op.drop_index('ix_service_callback_api_service_id', table_name='service_callback_api')
+    op.create_index(op.f('ix_service_callback_api_service_id'), 'service_callback_api', ['service_id'], unique=False)
+
+
+def downgrade():
+    op.drop_index(op.f('ix_service_callback_api_service_id'), table_name='service_callback_api')
+    op.create_index('ix_service_callback_api_service_id', 'service_callback_api', ['service_id'], unique=True)
+    op.drop_constraint('uix_service_callback_type', 'service_callback_api', type_='unique')

From d6603a0541c4855b0c2d5ad231922b3083933ed6 Mon Sep 17 00:00:00 2001
From: Tom Byers <tom.byers@digital.cabinet-office.gov.uk>
Date: Wed, 25 Jul 2018 15:19:25 +0100
Subject: [PATCH 06/13] Brings in spacing changes to the email template

---
 requirements-app.txt | 2 +-
 requirements.txt     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/requirements-app.txt b/requirements-app.txt
index 369024bac..abc9d8777 100644
--- a/requirements-app.txt
+++ b/requirements-app.txt
@@ -25,6 +25,6 @@ notifications-python-client==4.8.2
 # PaaS
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2
+git+https://github.com/alphagov/notifications-utils.git@29.3.3#egg=notifications-utils==29.3.3
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
diff --git a/requirements.txt b/requirements.txt
index c696af14d..ce2c67227 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -26,7 +26,7 @@ notifications-python-client==4.8.2
 # PaaS
 awscli-cwlogs>=1.4,<1.5
 
-git+https://github.com/alphagov/notifications-utils.git@29.3.2#egg=notifications-utils==29.3.2
+git+https://github.com/alphagov/notifications-utils.git@29.3.3#egg=notifications-utils==29.3.3
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
 
@@ -34,12 +34,12 @@ git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
 alembic==1.0.0
 amqp==1.4.9
 anyjson==0.3.3
-awscli==1.15.59
+awscli==1.15.64
 bcrypt==3.1.4
 billiard==3.3.0.23
 bleach==2.1.3
 boto3==1.6.16
-botocore==1.10.58
+botocore==1.10.63
 certifi==2018.4.16
 chardet==3.0.4
 click==6.7
@@ -47,7 +47,7 @@ colorama==0.3.9
 docutils==0.14
 Flask-Redis==0.3.0
 future==0.16.0
-greenlet==0.4.13
+greenlet==0.4.14
 html5lib==1.0.1
 idna==2.7
 itsdangerous==0.24

From a826f6e924cba1577d35123e2b3b2b6977a47fb8 Mon Sep 17 00:00:00 2001
From: Leo Hemsted <leo.hemsted@digital.cabinet-office.gov.uk>
Date: Thu, 26 Jul 2018 18:41:06 +0100
Subject: [PATCH 07/13] make rebuild-ft-billing-for-day remove unused rows

if the billing data was incorrect and needs to be rebuilt, we should
remove old rows. Previously we were only upserting new rows, but old
no-longer-relevant rows were staying in the database. This commit
makes the flask command remove *all* rows for that service and day,
before inserting the rows from the original notification data after.

This commit doesn't change the existing nightly task, nor does it
change the upsert that happens upon viewing the usage page. In normal
usage, there should never be a case where the number of billable units
for a rate decreases. It should only ever increase, thus, never need to
be deleted
---
 app/commands.py                      | 29 ++++++++++++++++++++++++----
 app/dao/fact_billing_dao.py          | 12 ++++++++++++
 tests/app/dao/test_ft_billing_dao.py | 29 ++++++++++++++++++++++++++--
 3 files changed, 64 insertions(+), 6 deletions(-)

diff --git a/app/commands.py b/app/commands.py
index 13fb5c4ed..982fbdc97 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -19,7 +19,11 @@ from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
 from app.dao.date_util import get_financial_year
-from app.dao.fact_billing_dao import fetch_billing_data_for_day, update_fact_billing
+from app.dao.fact_billing_dao import (
+    fetch_billing_data_for_day,
+    update_fact_billing,
+    delete_billing_data_for_service_for_day
+)
 from app.dao.monthly_billing_dao import (
     create_or_update_monthly_billing,
     get_monthly_billing_by_notification_type,
@@ -526,17 +530,34 @@ def rebuild_ft_billing_for_day(service_id, day):
     Rebuild the data in ft_billing for the given service_id and date
     """
     def rebuild_ft_data(process_day, service):
+        deleted_rows = delete_billing_data_for_service_for_day(process_day, service)
+        current_app.logger.info('deleted {} existing billing rows for {} on {}'.format(
+            deleted_rows,
+            service,
+            process_day
+        ))
         transit_data = fetch_billing_data_for_day(process_day=process_day, service_id=service)
+        # transit_data = every row that should exist
         for data in transit_data:
+            # upsert existing rows
             update_fact_billing(data, process_day)
+        current_app.logger.info('added/updated {} billing rows for {} on {}'.format(
+            len(transit_data),
+            service,
+            process_day
+        ))
+
     if service_id:
         # confirm the service exists
         dao_fetch_service_by_id(service_id)
         rebuild_ft_data(day, service_id)
     else:
-        services = get_service_ids_that_need_billing_populated(day, day)
-        for service_id in services:
-            rebuild_ft_data(day, service_id)
+        services = get_service_ids_that_need_billing_populated(
+            get_london_midnight_in_utc(day),
+            get_london_midnight_in_utc(day + timedelta(days=1))
+        )
+        for row in services:
+            rebuild_ft_data(day, row.service_id)
 
 
 @notify_command(name='compare-ft-billing-to-monthly-billing')
diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py
index db5758d69..0552fd032 100644
--- a/app/dao/fact_billing_dao.py
+++ b/app/dao/fact_billing_dao.py
@@ -125,6 +125,18 @@ def fetch_monthly_billing_for_year(service_id, year):
     return yearly_data
 
 
+def delete_billing_data_for_service_for_day(process_day, service_id):
+    """
+    Delete all ft_billing data for a given service on a given bst_date
+
+    Returns how many rows were deleted
+    """
+    return FactBilling.query.filter(
+        FactBilling.bst_date == process_day,
+        FactBilling.service_id == service_id
+    ).delete()
+
+
 def fetch_billing_data_for_day(process_day, service_id=None):
     start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
     end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py
index d286da29f..6a21732f1 100644
--- a/tests/app/dao/test_ft_billing_dao.py
+++ b/tests/app/dao/test_ft_billing_dao.py
@@ -6,9 +6,12 @@ from freezegun import freeze_time
 
 from app import db
 from app.dao.fact_billing_dao import (
-    fetch_monthly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing,
-    get_rate,
+    delete_billing_data_for_service_for_day,
+    fetch_billing_data_for_day,
     fetch_billing_totals_for_year,
+    fetch_monthly_billing_for_year,
+    get_rate,
+    get_rates_for_billing,
 )
 from app.models import FactBilling, Notification
 from app.utils import convert_utc_to_bst
@@ -353,3 +356,25 @@ def test_fetch_billing_totals_for_year(notify_db_session):
     assert results[3].notifications_sent == 365
     assert results[3].billable_units == 365
     assert results[3].rate == Decimal('0.162')
+
+
+def test_delete_billing_data(notify_db_session):
+    service_1 = create_service(service_name='1')
+    service_2 = create_service(service_name='2')
+    sms_template = create_template(service_1, 'sms')
+    email_template = create_template(service_1, 'email')
+    other_service_template = create_template(service_2, 'sms')
+
+    existing_rows_to_delete = [  # noqa
+        create_ft_billing('2018-01-01', 'sms', sms_template, service_1, billable_unit=1),
+        create_ft_billing('2018-01-01', 'email', email_template, service_1, billable_unit=2)
+    ]
+    other_day = create_ft_billing('2018-01-02', 'sms', sms_template, service_1, billable_unit=3)
+    other_service = create_ft_billing('2018-01-01', 'sms', other_service_template, service_2, billable_unit=4)
+
+    delete_billing_data_for_service_for_day('2018-01-01', service_1.id)
+
+    current_rows = FactBilling.query.all()
+    assert sorted(x.billable_units for x in current_rows) == sorted(
+        [other_day.billable_units, other_service.billable_units]
+    )

From c0f309a2a6b65586fd00e1fc12d1be3cce57dd9c Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Mon, 23 Jul 2018 14:57:44 +0100
Subject: [PATCH 08/13] Delete scheduled task to populate monthly_billing

---
 app/celery/scheduled_tasks.py            | 12 ----
 app/commands.py                          | 89 +-----------------------
 app/config.py                            |  5 --
 tests/app/celery/test_scheduled_tasks.py | 83 ----------------------
 4 files changed, 1 insertion(+), 188 deletions(-)

diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py
index 5de26e70e..85276aa30 100644
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -380,18 +380,6 @@ def raise_alert_if_letter_notifications_still_sending():
             current_app.logger.info(message)
 
 
-@notify_celery.task(name="populate_monthly_billing")
-@statsd(namespace="tasks")
-def populate_monthly_billing():
-    # for every service with billable units this month update billing totals for yesterday
-    # this will overwrite the existing amount.
-    yesterday = datetime.utcnow() - timedelta(days=1)
-    yesterday_in_bst = convert_utc_to_bst(yesterday)
-    start_date, end_date = get_month_start_and_end_date_in_utc(yesterday_in_bst)
-    services = get_service_ids_that_need_billing_populated(start_date=start_date, end_date=end_date)
-    [create_or_update_monthly_billing(service_id=s.service_id, billing_month=end_date) for s in services]
-
-
 @notify_celery.task(name="run-letter-jobs")
 @statsd(namespace="tasks")
 def run_letter_jobs():
diff --git a/app/commands.py b/app/commands.py
index 982fbdc97..8b1a9d286 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -24,11 +24,7 @@ from app.dao.fact_billing_dao import (
     update_fact_billing,
     delete_billing_data_for_service_for_day
 )
-from app.dao.monthly_billing_dao import (
-    create_or_update_monthly_billing,
-    get_monthly_billing_by_notification_type,
-    get_service_ids_that_need_billing_populated
-)
+
 from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
 from app.dao.service_callback_api_dao import get_service_delivery_status_callback_api_for_service
 from app.dao.services_dao import (
@@ -191,51 +187,6 @@ def fix_notification_statuses_not_in_sync():
         result = db.session.execute(subq_hist).fetchall()
 
 
-@notify_command()
-@click.option('-y', '--year', required=True, help="e.g. 2017", type=int)
-@click.option('-s', '--service_id', required=False, help="Enter the service id", type=click.UUID)
-@click.option('-m', '--month', required=False, help="e.g. 1 for January", type=int)
-def populate_monthly_billing(year, service_id=None, month=None):
-    """
-    Populate monthly billing table for all services for a given year.
-    If service_id and month provided then only rebuild monthly billing for that month.
-    """
-    def populate(service_id, year, month):
-        create_or_update_monthly_billing(service_id, datetime(year, month, 1))
-        sms_res = get_monthly_billing_by_notification_type(
-            service_id, datetime(year, month, 1), SMS_TYPE
-        )
-        email_res = get_monthly_billing_by_notification_type(
-            service_id, datetime(year, month, 1), EMAIL_TYPE
-        )
-        letter_res = get_monthly_billing_by_notification_type(
-            service_id, datetime(year, month, 1), 'letter'
-        )
-
-        print("Finished populating data for {}-{} for service id {}".format(month, year, str(service_id)))
-        print('SMS: {}'.format(sms_res.monthly_totals))
-        print('Email: {}'.format(email_res.monthly_totals))
-        print('Letter: {}'.format(letter_res.monthly_totals))
-
-    if service_id and month:
-            populate(service_id, year, month)
-    else:
-        service_ids = get_service_ids_that_need_billing_populated(
-            start_date=datetime(2016, 5, 1), end_date=datetime(2017, 8, 16)
-        )
-        start, end = 1, 13
-
-        if year == 2016:
-            start = 4
-
-        for service_id in service_ids:
-            print('Starting to populate data for service {}'.format(str(service_id)))
-            print('Starting populating monthly billing for {}'.format(year))
-            for i in range(start, end):
-                print('Population for {}-{}'.format(i, year))
-                populate(service_id, year, i)
-
-
 @notify_command()
 @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
 @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@@ -560,44 +511,6 @@ def rebuild_ft_billing_for_day(service_id, day):
             rebuild_ft_data(day, row.service_id)
 
 
-@notify_command(name='compare-ft-billing-to-monthly-billing')
-@click.option('-y', '--year', required=True)
-@click.option('-s', '--service_id', required=False, type=click.UUID)
-def compare_ft_billing_to_monthly_billing(year, service_id=None):
-    """
-    This command checks the results of monthly_billing to ft_billing for the given year.
-    If service id is not included all services are compared for the given year.
-    """
-    def compare_monthly_billing_to_ft_billing(ft_billing_resp, monthly_billing_resp):
-        # Remove the rows with 0 billing_units and rate, ft_billing doesn't populate those rows.
-        mo_json = json.loads(monthly_billing_resp.get_data(as_text=True))
-        rm_zero_rows = [x for x in mo_json if x['billing_units'] != 0 and x['rate'] != 0]
-        try:
-            assert rm_zero_rows == json.loads(ft_billing_resp.get_data(as_text=True))
-        except AssertionError:
-            print("Comparison failed for service: {} and year: {}".format(service_id, year))
-
-    if not service_id:
-        start_date, end_date = get_financial_year(year=int(year))
-        services = get_service_ids_that_need_billing_populated(start_date, end_date)
-        for service_id in services:
-            with current_app.test_request_context(
-                    path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
-                monthly_billing_response = get_yearly_usage_by_month(service_id)
-            with current_app.test_request_context(
-                    path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
-                ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
-            compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
-    else:
-        with current_app.test_request_context(
-                path='/service/{}/billing/monthly-usage?year={}'.format(service_id, year)):
-            monthly_billing_response = get_yearly_usage_by_month(service_id)
-        with current_app.test_request_context(
-                path='/service/{}/billing/ft-monthly-usage?year={}'.format(service_id, year)):
-            ft_billing_response = get_yearly_usage_by_monthly_from_ft_billing(service_id)
-        compare_monthly_billing_to_ft_billing(ft_billing_response, monthly_billing_response)
-
-
 @notify_command(name='migrate-data-to-ft-notification-status')
 @click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
 @click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
diff --git a/app/config.py b/app/config.py
index 0a490588f..f4486121e 100644
--- a/app/config.py
+++ b/app/config.py
@@ -238,11 +238,6 @@ class Config(object):
             'schedule': crontab(hour=4, minute=40),
             'options': {'queue': QueueNames.PERIODIC}
         },
-        'populate_monthly_billing': {
-            'task': 'populate_monthly_billing',
-            'schedule': crontab(hour=5, minute=10),
-            'options': {'queue': QueueNames.PERIODIC}
-        },
         'raise-alert-if-letter-notifications-still-sending': {
             'task': 'raise-alert-if-letter-notifications-still-sending',
             'schedule': crontab(hour=16, minute=30),
diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py
index 0b192972d..30b3308e8 100644
--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -27,7 +27,6 @@ from app.celery.scheduled_tasks import (
     run_scheduled_jobs,
     run_letter_jobs,
     trigger_letter_pdfs_for_day,
-    populate_monthly_billing,
     s3,
     send_daily_performance_platform_stats,
     send_scheduled_notifications,
@@ -48,7 +47,6 @@ from app.dao.provider_details_dao import (
 )
 from app.exceptions import NotificationTechnicalFailureException
 from app.models import (
-    MonthlyBilling,
     NotificationHistory,
     Service,
     StatsTemplateUsageByMonth,
@@ -125,8 +123,6 @@ def test_should_have_decorated_tasks_functions():
         'remove_transformed_dvla_files'
     assert delete_dvla_response_files_older_than_seven_days.__wrapped__.__name__ == \
         'delete_dvla_response_files_older_than_seven_days'
-    assert populate_monthly_billing.__wrapped__.__name__ == \
-        'populate_monthly_billing'
 
 
 @pytest.fixture(scope='function')
@@ -751,85 +747,6 @@ def test_tuesday_alert_if_letter_notifications_still_sending_reports_friday_lett
     )
 
 
-@freeze_time("2017-07-12 02:00:00")
-def test_populate_monthly_billing_populates_correctly(sample_template):
-    yesterday = datetime(2017, 7, 11, 13, 30)
-    jul_month_start = datetime(2017, 6, 30, 23)
-    jul_month_end = datetime(2017, 7, 31, 22, 59, 59, 99999)
-    create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
-
-    create_notification(template=sample_template, status='delivered', created_at=yesterday)
-    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=1))
-    create_notification(template=sample_template, status='delivered', created_at=yesterday + timedelta(days=1))
-    # not included in billing
-    create_notification(template=sample_template, status='delivered', created_at=yesterday - timedelta(days=30))
-
-    populate_monthly_billing()
-
-    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
-
-    assert len(monthly_billing) == 3
-
-    assert monthly_billing[0].service_id == sample_template.service_id
-    assert monthly_billing[0].start_date == jul_month_start
-    assert monthly_billing[0].end_date == jul_month_end
-    assert monthly_billing[0].notification_type == 'email'
-    assert monthly_billing[0].monthly_totals == []
-
-    assert monthly_billing[1].service_id == sample_template.service_id
-    assert monthly_billing[1].start_date == jul_month_start
-    assert monthly_billing[1].end_date == jul_month_end
-    assert monthly_billing[1].notification_type == 'sms'
-    assert sorted(monthly_billing[1].monthly_totals[0]) == sorted(
-        {
-            'international': False,
-            'rate_multiplier': 1,
-            'billing_units': 3,
-            'rate': 0.0123,
-            'total_cost': 0.0369
-        }
-    )
-
-    assert monthly_billing[2].service_id == sample_template.service_id
-    assert monthly_billing[2].start_date == jul_month_start
-    assert monthly_billing[2].end_date == jul_month_end
-    assert monthly_billing[2].notification_type == 'letter'
-    assert monthly_billing[2].monthly_totals == []
-
-
-@freeze_time("2016-04-01 23:00:00")
-def test_populate_monthly_billing_updates_correct_month_in_bst(sample_template):
-    yesterday = datetime.utcnow() - timedelta(days=1)
-    apr_month_start = datetime(2016, 3, 31, 23)
-    apr_month_end = datetime(2016, 4, 30, 22, 59, 59, 99999)
-    create_rate(datetime(2016, 1, 1), 0.0123, 'sms')
-    create_notification(template=sample_template, status='delivered', created_at=yesterday)
-    populate_monthly_billing()
-
-    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
-
-    assert len(monthly_billing) == 3
-
-    assert monthly_billing[0].service_id == sample_template.service_id
-    assert monthly_billing[0].start_date == apr_month_start
-    assert monthly_billing[0].end_date == apr_month_end
-    assert monthly_billing[0].notification_type == 'email'
-    assert monthly_billing[0].monthly_totals == []
-
-    assert monthly_billing[1].service_id == sample_template.service_id
-    assert monthly_billing[1].start_date == apr_month_start
-    assert monthly_billing[1].end_date == apr_month_end
-    assert monthly_billing[1].notification_type == 'sms'
-    assert monthly_billing[1].monthly_totals[0]['billing_units'] == 1
-    assert monthly_billing[1].monthly_totals[0]['total_cost'] == 0.0123
-
-    assert monthly_billing[2].service_id == sample_template.service_id
-    assert monthly_billing[2].start_date == apr_month_start
-    assert monthly_billing[2].end_date == apr_month_end
-    assert monthly_billing[2].notification_type == 'letter'
-    assert monthly_billing[2].monthly_totals == []
-
-
 def test_run_letter_jobs(client, mocker, sample_letter_template):
     jobs = [create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND),
             create_job(template=sample_letter_template, job_status=JOB_STATUS_READY_TO_SEND)]

From ca2b350a99949f065e07b3c6836fc0f59201254a Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Mon, 23 Jul 2018 15:14:37 +0100
Subject: [PATCH 09/13] Remove references to monthly_billing table from api

---
 app/billing/rest.py                      |  42 +-
 app/celery/scheduled_tasks.py            |   8 -
 app/commands.py                          |   3 +-
 app/dao/fact_billing_dao.py              |  11 +
 app/dao/monthly_billing_dao.py           | 128 ------
 app/schemas.py                           |   1 -
 tests/app/billing/test_billing.py        | 316 +-------------
 tests/app/celery/test_scheduled_tasks.py |   3 +-
 tests/app/dao/test_monthly_billing.py    | 521 -----------------------
 tests/app/db.py                          |  22 -
 10 files changed, 15 insertions(+), 1040 deletions(-)
 delete mode 100644 app/dao/monthly_billing_dao.py
 delete mode 100644 tests/app/dao/test_monthly_billing.py

diff --git a/app/billing/rest.py b/app/billing/rest.py
index e18f08282..ffdc7a068 100644
--- a/app/billing/rest.py
+++ b/app/billing/rest.py
@@ -1,4 +1,3 @@
-import json
 from datetime import datetime
 
 from flask import Blueprint, jsonify, request
@@ -15,12 +14,8 @@ from app.dao.annual_billing_dao import (
     dao_update_annual_billing_for_future_years
 )
 from app.dao.date_util import get_current_financial_year_start_year
-from app.dao.date_util import get_months_for_financial_year
 from app.dao.fact_billing_dao import fetch_monthly_billing_for_year, fetch_billing_totals_for_year
-from app.dao.monthly_billing_dao import (
-    get_billing_data_for_financial_year,
-    get_monthly_billing_by_notification_type
-)
+
 from app.errors import InvalidRequest
 from app.errors import register_errors
 from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE
@@ -60,41 +55,6 @@ def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     return jsonify(data)
 
 
-@billing_blueprint.route('/monthly-usage')
-def get_yearly_usage_by_month(service_id):
-    try:
-        year = int(request.args.get('year'))
-        results = []
-        for month in get_months_for_financial_year(year):
-            letter_billing_for_month = get_monthly_billing_by_notification_type(service_id, month, LETTER_TYPE)
-            if letter_billing_for_month:
-                results.extend(_transform_billing_for_month_letters(letter_billing_for_month))
-            billing_for_month = get_monthly_billing_by_notification_type(service_id, month, SMS_TYPE)
-            if billing_for_month:
-                results.append(_transform_billing_for_month_sms(billing_for_month))
-        return jsonify(results)
-
-    except TypeError:
-        return jsonify(result='error', message='No valid year provided'), 400
-
-
-@billing_blueprint.route('/yearly-usage-summary')
-def get_yearly_billing_usage_summary(service_id):
-    try:
-        year = int(request.args.get('year'))
-        billing_data = get_billing_data_for_financial_year(service_id, year)
-        notification_types = [EMAIL_TYPE, LETTER_TYPE, SMS_TYPE]
-        response = [
-            _get_total_billable_units_and_rate_for_notification_type(billing_data, notification_type)
-            for notification_type in notification_types
-        ]
-
-        return json.dumps(response)
-
-    except TypeError:
-        return jsonify(result='error', message='No valid year provided'), 400
-
-
 def _get_total_billable_units_and_rate_for_notification_type(billing_data, noti_type):
     total_sent = 0
     rate = 0
diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py
index 85276aa30..0b0b6ad85 100644
--- a/app/celery/scheduled_tasks.py
+++ b/app/celery/scheduled_tasks.py
@@ -19,7 +19,6 @@ from app.celery.service_callback_tasks import (
 )
 from app.celery.tasks import process_job
 from app.config import QueueNames, TaskNames
-from app.dao.date_util import get_month_start_and_end_date_in_utc
 from app.dao.inbound_sms_dao import delete_inbound_sms_created_more_than_a_week_ago
 from app.dao.invited_org_user_dao import delete_org_invitations_created_more_than_two_days_ago
 from app.dao.invited_user_dao import delete_invitations_created_more_than_two_days_ago
@@ -29,10 +28,6 @@ from app.dao.jobs_dao import (
     dao_get_jobs_older_than_limited_by
 )
 from app.dao.jobs_dao import dao_update_job
-from app.dao.monthly_billing_dao import (
-    get_service_ids_that_need_billing_populated,
-    create_or_update_monthly_billing
-)
 from app.dao.notifications_dao import (
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
@@ -67,9 +62,6 @@ from app.models import (
 )
 from app.notifications.process_notifications import send_notification_to_queue
 from app.performance_platform import total_sent_notifications, processing_time
-from app.utils import (
-    convert_utc_to_bst
-)
 from app.v2.errors import JobIncompleteError
 
 
diff --git a/app/commands.py b/app/commands.py
index 8b1a9d286..530fedd77 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -13,7 +13,6 @@ from sqlalchemy import func
 from notifications_utils.statsd_decorators import statsd
 
 from app import db, DATETIME_FORMAT, encryption, redis_store
-from app.billing.rest import get_yearly_usage_by_month, get_yearly_usage_by_monthly_from_ft_billing
 from app.celery.scheduled_tasks import send_total_sent_notifications_to_performance_platform
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
@@ -33,7 +32,7 @@ from app.dao.services_dao import (
     dao_fetch_service_by_id
 )
 from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
-from app.models import PROVIDERS, User, SMS_TYPE, EMAIL_TYPE, Notification
+from app.models import PROVIDERS, User, Notification
 from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
 from app.utils import (
     cache_key_for_service_template_usage_per_day,
diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py
index 0552fd032..f07e6c85d 100644
--- a/app/dao/fact_billing_dao.py
+++ b/app/dao/fact_billing_dao.py
@@ -200,6 +200,17 @@ def get_rates_for_billing():
     return non_letter_rates, letter_rates
 
 
+def get_service_ids_that_need_billing_populated(start_date, end_date):
+    return db.session.query(
+        NotificationHistory.service_id
+    ).filter(
+        NotificationHistory.created_at >= start_date,
+        NotificationHistory.created_at <= end_date,
+        NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]),
+        NotificationHistory.billable_units != 0
+    ).distinct().all()
+
+
 def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, letter_page_count=None):
     if notification_type == LETTER_TYPE:
         return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and letter_page_count == r[2])
diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py
deleted file mode 100644
index 502df8fa5..000000000
--- a/app/dao/monthly_billing_dao.py
+++ /dev/null
@@ -1,128 +0,0 @@
-from datetime import datetime
-
-from notifications_utils.statsd_decorators import statsd
-
-from app import db
-from app.dao.dao_utils import transactional
-from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year
-from app.dao.notification_usage_dao import get_billing_data_for_month
-from app.models import (
-    SMS_TYPE,
-    EMAIL_TYPE,
-    LETTER_TYPE,
-    MonthlyBilling,
-    NotificationHistory
-)
-from app.utils import convert_utc_to_bst
-
-
-def get_service_ids_that_need_billing_populated(start_date, end_date):
-    return db.session.query(
-        NotificationHistory.service_id
-    ).filter(
-        NotificationHistory.created_at >= start_date,
-        NotificationHistory.created_at <= end_date,
-        NotificationHistory.notification_type.in_([SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]),
-        NotificationHistory.billable_units != 0
-    ).distinct().all()
-
-
-@statsd(namespace="dao")
-def create_or_update_monthly_billing(service_id, billing_month):
-    start_date, end_date = get_month_start_and_end_date_in_utc(billing_month)
-    _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE)
-    _update_monthly_billing(service_id, start_date, end_date, EMAIL_TYPE)
-    _update_monthly_billing(service_id, start_date, end_date, LETTER_TYPE)
-
-
-def _monthly_billing_data_to_json(billing_data):
-    results = []
-    if billing_data:
-        # total cost must take into account the free allowance.
-        # might be a good idea to capture free allowance in this table
-        results = [{
-            "billing_units": x.billing_units,
-            "rate_multiplier": x.rate_multiplier,
-            "international": x.international,
-            "rate": x.rate,
-            "total_cost": (x.billing_units * x.rate_multiplier) * x.rate
-        } for x in billing_data]
-    return results
-
-
-@statsd(namespace="dao")
-@transactional
-def _update_monthly_billing(service_id, start_date, end_date, notification_type):
-    billing_data = get_billing_data_for_month(
-        service_id=service_id,
-        start_date=start_date,
-        end_date=end_date,
-        notification_type=notification_type
-    )
-    monthly_totals = _monthly_billing_data_to_json(billing_data)
-    row = get_monthly_billing_entry(service_id, start_date, notification_type)
-    if row:
-        row.monthly_totals = monthly_totals
-        row.updated_at = datetime.utcnow()
-    else:
-        row = MonthlyBilling(
-            service_id=service_id,
-            notification_type=notification_type,
-            monthly_totals=monthly_totals,
-            start_date=start_date,
-            end_date=end_date
-        )
-
-    db.session.add(row)
-
-
-def get_monthly_billing_entry(service_id, start_date, notification_type):
-    entry = MonthlyBilling.query.filter_by(
-        service_id=service_id,
-        start_date=start_date,
-        notification_type=notification_type
-    ).first()
-
-    return entry
-
-
-@statsd(namespace="dao")
-def get_yearly_billing_data_for_date_range(
-    service_id, start_date, end_date, notification_types
-):
-    results = db.session.query(
-        MonthlyBilling.notification_type,
-        MonthlyBilling.monthly_totals,
-        MonthlyBilling.start_date,
-    ).filter(
-        MonthlyBilling.service_id == service_id,
-        MonthlyBilling.start_date >= start_date,
-        MonthlyBilling.end_date <= end_date,
-        MonthlyBilling.notification_type.in_(notification_types)
-    ).order_by(
-        MonthlyBilling.start_date,
-        MonthlyBilling.notification_type,
-    ).all()
-
-    return results
-
-
-@statsd(namespace="dao")
-def get_monthly_billing_by_notification_type(service_id, billing_month, notification_type):
-    billing_month_in_bst = convert_utc_to_bst(billing_month)
-    start_date, _ = get_month_start_and_end_date_in_utc(billing_month_in_bst)
-    return get_monthly_billing_entry(service_id, start_date, notification_type)
-
-
-@statsd(namespace="dao")
-def get_billing_data_for_financial_year(service_id, year, notification_types=[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]):
-    now = convert_utc_to_bst(datetime.utcnow())
-    start_date, end_date = get_financial_year(year)
-    if start_date <= now <= end_date:
-        # Update totals to the latest so we include data for today
-        create_or_update_monthly_billing(service_id=service_id, billing_month=now)
-
-    results = get_yearly_billing_data_for_date_range(
-        service_id, start_date, end_date, notification_types
-    )
-    return results
diff --git a/app/schemas.py b/app/schemas.py
index db21fef8e..d9bb60e9e 100644
--- a/app/schemas.py
+++ b/app/schemas.py
@@ -232,7 +232,6 @@ class ServiceSchema(BaseSchema):
             'service_provider_stats',
             'service_notification_stats',
             'service_sms_senders',
-            'monthly_billing',
             'reply_to_email_addresses',
             'letter_contacts',
             'complaints',
diff --git a/tests/app/billing/test_billing.py b/tests/app/billing/test_billing.py
index 02de27604..3ea91bdda 100644
--- a/tests/app/billing/test_billing.py
+++ b/tests/app/billing/test_billing.py
@@ -4,18 +4,12 @@ import json
 
 import pytest
 
-from app.billing.rest import _transform_billing_for_month_sms
-from app.dao.monthly_billing_dao import (
-    create_or_update_monthly_billing,
-    get_monthly_billing_by_notification_type,
-)
-from app.models import SMS_TYPE, EMAIL_TYPE, LETTER_TYPE, FactBilling
+from app.models import FactBilling
 from app.dao.date_util import get_current_financial_year_start_year, get_month_start_and_end_date_in_utc
 from app.dao.annual_billing_dao import dao_get_free_sms_fragment_limit_for_year
 from tests.app.db import (
     create_notification,
     create_rate,
-    create_monthly_billing_entry,
     create_annual_billing,
     create_template,
     create_service,
@@ -36,253 +30,6 @@ def _assert_dict_equals(actual, expected_dict):
     assert actual == expected_dict
 
 
-def test_get_yearly_billing_summary_returns_correct_breakdown(client, sample_template):
-    create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE)
-    create_notification(
-        template=sample_template, created_at=IN_MAY_2016,
-        billable_units=1, rate_multiplier=2, status='delivered'
-    )
-    create_notification(
-        template=sample_template, created_at=IN_JUN_2016,
-        billable_units=2, rate_multiplier=3, status='delivered'
-    )
-
-    letter_template = create_template(service=sample_template.service, template_type=LETTER_TYPE)
-    create_notification(template=letter_template, created_at=IN_MAY_2016, status='delivered', billable_units=1)
-    create_notification(template=letter_template, created_at=IN_JUN_2016, status='delivered', billable_units=1)
-
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_MAY_2016)
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_JUN_2016)
-
-    response = client.get(
-        '/service/{}/billing/yearly-usage-summary?year=2016'.format(sample_template.service.id),
-        headers=[create_authorization_header()]
-    )
-    assert response.status_code == 200
-
-    resp_json = json.loads(response.get_data(as_text=True))
-    assert len(resp_json) == 3
-
-    _assert_dict_equals(resp_json[0], {
-        'notification_type': EMAIL_TYPE,
-        'billing_units': 0,
-        'rate': 0,
-        'letter_total': 0
-    })
-    _assert_dict_equals(resp_json[1], {
-        'notification_type': LETTER_TYPE,
-        'billing_units': 2,
-        'rate': 0,
-        'letter_total': 0.66
-    })
-    _assert_dict_equals(resp_json[2], {
-        'notification_type': SMS_TYPE,
-        'billing_units': 8,
-        'rate': 0.12,
-        'letter_total': 0
-    })
-
-
-def test_get_yearly_billing_usage_breakdown_returns_400_if_missing_year(client, sample_service):
-    response = client.get(
-        '/service/{}/billing/yearly-usage-summary'.format(sample_service.id),
-        headers=[create_authorization_header()]
-    )
-    assert response.status_code == 400
-    assert json.loads(response.get_data(as_text=True)) == {
-        'message': 'No valid year provided', 'result': 'error'
-    }
-
-
-def test_get_yearly_usage_by_month_returns_400_if_missing_year(client, sample_service):
-    response = client.get(
-        '/service/{}/billing/monthly-usage'.format(sample_service.id),
-        headers=[create_authorization_header()]
-    )
-    assert response.status_code == 400
-    assert json.loads(response.get_data(as_text=True)) == {
-        'message': 'No valid year provided', 'result': 'error'
-    }
-
-
-def test_get_yearly_usage_by_month_returns_empty_list_if_no_usage(client, sample_template):
-    create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE)
-    response = client.get(
-        '/service/{}/billing/monthly-usage?year=2016'.format(sample_template.service.id),
-        headers=[create_authorization_header()]
-    )
-    assert response.status_code == 200
-
-    results = json.loads(response.get_data(as_text=True))
-    assert results == []
-
-
-def test_get_yearly_usage_by_month_returns_correctly(client, sample_template):
-    create_rate(start_date=IN_MAY_2016 - timedelta(days=1), value=0.12, notification_type=SMS_TYPE)
-    create_notification(
-        template=sample_template, created_at=IN_MAY_2016,
-        billable_units=1, rate_multiplier=2, status='delivered'
-    )
-    create_notification(
-        template=sample_template, created_at=IN_JUN_2016,
-        billable_units=2, rate_multiplier=3, status='delivered'
-    )
-
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_MAY_2016)
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=IN_JUN_2016)
-
-    response = client.get(
-        '/service/{}/billing/monthly-usage?year=2016'.format(sample_template.service.id),
-        headers=[create_authorization_header()]
-    )
-
-    assert response.status_code == 200
-
-    resp_json = json.loads(response.get_data(as_text=True))
-    _assert_dict_equals(resp_json[0], {
-        'billing_units': 0,
-        'month': 'May',
-        'notification_type': LETTER_TYPE,
-        'rate': 0
-    })
-    _assert_dict_equals(resp_json[1], {
-        'billing_units': 2,
-        'month': 'May',
-        'notification_type': SMS_TYPE,
-        'rate': 0.12
-    })
-    _assert_dict_equals(resp_json[2], {
-        'billing_units': 0,
-        'month': 'June',
-        'notification_type': LETTER_TYPE,
-        'rate': 0
-    })
-    _assert_dict_equals(resp_json[3], {
-        'billing_units': 6,
-        'month': 'June',
-        'notification_type': SMS_TYPE,
-        'rate': 0.12
-    })
-
-
-def test_transform_billing_for_month_returns_empty_if_no_monthly_totals(sample_service):
-    create_monthly_billing_entry(
-        service=sample_service,
-        monthly_totals=[],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type(
-        sample_service.id, APR_2016_MONTH_START, SMS_TYPE
-    ))
-
-    _assert_dict_equals(transformed_billing_data, {
-        'notification_type': SMS_TYPE,
-        'billing_units': 0,
-        'month': 'April',
-        'rate': 0,
-    })
-
-
-def test_transform_billing_for_month_formats_monthly_totals_correctly(sample_service):
-    create_monthly_billing_entry(
-        service=sample_service,
-        monthly_totals=[{
-            "billing_units": 12,
-            "rate": 0.0158,
-            "rate_multiplier": 5,
-            "total_cost": 2.1804,
-            "international": False
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type(
-        sample_service.id, APR_2016_MONTH_START, SMS_TYPE
-    ))
-
-    _assert_dict_equals(transformed_billing_data, {
-        'notification_type': SMS_TYPE,
-        'billing_units': 60,
-        'month': 'April',
-        'rate': 0.0158,
-    })
-
-
-def test_transform_billing_sums_billable_units(sample_service):
-    create_monthly_billing_entry(
-        service=sample_service,
-        monthly_totals=[{
-            'billing_units': 1321,
-            'international': False,
-            'month': 'May',
-            'notification_type': SMS_TYPE,
-            'rate': 0.12,
-            'rate_multiplier': 1
-        }, {
-            'billing_units': 1,
-            'international': False,
-            'month': 'May',
-            'notification_type': SMS_TYPE,
-            'rate': 0.12,
-            'rate_multiplier': 1
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type(
-        sample_service.id, APR_2016_MONTH_START, SMS_TYPE
-    ))
-
-    _assert_dict_equals(transformed_billing_data, {
-        'notification_type': SMS_TYPE,
-        'billing_units': 1322,
-        'month': 'April',
-        'rate': 0.12,
-    })
-
-
-def test_transform_billing_calculates_with_different_rate_multipliers(sample_service):
-    create_monthly_billing_entry(
-        service=sample_service,
-        monthly_totals=[{
-            'billing_units': 1321,
-            'international': False,
-            'month': 'May',
-            'notification_type': SMS_TYPE,
-            'rate': 0.12,
-            'rate_multiplier': 1
-        }, {
-            'billing_units': 1,
-            'international': False,
-            'month': 'May',
-            'notification_type': SMS_TYPE,
-            'rate': 0.12,
-            'rate_multiplier': 3
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    transformed_billing_data = _transform_billing_for_month_sms(get_monthly_billing_by_notification_type(
-        sample_service.id, APR_2016_MONTH_START, SMS_TYPE
-    ))
-
-    _assert_dict_equals(transformed_billing_data, {
-        'notification_type': SMS_TYPE,
-        'billing_units': 1324,
-        'month': 'April',
-        'rate': 0.12,
-    })
-
-
 def test_create_update_free_sms_fragment_limit_invalid_schema(client, sample_service):
 
     response = client.post('service/{}/billing/free-sms-fragment-limit'.format(sample_service.id),
@@ -480,21 +227,6 @@ def test_get_yearly_usage_by_monthly_from_ft_billing(client, notify_db_session):
     assert len(ft_email) == 0
 
 
-def test_compare_ft_billing_to_monthly_billing(client, notify_db_session):
-    service = set_up_yearly_data()
-
-    monthly_billing_response = client.get('/service/{}/billing/monthly-usage?year=2016'.format(service.id),
-                                          headers=[create_authorization_header()])
-
-    ft_billing_response = client.get('service/{}/billing/ft-monthly-usage?year=2016'.format(service.id),
-                                     headers=[('Content-Type', 'application/json'), create_authorization_header()])
-
-    monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
-    ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
-
-    assert monthly_billing_json_resp == ft_billing_json_resp
-
-
 def set_up_yearly_data():
     service = create_service()
     sms_template = create_template(service=service, template_type="sms")
@@ -527,55 +259,9 @@ def set_up_yearly_data():
                               notification_type='letter',
                               rate=0.33)
         start_date, end_date = get_month_start_and_end_date_in_utc(datetime(2016, int(mon), 1))
-        create_monthly_billing_entry(service=service, start_date=start_date,
-                                     end_date=end_date,
-                                     notification_type='sms',
-                                     monthly_totals=[
-                                         {"rate": 0.0162, "international": False,
-                                          "rate_multiplier": 1, "billing_units": int(d),
-                                          "total_cost": 0.0162 * int(d)},
-                                         {"rate": 0.0162, "international": False,
-                                          "rate_multiplier": 2, "billing_units": int(d),
-                                          "total_cost": 0.0162 * int(d)}]
-                                     )
-        create_monthly_billing_entry(service=service, start_date=start_date,
-                                     end_date=end_date,
-                                     notification_type='email',
-                                     monthly_totals=[
-                                         {"rate": 0, "international": False,
-                                          "rate_multiplier": 1, "billing_units": int(d),
-                                          "total_cost": 0}]
-                                     )
-        create_monthly_billing_entry(service=service, start_date=start_date,
-                                     end_date=end_date,
-                                     notification_type='letter',
-                                     monthly_totals=[
-                                         {"rate": 0.33, "international": False,
-                                          "rate_multiplier": 1, "billing_units": int(d),
-                                          "total_cost": 0.33 * int(d)}]
-                                     )
     return service
 
 
-def test_get_yearly_billing_usage_summary_from_ft_billing_compare_to_monthly_billing(
-        client, notify_db_session
-):
-    service = set_up_yearly_data()
-    monthly_billing_response = client.get('/service/{}/billing/yearly-usage-summary?year=2016'.format(service.id),
-                                          headers=[create_authorization_header()])
-
-    ft_billing_response = client.get('service/{}/billing/ft-yearly-usage-summary?year=2016'.format(service.id),
-                                     headers=[('Content-Type', 'application/json'), create_authorization_header()])
-
-    monthly_billing_json_resp = json.loads(monthly_billing_response.get_data(as_text=True))
-    ft_billing_json_resp = json.loads(ft_billing_response.get_data(as_text=True))
-
-    assert len(monthly_billing_json_resp) == 3
-    assert len(ft_billing_json_resp) == 3
-    for i in range(0, 3):
-        assert sorted(monthly_billing_json_resp[i]) == sorted(ft_billing_json_resp[i])
-
-
 def test_get_yearly_billing_usage_summary_from_ft_billing_returns_400_if_missing_year(client, sample_service):
     response = client.get(
         '/service/{}/billing/ft-yearly-usage-summary'.format(sample_service.id),
diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py
index 30b3308e8..35b4bf2e7 100644
--- a/tests/app/celery/test_scheduled_tasks.py
+++ b/tests/app/celery/test_scheduled_tasks.py
@@ -61,8 +61,7 @@ from app.utils import get_london_midnight_in_utc
 from app.celery.service_callback_tasks import create_delivery_status_callback_data
 from app.v2.errors import JobIncompleteError
 from tests.app.db import (
-    create_notification, create_service, create_template, create_job, create_rate,
-    create_service_callback_api
+    create_notification, create_service, create_template, create_job, create_service_callback_api
 )
 
 from tests.app.conftest import (
diff --git a/tests/app/dao/test_monthly_billing.py b/tests/app/dao/test_monthly_billing.py
deleted file mode 100644
index f422a0b76..000000000
--- a/tests/app/dao/test_monthly_billing.py
+++ /dev/null
@@ -1,521 +0,0 @@
-from datetime import datetime, timedelta
-from dateutil.relativedelta import relativedelta
-from freezegun import freeze_time
-from functools import partial
-
-from app.dao.monthly_billing_dao import (
-    create_or_update_monthly_billing,
-    get_monthly_billing_entry,
-    get_monthly_billing_by_notification_type,
-    get_service_ids_that_need_billing_populated,
-    get_billing_data_for_financial_year
-)
-from app.models import MonthlyBilling, SMS_TYPE, EMAIL_TYPE
-from tests.app.conftest import sample_letter_template
-from tests.app.db import (
-    create_notification,
-    create_rate,
-    create_service,
-    create_template,
-    create_monthly_billing_entry,
-)
-
-FEB_2016_MONTH_START = datetime(2016, 2, 1)
-FEB_2016_MONTH_END = datetime(2016, 2, 29, 23, 59, 59, 99999)
-
-MAR_2016_MONTH_START = datetime(2016, 3, 1)
-MAR_2016_MONTH_END = datetime(2016, 3, 31, 22, 59, 59, 99999)
-
-APR_2016_MONTH_START = datetime(2016, 3, 31, 23, 00, 00)
-APR_2016_MONTH_END = datetime(2016, 4, 30, 22, 59, 59, 99999)
-
-MAY_2016_MONTH_START = datetime(2016, 5, 31, 23, 00, 00)
-MAY_2016_MONTH_END = MAY_2016_MONTH_START + relativedelta(months=1, seconds=-1)
-
-APR_2017_MONTH_START = datetime(2017, 3, 31, 23, 00, 00)
-APR_2017_MONTH_END = datetime(2017, 4, 30, 23, 59, 59, 99999)
-
-JAN_2017_MONTH_START = datetime(2017, 1, 1)
-JAN_2017_MONTH_END = datetime(2017, 1, 31, 23, 59, 59, 99999)
-
-FEB_2017 = datetime(2017, 2, 15)
-APR_2016 = datetime(2016, 4, 10)
-
-NO_BILLING_DATA = {
-    "billing_units": 0,
-    "rate_multiplier": 1,
-    "international": False,
-    "rate": 0,
-    "total_cost": 0
-}
-
-
-def _assert_monthly_billing(monthly_billing, service_id, notification_type, month_start, month_end):
-    assert monthly_billing.service_id == service_id
-    assert monthly_billing.notification_type == notification_type
-    assert monthly_billing.start_date == month_start
-    assert monthly_billing.end_date == month_end
-
-
-def _assert_monthly_billing_totals(monthly_billing_totals, expected_dict):
-    assert sorted(monthly_billing_totals.keys()) == sorted(expected_dict.keys())
-    assert sorted(monthly_billing_totals.values()) == sorted(expected_dict.values())
-
-
-def test_get_monthly_billing_by_notification_type_returns_correct_totals(notify_db, notify_db_session):
-    service = create_service(service_name="Service One")
-
-    create_monthly_billing_entry(
-        service=service,
-        monthly_totals=[{
-            "billing_units": 12,
-            "rate": 0.0158,
-            "rate_multiplier": 5,
-            "total_cost": 2.1804,
-            "international": False
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, SMS_TYPE)
-
-    _assert_monthly_billing(
-        monthly_billing_data, service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing_data.monthly_totals[0], {
-        "billing_units": 12,
-        "rate_multiplier": 5,
-        "international": False,
-        "rate": 0.0158,
-        "total_cost": 2.1804
-    })
-
-
-def test_get_monthly_billing_by_notification_type_filters_by_type(notify_db, notify_db_session):
-    service = create_service(service_name="Service One")
-
-    create_monthly_billing_entry(
-        service=service,
-        monthly_totals=[{
-            "billing_units": 138,
-            "rate": 0.0158,
-            "rate_multiplier": 1,
-            "total_cost": 2.1804,
-            "international": None
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    create_monthly_billing_entry(
-        service=service,
-        monthly_totals=[],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=EMAIL_TYPE
-    )
-
-    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016, EMAIL_TYPE)
-
-    _assert_monthly_billing(
-        monthly_billing_data, service.id, 'email', APR_2016_MONTH_START, APR_2016_MONTH_END
-    )
-    assert monthly_billing_data.monthly_totals == []
-
-
-def test_get_monthly_billing_by_notification_type_normalises_start_date(notify_db, notify_db_session):
-    service = create_service(service_name="Service One")
-
-    create_monthly_billing_entry(
-        service=service,
-        monthly_totals=[{
-            "billing_units": 321,
-            "rate": 0.0158,
-            "rate_multiplier": 1,
-            "total_cost": 2.1804,
-            "international": None
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    monthly_billing_data = get_monthly_billing_by_notification_type(service.id, APR_2016 + timedelta(days=5), SMS_TYPE)
-
-    assert monthly_billing_data.start_date == APR_2016_MONTH_START
-    assert monthly_billing_data.monthly_totals[0]['billing_units'] == 321
-
-
-def test_add_monthly_billing_for_single_month_populates_correctly(
-    sample_template, sample_email_template
-):
-    create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE)
-    letter_template = sample_letter_template(sample_template.service)
-    create_notification(
-        template=sample_template, created_at=JAN_2017_MONTH_START,
-        billable_units=1, rate_multiplier=2, status='delivered'
-    )
-    create_notification(template=sample_email_template, created_at=JAN_2017_MONTH_START,
-                        status='delivered')
-    create_notification(template=letter_template, created_at=JAN_2017_MONTH_START, status='delivered')
-
-    create_or_update_monthly_billing(
-        service_id=sample_template.service_id,
-        billing_month=JAN_2017_MONTH_START
-    )
-
-    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
-
-    assert len(monthly_billing) == 3
-    _assert_monthly_billing(
-        monthly_billing[0], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(
-        monthly_billing[0].monthly_totals[0], {
-            "billing_units": 1,
-            "rate_multiplier": 1,
-            "international": False,
-            "rate": 0.0,
-            "total_cost": 0
-        })
-
-    _assert_monthly_billing(
-        monthly_billing[1], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 2,
-        "international": False,
-        "rate": 0.0158,
-        "total_cost": 1 * 2 * 0.0158
-    })
-
-    _assert_monthly_billing(
-        monthly_billing[2], sample_template.service.id, 'letter', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.33,
-        "total_cost": 1 * 0.33
-    })
-
-
-def test_add_monthly_billing_for_multiple_months_populate_correctly(
-    sample_template
-):
-    create_rate(start_date=FEB_2016_MONTH_START - timedelta(days=1), value=0.12, notification_type=SMS_TYPE)
-    create_notification(
-        template=sample_template, created_at=FEB_2016_MONTH_START,
-        billable_units=1, rate_multiplier=2, status='delivered'
-    )
-    create_notification(
-        template=sample_template, created_at=MAR_2016_MONTH_START,
-        billable_units=2, rate_multiplier=3, status='delivered'
-    )
-
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=FEB_2016_MONTH_START)
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=MAR_2016_MONTH_START)
-
-    monthly_billing = MonthlyBilling.query.order_by(
-        MonthlyBilling.notification_type,
-        MonthlyBilling.start_date
-    ).all()
-
-    assert len(monthly_billing) == 6
-    _assert_monthly_billing(
-        monthly_billing[0], sample_template.service.id, 'email', FEB_2016_MONTH_START, FEB_2016_MONTH_END
-    )
-    assert monthly_billing[0].monthly_totals == []
-
-    _assert_monthly_billing(
-        monthly_billing[1], sample_template.service.id, 'email', MAR_2016_MONTH_START, MAR_2016_MONTH_END
-    )
-    assert monthly_billing[1].monthly_totals == []
-
-    _assert_monthly_billing(
-        monthly_billing[2], sample_template.service.id, 'sms', FEB_2016_MONTH_START, FEB_2016_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 2,
-        "international": False,
-        "rate": 0.12,
-        "total_cost": 0.24
-    })
-
-    _assert_monthly_billing(
-        monthly_billing[3], sample_template.service.id, 'sms', MAR_2016_MONTH_START, MAR_2016_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[3].monthly_totals[0], {
-        "billing_units": 2,
-        "rate_multiplier": 3,
-        "international": False,
-        "rate": 0.12,
-        "total_cost": 0.72
-    })
-
-    _assert_monthly_billing(
-        monthly_billing[4], sample_template.service.id, 'letter', FEB_2016_MONTH_START, FEB_2016_MONTH_END
-    )
-    assert monthly_billing[4].monthly_totals == []
-
-    _assert_monthly_billing(
-        monthly_billing[5], sample_template.service.id, 'letter', MAR_2016_MONTH_START, MAR_2016_MONTH_END
-    )
-    assert monthly_billing[5].monthly_totals == []
-
-
-def test_add_monthly_billing_with_multiple_rates_populate_correctly(
-    sample_template, sample_email_template
-):
-    letter_template = sample_letter_template(sample_template.service)
-    create_rate(start_date=JAN_2017_MONTH_START, value=0.0158, notification_type=SMS_TYPE)
-    create_rate(start_date=JAN_2017_MONTH_START + timedelta(days=5), value=0.123, notification_type=SMS_TYPE)
-    create_notification(template=sample_template, created_at=JAN_2017_MONTH_START, billable_units=1, status='delivered')
-    create_notification(
-        template=sample_template, created_at=JAN_2017_MONTH_START + timedelta(days=6),
-        billable_units=2, status='delivered'
-    )
-
-    create_notification(template=sample_email_template, created_at=JAN_2017_MONTH_START, status='delivered')
-    create_notification(template=letter_template, created_at=JAN_2017_MONTH_START, status='delivered',
-                        billable_units=1)
-
-    create_or_update_monthly_billing(service_id=sample_template.service_id, billing_month=JAN_2017_MONTH_START)
-
-    monthly_billing = MonthlyBilling.query.order_by(MonthlyBilling.notification_type).all()
-
-    assert len(monthly_billing) == 3
-    _assert_monthly_billing(
-        monthly_billing[0], sample_template.service.id, 'email', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[0].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.0,
-        "total_cost": 0.0
-    })
-
-    _assert_monthly_billing(
-        monthly_billing[1], sample_template.service.id, 'sms', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.0158,
-        "total_cost": 0.0158
-    })
-    _assert_monthly_billing_totals(monthly_billing[1].monthly_totals[1], {
-        "billing_units": 2,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.123,
-        "total_cost": 0.246
-    })
-
-    _assert_monthly_billing(
-        monthly_billing[2], sample_template.service.id, 'letter', JAN_2017_MONTH_START, JAN_2017_MONTH_END
-    )
-    _assert_monthly_billing_totals(monthly_billing[2].monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.33,
-        "total_cost": 0.33
-    })
-
-
-def test_update_monthly_billing_overwrites_old_totals(sample_template):
-    create_rate(APR_2016_MONTH_START, 0.123, SMS_TYPE)
-    create_notification(template=sample_template, created_at=APR_2016_MONTH_START, billable_units=1, status='delivered')
-
-    create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END)
-    first_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE)
-
-    _assert_monthly_billing(
-        first_update, sample_template.service.id, 'sms', APR_2016_MONTH_START, APR_2016_MONTH_END
-    )
-    _assert_monthly_billing_totals(first_update.monthly_totals[0], {
-        "billing_units": 1,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.123,
-        "total_cost": 0.123
-    })
-
-    first_updated_at = first_update.updated_at
-
-    with freeze_time(APR_2016_MONTH_START + timedelta(days=3)):
-        create_notification(template=sample_template, billable_units=2, status='delivered')
-        create_or_update_monthly_billing(sample_template.service_id, APR_2016_MONTH_END)
-
-    second_update = get_monthly_billing_by_notification_type(sample_template.service_id, APR_2016_MONTH_START, SMS_TYPE)
-
-    _assert_monthly_billing_totals(second_update.monthly_totals[0], {
-        "billing_units": 3,
-        "rate_multiplier": 1,
-        "international": False,
-        "rate": 0.123,
-        "total_cost": 0.369
-    })
-
-    assert second_update.updated_at == APR_2016_MONTH_START + timedelta(days=3)
-    assert first_updated_at != second_update.updated_at
-
-
-def test_get_service_ids_that_need_billing_populated_return_correctly(notify_db_session):
-    service_1 = create_service(service_name="Service One")
-    template_1 = create_template(service=service_1)
-    service_2 = create_service(service_name="Service Two")
-    template_2 = create_template(service=service_2)
-    create_notification(template=template_1, created_at=datetime(2017, 6, 30, 13, 30), status='delivered')
-    create_notification(template=template_1, created_at=datetime(2017, 7, 1, 14, 30), status='delivered')
-    create_notification(template=template_2, created_at=datetime(2017, 7, 15, 13, 30))
-    create_notification(template=template_2, created_at=datetime(2017, 7, 31, 13, 30))
-    services = get_service_ids_that_need_billing_populated(
-        start_date=datetime(2017, 7, 1), end_date=datetime(2017, 7, 16)
-    )
-    expected_services = [service_1.id, service_2.id]
-    assert sorted([x.service_id for x in services]) == sorted(expected_services)
-
-
-def test_get_monthly_billing_entry_filters_by_service(notify_db, notify_db_session):
-    service_1 = create_service(service_name="Service One")
-    service_2 = create_service(service_name="Service Two")
-    now = datetime.utcnow()
-
-    create_monthly_billing_entry(
-        service=service_1,
-        monthly_totals=[],
-        start_date=now,
-        end_date=now + timedelta(days=30),
-        notification_type=SMS_TYPE
-    )
-
-    create_monthly_billing_entry(
-        service=service_2,
-        monthly_totals=[],
-        start_date=now,
-        end_date=now + timedelta(days=30),
-        notification_type=SMS_TYPE
-    )
-
-    entry = get_monthly_billing_entry(service_2.id, now, SMS_TYPE)
-
-    assert entry.start_date == now
-    assert entry.service_id == service_2.id
-
-
-def test_get_yearly_billing_data_for_year_returns_within_year_only(
-    sample_template
-):
-    monthly_billing_entry = partial(
-        create_monthly_billing_entry, service=sample_template.service, notification_type=SMS_TYPE
-    )
-    monthly_billing_entry(start_date=FEB_2016_MONTH_START, end_date=FEB_2016_MONTH_END)
-    monthly_billing_entry(
-        monthly_totals=[{
-            "billing_units": 138,
-            "rate": 0.0158,
-            "rate_multiplier": 1,
-            "total_cost": 2.1804,
-            "international": None
-        }],
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-    monthly_billing_entry(start_date=APR_2017_MONTH_START, end_date=APR_2017_MONTH_END)
-
-    billing_data = get_billing_data_for_financial_year(sample_template.service.id, 2016, [SMS_TYPE])
-
-    assert len(billing_data) == 1
-    assert billing_data[0].monthly_totals[0]['billing_units'] == 138
-
-
-def test_get_yearly_billing_data_for_year_returns_multiple_notification_types(sample_template):
-    monthly_billing_entry = partial(
-        create_monthly_billing_entry, service=sample_template.service,
-        start_date=APR_2016_MONTH_START, end_date=APR_2016_MONTH_END
-    )
-
-    monthly_billing_entry(
-        notification_type=SMS_TYPE, monthly_totals=[]
-    )
-    monthly_billing_entry(
-        notification_type=EMAIL_TYPE,
-        monthly_totals=[{
-            "billing_units": 2,
-            "rate": 1.3,
-            "rate_multiplier": 3,
-            "total_cost": 2.1804,
-            "international": False
-        }]
-    )
-
-    billing_data = get_billing_data_for_financial_year(
-        service_id=sample_template.service.id,
-        year=2016,
-        notification_types=[SMS_TYPE, EMAIL_TYPE]
-    )
-
-    assert len(billing_data) == 2
-
-    assert billing_data[0].notification_type == EMAIL_TYPE
-    assert billing_data[0].monthly_totals[0]['billing_units'] == 2
-    assert billing_data[1].notification_type == SMS_TYPE
-
-
-@freeze_time("2016-04-21 11:00:00")
-def test_get_yearly_billing_data_for_year_includes_current_day_totals(sample_template):
-    create_rate(start_date=FEB_2016_MONTH_START, value=0.0158, notification_type=SMS_TYPE)
-
-    create_monthly_billing_entry(
-        service=sample_template.service,
-        start_date=APR_2016_MONTH_START,
-        end_date=APR_2016_MONTH_END,
-        notification_type=SMS_TYPE
-    )
-
-    billing_data = get_billing_data_for_financial_year(
-        service_id=sample_template.service.id,
-        year=2016,
-        notification_types=[SMS_TYPE]
-    )
-
-    assert len(billing_data) == 1
-    assert billing_data[0].notification_type == SMS_TYPE
-    assert billing_data[0].monthly_totals == []
-
-    create_notification(
-        template=sample_template,
-        created_at=datetime.utcnow(),
-        sent_at=datetime.utcnow(),
-        status='sending',
-        billable_units=3
-    )
-
-    billing_data = get_billing_data_for_financial_year(
-        service_id=sample_template.service.id,
-        year=2016,
-        notification_types=[SMS_TYPE]
-    )
-
-    assert billing_data[0].monthly_totals[0]['billing_units'] == 3
-
-
-@freeze_time("2017-06-16 13:00:00")
-def test_get_billing_data_for_financial_year_updated_monthly_billing_if_today_is_in_current_year(
-        sample_service,
-        mocker
-):
-    mock = mocker.patch("app.dao.monthly_billing_dao.create_or_update_monthly_billing")
-    get_billing_data_for_financial_year(sample_service.id, 2016)
-    mock.assert_not_called()
diff --git a/tests/app/db.py b/tests/app/db.py
index b42203a6b..4e275155d 100644
--- a/tests/app/db.py
+++ b/tests/app/db.py
@@ -14,7 +14,6 @@ from app.models import (
     InboundSms,
     InboundNumber,
     Job,
-    MonthlyBilling,
     Notification,
     EmailBranding,
     Organisation,
@@ -382,27 +381,6 @@ def create_inbound_number(number, provider='mmg', active=True, service_id=None):
     return inbound_number
 
 
-def create_monthly_billing_entry(
-        service,
-        start_date,
-        end_date,
-        notification_type,
-        monthly_totals=[]
-):
-    entry = MonthlyBilling(
-        service_id=service.id,
-        notification_type=notification_type,
-        monthly_totals=monthly_totals,
-        start_date=start_date,
-        end_date=end_date
-    )
-
-    db.session.add(entry)
-    db.session.commit()
-
-    return entry
-
-
 def create_reply_to_email(
         service,
         email_address,

From 4e49df2479cbc1112d496e973d7856764a187152 Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Mon, 23 Jul 2018 17:11:40 +0100
Subject: [PATCH 10/13] Delete MonthlyBilling model

---
 app/models.py | 29 -----------------------------
 1 file changed, 29 deletions(-)

diff --git a/app/models.py b/app/models.py
index bff4a664a..f5bf62087 100644
--- a/app/models.py
+++ b/app/models.py
@@ -1643,35 +1643,6 @@ class LetterRate(db.Model):
     post_class = db.Column(db.String, nullable=False)
 
 
-class MonthlyBilling(db.Model):
-    __tablename__ = 'monthly_billing'
-
-    id = db.Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
-    service_id = db.Column(UUID(as_uuid=True), db.ForeignKey('services.id'), index=True, nullable=False)
-    service = db.relationship('Service', backref='monthly_billing')
-    start_date = db.Column(db.DateTime, nullable=False)
-    end_date = db.Column(db.DateTime, nullable=False)
-    notification_type = db.Column(notification_types, nullable=False)
-    monthly_totals = db.Column(JSON, nullable=False)
-    updated_at = db.Column(db.DateTime, nullable=False, default=datetime.datetime.utcnow)
-
-    __table_args__ = (
-        UniqueConstraint('service_id', 'start_date', 'notification_type', name='uix_monthly_billing'),
-    )
-
-    def serialized(self):
-        return {
-            "start_date": self.start_date,
-            "end_date": self.end_date,
-            "service_id": str(self.service_id),
-            "notification_type": self.notification_type,
-            "monthly_totals": self.monthly_totals
-        }
-
-    def __repr__(self):
-        return str(self.serialized())
-
-
 class ServiceEmailReplyTo(db.Model):
     __tablename__ = "service_email_reply_to"
 

From b28ec8beda342d6227ebdc79579c22c535620d01 Mon Sep 17 00:00:00 2001
From: Pea Tyczynska <pea.tyczynska@digital.cabinet-office.gov.uk>
Date: Fri, 27 Jul 2018 14:41:53 +0100
Subject: [PATCH 11/13] Use old monthly_billing-related routes for new
 functions

---
 app/billing/rest.py | 2 ++
 app/commands.py     | 8 ++++----
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/app/billing/rest.py b/app/billing/rest.py
index ffdc7a068..418bb39d5 100644
--- a/app/billing/rest.py
+++ b/app/billing/rest.py
@@ -33,6 +33,7 @@ register_errors(billing_blueprint)
 
 
 @billing_blueprint.route('/ft-monthly-usage')
+@billing_blueprint.route('/monthly-usage')
 def get_yearly_usage_by_monthly_from_ft_billing(service_id):
     try:
         year = int(request.args.get('year'))
@@ -44,6 +45,7 @@ def get_yearly_usage_by_monthly_from_ft_billing(service_id):
 
 
 @billing_blueprint.route('/ft-yearly-usage-summary')
+@billing_blueprint.route('/yearly-usage-summary')
 def get_yearly_billing_usage_summary_from_ft_billing(service_id):
     try:
         year = int(request.args.get('year'))
diff --git a/app/commands.py b/app/commands.py
index 530fedd77..cd61bdbc7 100644
--- a/app/commands.py
+++ b/app/commands.py
@@ -17,11 +17,11 @@ from app.celery.scheduled_tasks import send_total_sent_notifications_to_performa
 from app.celery.service_callback_tasks import send_delivery_status_to_service
 from app.celery.letters_pdf_tasks import create_letters_pdf
 from app.config import QueueNames
-from app.dao.date_util import get_financial_year
 from app.dao.fact_billing_dao import (
+    delete_billing_data_for_service_for_day,
     fetch_billing_data_for_day,
+    get_service_ids_that_need_billing_populated,
     update_fact_billing,
-    delete_billing_data_for_service_for_day
 )
 
 from app.dao.provider_rates_dao import create_provider_rates as dao_create_provider_rates
@@ -31,9 +31,9 @@ from app.dao.services_dao import (
     dao_fetch_all_services_by_user,
     dao_fetch_service_by_id
 )
-from app.dao.users_dao import (delete_model_user, delete_user_verify_codes)
+from app.dao.users_dao import delete_model_user, delete_user_verify_codes
 from app.models import PROVIDERS, User, Notification
-from app.performance_platform.processing_time import (send_processing_time_for_start_and_end)
+from app.performance_platform.processing_time import send_processing_time_for_start_and_end
 from app.utils import (
     cache_key_for_service_template_usage_per_day,
     get_london_midnight_in_utc,

From b8913b62ece77dbf878efb14001f882a3f72dc97 Mon Sep 17 00:00:00 2001
From: Chris Hill-Scott <me@quis.cc>
Date: Mon, 30 Jul 2018 15:25:48 +0100
Subject: [PATCH 12/13] Fix broken Markdown headings in README

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 810718d97..dd0c7fab2 100644
--- a/README.md
+++ b/README.md
@@ -16,13 +16,13 @@ To run the API you will need appropriate AWS credentials. You should receive the
 
 Your aws credentials should be stored in a folder located at `~/.aws`. Follow [Amazon's instructions](http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html#cli-config-files) for storing them correctly.
 
-### Virtualenv
+### Virtualenv
 
 ```
 mkvirtualenv -p /usr/local/bin/python3 notifications-api
 ```
 
-### `environment.sh`
+### `environment.sh`
 
 Creating the environment.sh file. Replace [unique-to-environment] with your something unique to the environment. Your AWS credentials should be set up for notify-tools (the development/CI AWS account).
 

From ce5bb1f7627dc2b725364615244e10f263ae5af3 Mon Sep 17 00:00:00 2001
From: Alexey Bezhan <al@fueledbylemons.com>
Date: Mon, 30 Jul 2018 16:22:20 +0100
Subject: [PATCH 13/13] Make pyup ignore requirements.txt

We don't want pyup.io upgrading sub-dependencies listed in the
requirements.txt file since it does it whenever a new version is
available regardless of what our application dependencies require.
---
 Makefile         | 3 ++-
 requirements.txt | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 5954b4705..3612540bc 100644
--- a/Makefile
+++ b/Makefile
@@ -98,7 +98,8 @@ freeze-requirements:
 	rm -rf venv-freeze
 	virtualenv -p python3 venv-freeze
 	$$(pwd)/venv-freeze/bin/pip install -r requirements-app.txt
-	echo '# This file is autogenerated. Do not edit it manually.' > requirements.txt
+	echo '# pyup: ignore file' > requirements.txt
+	echo '# This file is autogenerated. Do not edit it manually.' >> requirements.txt
 	cat requirements-app.txt >> requirements.txt
 	echo '' >> requirements.txt
 	$$(pwd)/venv-freeze/bin/pip freeze -r <(sed '/^--/d' requirements-app.txt) | sed -n '/The following requirements were added by pip freeze/,$$p' >> requirements.txt
diff --git a/requirements.txt b/requirements.txt
index ce2c67227..2dcc668b7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+# pyup: ignore file
 # This file is autogenerated. Do not edit it manually.
 # Run `make freeze-requirements` to update requirements.txt
 # with package version changes made in requirements-app.txt