move monthly notification stats from notification history to ft tables

it now uses the ft_notification_status table - nice and quick. if you
ask for the current tax year it'll top up the data with numbers from
the notification table.

the data is in exactly the same shape as the old endpoint - no changes
to the admin app are necessary

app/dao/fact_notification_status_dao.py

@@ -6,7 +6,7 @@ from sqlalchemy.dialects.postgresql import insert
 
 from app import db
 from app.models import Notification, NotificationHistory, FactNotificationStatus
-from app.utils import convert_bst_to_utc
+from app.utils import convert_bst_to_utc, get_london_midnight_in_utc, get_london_month_from_utc_column
 
 
 def fetch_notification_status_for_day(process_day, service_id=None):
@@ -73,3 +73,41 @@ def update_fact_notification_status(data, process_day):
         )
         db.session.connection().execute(stmt)
         db.session.commit()
+
+
+def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):
+    return db.session.query(
+        func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status,
+        func.sum(FactNotificationStatus.notification_count).label('count')
+    ).filter(
+        FactNotificationStatus.service_id == service_id,
+        FactNotificationStatus.bst_date >= start_date,
+        FactNotificationStatus.bst_date < end_date,
+    ).group_by(
+        func.date_trunc('month', FactNotificationStatus.bst_date).label('month'),
+        FactNotificationStatus.notification_type,
+        FactNotificationStatus.notification_status
+    ).all()
+
+
+def fetch_notification_status_for_service_for_day(bst_day, service_id):
+    return db.session.query(
+        # return current month as a datetime so the data has the same shape as the ft_notification_status query
+        get_london_month_from_utc_column(func.current_date()),
+        Notification.notification_type,
+        Notification.status,
+        func.count()
+    ).filter(
+        Notification.created_at >= get_london_midnight_in_utc(bst_day),
+        Notification.created_at < get_london_midnight_in_utc(bst_day + timedelta(days=1)),
+        Notification.service_id == service_id
+    ).group_by(
+        Notification.template_id,
+        Notification.service_id,
+        'job_id',
+        Notification.notification_type,
+        Notification.key_type,
+        Notification.status
+    ).all()

app/service/rest.py

@@ -12,11 +12,16 @@ from sqlalchemy.orm.exc import NoResultFound
 
 from app.dao import notifications_dao
 from app.dao.dao_utils import dao_rollback
+from app.dao.date_util import get_financial_year
 from app.dao.api_key_dao import (
     save_model_api_key,
     get_model_api_keys,
     get_unsigned_secret,
     expire_api_key)
+from app.dao.fact_notification_status_dao import (
+    fetch_notification_status_for_service_by_month,
+    fetch_notification_status_for_service_for_day
+)
 from app.dao.inbound_numbers_dao import dao_allocate_number_for_service
 from app.dao.organisation_dao import dao_get_organisation_by_service_id
 from app.dao.service_sms_sender_dao import (
@@ -33,7 +38,6 @@ from app.dao.services_dao import (
     dao_create_service,
     dao_fetch_all_services,
     dao_fetch_all_services_by_user,
-    dao_fetch_monthly_historical_stats_for_service,
     dao_fetch_monthly_historical_usage_by_template_for_service,
     dao_fetch_service_by_id,
     dao_fetch_stats_for_service,
@@ -89,7 +93,7 @@ from app.schemas import (
     notifications_filter_schema,
     detailed_service_schema
 )
-from app.utils import pagination_links
+from app.utils import pagination_links, convert_utc_to_bst
 
 service_blueprint = Blueprint('service', __name__)
 
@@ -397,15 +401,28 @@ def search_for_notification_by_to_field(service_id, search_term, statuses, notif
 
 @service_blueprint.route('/<uuid:service_id>/notifications/monthly', methods=['GET'])
 def get_monthly_notification_stats(service_id):
-    service = dao_fetch_service_by_id(service_id)
+    # check service_id validity
+    dao_fetch_service_by_id(service_id)
+
     try:
-        return jsonify(data=dao_fetch_monthly_historical_stats_for_service(
-            service.id,
-            int(request.args.get('year', 'NaN'))
-        ))
+        year = int(request.args.get('year', 'NaN'))
     except ValueError:
         raise InvalidRequest('Year must be a number', status_code=400)
 
+    start_date, end_date = get_financial_year(year)
+
+    data = statistics.create_empty_monthly_notification_status_stats_dict(year)
+
+    stats = fetch_notification_status_for_service_by_month(start_date, end_date, service_id)
+    statistics.add_monthly_notification_status_stats(data, stats)
+
+    now = datetime.utcnow()
+    if end_date > now:
+        todays_deltas = fetch_notification_status_for_service_for_day(convert_utc_to_bst(now), service_id=service_id)
+        statistics.add_monthly_notification_status_stats(data, todays_deltas)
+
+    return jsonify(data=data)
+
 
 def get_detailed_service(service_id, today_only=False):
     service = dao_fetch_service_by_id(service_id)

app/service/statistics.py

@@ -1,6 +1,9 @@
+from collections import defaultdict
 from datetime import datetime
 
 from app.models import NOTIFICATION_STATUS_TYPES, TEMPLATE_TYPES
+from app.utils import convert_utc_to_bst
+from app.dao.date_util import get_months_for_financial_year
 
 
 def format_statistics(statistics):
@@ -82,3 +85,24 @@ def _update_statuses_from_row(update_dict, row):
         update_dict['delivered'] += row.count
     elif row.status in ('failed', 'technical-failure', 'temporary-failure', 'permanent-failure'):
         update_dict['failed'] += row.count
+
+
+def create_empty_monthly_notification_status_stats_dict(year):
+    utc_month_starts = get_months_for_financial_year(year)
+    # nested dicts - data[month][template type][status] = count
+    return {
+        convert_utc_to_bst(start).strftime('%Y-%m'): {
+            template_type: defaultdict(int)
+            for template_type in TEMPLATE_TYPES
+        }
+        for start in utc_month_starts
+    }
+
+
+def add_monthly_notification_status_stats(data, stats):
+    for row in stats:
+        month = row.month.strftime('%Y-%m')
+
+        data[month][row.notification_type][row.notification_status] = row.count
+
+    return data