Merge pull request #807 from alphagov/monthly-service-usage

Add endpoint for breakdown of activity by month

app/dao/notifications_dao.py

@@ -27,6 +27,7 @@ from app.models import (
 
 from app.dao.dao_utils import transactional
 from app.statsd_decorators import statsd
+from app.utils import get_london_month_from_utc_column
 
 
 def dao_get_notification_statistics_for_service_and_day(service_id, day):
@@ -225,24 +226,15 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page
 def get_notification_billable_unit_count_per_month(service_id, year):
     start, end = get_financial_year(year)
 
-    """
-     The query needs to sum the billable_units per month, but this needs to be the month in BST (British Standard Time).
-     The database stores all timestamps as UTC without the timezone.
-      - First set the timezone on created_at to UTC
-      - then convert the timezone to BST (or Europe/London)
-      - lastly truncate the datetime to month to group the sum of the billable_units
-    """
-    month = func.date_trunc("month",
-                            func.timezone("Europe/London", func.timezone("UTC",
-                                                                         NotificationHistory.created_at)))
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+
     notifications = db.session.query(
         month,
         func.sum(NotificationHistory.billable_units)
     ).filter(
         NotificationHistory.billable_units != 0,
         NotificationHistory.service_id == service_id,
-        NotificationHistory.created_at >= start,
-        NotificationHistory.created_at < end
+        NotificationHistory.created_at.between(*get_financial_year(year)),
     ).group_by(
         month
     ).order_by(

app/dao/services_dao.py

@@ -9,6 +9,7 @@ from app.dao.dao_utils import (
     transactional,
     version_class
 )
+from app.dao.notifications_dao import get_financial_year
 from app.models import (
     NotificationStatistics,
     TemplateStatistics,
@@ -24,8 +25,12 @@ from app.models import (
     User,
     InvitedUser,
     Service,
-    KEY_TYPE_TEST)
+    KEY_TYPE_TEST,
+    NOTIFICATION_STATUS_TYPES,
+    TEMPLATE_TYPES,
+)
 from app.statsd_decorators import statsd
+from app.utils import get_london_month_from_utc_column
 
 
 def dao_fetch_all_services(only_active=False):
@@ -238,6 +243,50 @@ def dao_fetch_weekly_historical_stats_for_service(service_id):
     ).all()
 
 
+@statsd(namespace="dao")
+def dao_fetch_monthly_historical_stats_for_service(service_id, year):
+    monday_of_notification_week = func.date_trunc('week', NotificationHistory.created_at).label('week_start')
+    start, end = get_financial_year(year)
+
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+
+    rows = db.session.query(
+        NotificationHistory.notification_type,
+        NotificationHistory.status,
+        month,
+        func.count(NotificationHistory.id).label('count')
+    ).filter(
+        NotificationHistory.service_id == service_id,
+        NotificationHistory.created_at.between(*get_financial_year(year)),
+    ).group_by(
+        NotificationHistory.notification_type,
+        NotificationHistory.status,
+        month
+    ).order_by(
+        month
+    )
+
+    months = {
+        datetime.strftime(date, '%Y-%m'): dict.fromkeys(
+            TEMPLATE_TYPES,
+            dict.fromkeys(
+                NOTIFICATION_STATUS_TYPES,
+                0
+            )
+        )
+        for date in [
+            datetime(year, month, 1) for month in range(4, 13)
+        ] + [
+            datetime(year + 1, month, 1) for month in range(1, 4)
+        ]
+    }
+
+    for notification_type, status, date, count in rows:
+        months[datetime.strftime(date, "%Y-%m")][notification_type][status] = count
+
+    return months
+
+
 @statsd(namespace='dao')
 def dao_fetch_todays_stats_for_all_services(include_from_test_key=True):
     query = db.session.query(

app/service/rest.py

@@ -30,7 +30,9 @@ from app.dao.services_dao import (
     dao_archive_service,
     fetch_stats_by_date_range_for_all_services,
     dao_suspend_service,
-    dao_resume_service)
+    dao_resume_service,
+    dao_fetch_monthly_historical_stats_for_service,
+)
 from app.dao.service_whitelist_dao import (
     dao_fetch_service_whitelist,
     dao_add_and_commit_whitelisted_contacts,
@@ -273,6 +275,18 @@ def get_weekly_notification_stats(service_id):
     return jsonify(data={week.date().isoformat(): statistics for week, statistics in stats.items()})
 
 
+@service_blueprint.route('/<uuid:service_id>/notifications/monthly', methods=['GET'])
+def get_monthly_notification_stats(service_id):
+    service = dao_fetch_service_by_id(service_id)
+    try:
+        return jsonify(data=dao_fetch_monthly_historical_stats_for_service(
+            service.id,
+            int(request.args.get('year', 'NaN'))
+        ))
+    except ValueError:
+        raise InvalidRequest('Year must be a number', status_code=400)
+
+
 def get_detailed_service(service_id, today_only=False):
     service = dao_fetch_service_by_id(service_id)
     stats_fn = dao_fetch_todays_stats_for_service if today_only else dao_fetch_stats_for_service

app/utils.py

@@ -2,6 +2,7 @@ from datetime import datetime, timedelta
 
 import pytz
 from flask import url_for
+from sqlalchemy import func
 from notifications_utils.template import SMSMessageTemplate, PlainTextEmailTemplate
 
 
@@ -46,3 +47,19 @@ def get_london_midnight_in_utc(date):
 def get_midnight_for_day_before(date):
     day_before = date - timedelta(1)
     return get_london_midnight_in_utc(day_before)
+
+
+def get_london_month_from_utc_column(column):
+    """
+     Where queries need to count notifications by month it needs to be
+     the month in BST (British Summer Time).
+     The database stores all timestamps as UTC without the timezone.
+      - First set the timezone on created_at to UTC
+      - then convert the timezone to BST (or Europe/London)
+      - lastly truncate the datetime to month with which we can group
+        queries
+    """
+    return func.date_trunc(
+        "month",
+        func.timezone("Europe/London", func.timezone("UTC", column))
+    )

tests/app/dao/test_services_dao.py

@@ -20,6 +20,7 @@ from app.dao.services_dao import (
     dao_fetch_stats_for_service,
     dao_fetch_todays_stats_for_service,
     dao_fetch_weekly_historical_stats_for_service,
+    dao_fetch_monthly_historical_stats_for_service,
     fetch_todays_total_message_count,
     dao_fetch_todays_stats_for_all_services,
     fetch_stats_by_date_range_for_all_services,
@@ -507,6 +508,50 @@ def test_fetch_weekly_historical_stats_separates_weeks(notify_db, notify_db_sess
     assert ret[-1].count == 1
 
 
+def test_fetch_monthly_historical_stats_separates_weeks(notify_db, notify_db_session, sample_template):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        sample_template
+    )
+    _before_start_of_financial_year = notification_history(created_at=datetime(2016, 3, 31))
+    start_of_financial_year = notification_history(created_at=datetime(2016, 4, 1))
+    start_of_summer = notification_history(created_at=datetime(2016, 6, 20))
+    start_of_autumn = notification_history(created_at=datetime(2016, 9, 30, 23, 30, 0))  # October because BST
+    start_of_winter = notification_history(created_at=datetime(2016, 12, 1), status='delivered')
+    start_of_spring = notification_history(created_at=datetime(2017, 3, 11))
+    end_of_financial_year = notification_history(created_at=datetime(2017, 3, 31))
+    _after_end_of_financial_year = notification_history(created_at=datetime(2017, 3, 31, 23, 30))  # after because BST
+
+    result = dao_fetch_monthly_historical_stats_for_service(sample_template.service_id, 2016)
+
+    assert result['2016-04']['sms']['created'] == 1
+    assert result['2016-04']['sms']['sending'] == 0
+    assert result['2016-04']['sms']['delivered'] == 0
+    assert result['2016-04']['sms']['pending'] == 0
+    assert result['2016-04']['sms']['failed'] == 0
+    assert result['2016-04']['sms']['technical-failure'] == 0
+    assert result['2016-04']['sms']['temporary-failure'] == 0
+    assert result['2016-04']['sms']['permanent-failure'] == 0
+
+    assert result['2016-06']['sms']['created'] == 1
+
+    assert result['2016-10']['sms']['created'] == 1
+
+    assert result['2016-12']['sms']['created'] == 0
+    assert result['2016-12']['sms']['delivered'] == 1
+
+    assert result['2017-03']['sms']['created'] == 2
+
+    assert result.keys() == {
+        '2016-04', '2016-05', '2016-06',
+        '2016-07', '2016-08', '2016-09',
+        '2016-10', '2016-11', '2016-12',
+        '2017-01', '2017-02', '2017-03',
+    }
+
+
 def test_fetch_weekly_historical_stats_ignores_second_service(notify_db, notify_db_session, service_factory):
     template_1 = service_factory.get('1').templates[0]
     template_2 = service_factory.get('2').templates[0]

tests/app/service/test_rest.py

@@ -1221,6 +1221,38 @@ def test_get_weekly_notification_stats(notify_api, notify_db, notify_db_session)
     }
 
 
+@pytest.mark.parametrize(
+    'url, expected_status, expected_json', [
+        (
+            '/service/{}/notifications/monthly?year=2001',
+            200,
+            {'data': {'foo': 'bar'}},
+        ),
+        (
+            '/service/{}/notifications/monthly?year=baz',
+            400,
+            {'message': 'Year must be a number', 'result': 'error'},
+        ),
+        (
+            '/service/{}/notifications/monthly',
+            400,
+            {'message': 'Year must be a number', 'result': 'error'},
+        ),
+    ]
+)
+def test_get_monthly_notification_stats(mocker, client, sample_service, url, expected_status, expected_json):
+    mock_dao = mocker.patch(
+        'app.service.rest.dao_fetch_monthly_historical_stats_for_service',
+        return_value={'foo': 'bar'},
+    )
+    response = client.get(
+        url.format(sample_service.id),
+        headers=[create_authorization_header()]
+    )
+    assert response.status_code == expected_status
+    assert json.loads(response.get_data(as_text=True)) == expected_json
+
+
 def test_get_services_with_detailed_flag(notify_api, notify_db, notify_db_session):
     notifications = [
         create_sample_notification(notify_db, notify_db_session),