Added Scheduled task to get stats for template usage

Currently some pages are timing out due to the time it takes to perform
database queries. This is an attempt to improve the performance by
performing the query against the notification history table once a day
and use the notification table for a delta between midnight and the when
the page is run and combine the results.

- Added Celery task for doing the work
- Added a dao to handle the insert and update of the stats table
- Updated tests to test the new functionality

app/celery/scheduled_tasks.py

@@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload
 
 from app.aws import s3
 from app import notify_celery
+from app.dao.services_dao import (
+    dao_fetch_monthly_historical_stats_by_template
+)
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
 from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
@@ -402,3 +406,17 @@ def check_job_status():
             queue=QueueNames.JOBS
         )
         raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
+
+
+@notify_celery.task(name='daily-stats-template_usage_by_month')
+@statsd(namespace="tasks")
+def daily_stats_template_usage_my_month():
+    results = dao_fetch_monthly_historical_stats_by_template()
+
+    for result in results:
+        insert_or_update_stats_for_template(
+            result.template_id,
+            result.month.month,
+            result.year.year,
+            result.count
+        )

app/config.py

@@ -241,6 +241,11 @@ class Config(object):
             'task': 'check-job-status',
             'schedule': crontab(),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'daily-stats-template_usage_by_month': {
+            'task': 'daily-stats-template_usage_by_month',
+            'schedule': crontab(hour=00, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/services_dao.py

@@ -1,7 +1,7 @@
 import uuid
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, time
 
-from sqlalchemy import asc, func
+from sqlalchemy import asc, func, extract
 from sqlalchemy.orm import joinedload
 from flask import current_app
 
@@ -40,7 +40,7 @@ from app.models import (
 )
 from app.service.statistics import format_monthly_template_notification_stats
 from app.statsd_decorators import statsd
-from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc
+from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc, get_london_year_from_utc_column
 from app.dao.annual_billing_dao import dao_insert_annual_billing
 
 DEFAULT_SERVICE_PERMISSIONS = [
@@ -520,3 +520,25 @@ def dao_fetch_active_users_for_service(service_id):
     )
 
     return query.all()
+
+
+@statsd(namespace="dao")
+def dao_fetch_monthly_historical_stats_by_template():
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+    year = get_london_year_from_utc_column(NotificationHistory.created_at)
+    end_date = datetime.combine(date.today(), time.min)
+
+    return db.session.query(
+        NotificationHistory.template_id,
+        month.label('month'),
+        year.label('year'),
+        func.count().label('count')
+    ).filter(
+        NotificationHistory.created_at < end_date
+    ).group_by(
+        NotificationHistory.template_id,
+        month,
+        year
+    ).order_by(
+        NotificationHistory.template_id
+    ).all()

app/dao/stats_template_usage_by_month_dao.py

@@ -0,0 +1,24 @@
+from app import db
+from app.models import StatsTemplateUsageByMonth
+
+
+def insert_or_update_stats_for_template(template_id, month, year, count):
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).filter(
+        StatsTemplateUsageByMonth.template_id == template_id,
+        StatsTemplateUsageByMonth.month == month,
+        StatsTemplateUsageByMonth.year == year
+    ).update(
+        {
+            'count': count
+        }
+    )
+    if result == 0:
+        new_sms_sender = StatsTemplateUsageByMonth(
+            template_id=template_id,
+            month=month,
+            year=year,
+            count=count
+        )
+        db.session.add(new_sms_sender)

app/utils.py

@@ -75,6 +75,22 @@ def get_london_month_from_utc_column(column):
     )
 
 
+def get_london_year_from_utc_column(column):
+    """
+     Where queries need to count notifications by month it needs to be
+     the month in BST (British Summer Time).
+     The database stores all timestamps as UTC without the timezone.
+      - First set the timezone on created_at to UTC
+      - then convert the timezone to BST (or Europe/London)
+      - lastly truncate the datetime to month with which we can group
+        queries
+    """
+    return func.date_trunc(
+        "month",
+        func.timezone("Europe/London", func.timezone("UTC", column))
+    )
+
+
 def cache_key_for_service_template_counter(service_id, limit_days=7):
     return "{}-template-counter-limit-{}-days".format(service_id, limit_days)
 

tests/app/celery/test_scheduled_tasks.py

@@ -2,11 +2,13 @@ from datetime import datetime, timedelta
 from functools import partial
 from unittest.mock import call, patch, PropertyMock
 
+import functools
 from flask import current_app
 
 import pytest
 from freezegun import freeze_time
 
+from app import db
 from app.celery import scheduled_tasks
 from app.celery.scheduled_tasks import (
     check_job_status,
@@ -30,8 +32,8 @@ from app.celery.scheduled_tasks import (
     send_total_sent_notifications_to_performance_platform,
     switch_current_sms_provider_on_slow_delivery,
     timeout_job_statistics,
-    timeout_notifications
-)
+    timeout_notifications,
+    daily_stats_template_usage_my_month)
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.config import QueueNames, TaskNames
 from app.dao.jobs_dao import dao_get_job_by_id
@@ -49,19 +51,24 @@ from app.models import (
     NOTIFICATION_PENDING,
     NOTIFICATION_CREATED,
     KEY_TYPE_TEST,
-    MonthlyBilling
-)
+    MonthlyBilling,
+    StatsTemplateUsageByMonth)
 from app.utils import get_london_midnight_in_utc
 from app.v2.errors import JobIncompleteError
 from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
 from tests.app.conftest import (
     sample_job as create_sample_job,
-    sample_notification_history as create_notification_history,
     create_custom_template,
     datetime_in_past
 )
 from tests.app.aws.test_s3 import single_s3_object_stub
-from tests.conftest import set_config_values
+from tests.conftest import set_config_values, notify_db, notify_db_session
+
+from tests.app.conftest import (
+    sample_notification as create_notification,
+    sample_notification_history as create_notification_history,
+    sample_template as create_sample_template
+)
 
 
 def _create_slow_delivery_notification(provider='mmg'):
@@ -834,3 +841,108 @@ def test_check_job_status_task_raises_job_incomplete_error_for_multiple_jobs(moc
         args=([str(job.id), str(job_2.id)],),
         queue=QueueNames.JOBS
     )
+
+
+def test_daily_stats_template_usage_my_month(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session)
+    template_two = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_my_month()
+
+    results = db.session.query(StatsTemplateUsageByMonth).all()
+
+    assert len(results) == 2
+
+    for result in results:
+        if result.template_id == template_one.id:
+            assert result.template_id == template_one.id
+            assert result.month == 10
+            assert result.year == 2017
+            assert result.count == 1
+        elif result.template_id == template_two.id:
+            assert result.template_id == template_two.id
+            assert result.month == 4
+            assert result.year == 2016
+            assert result.count == 2
+        else:
+            raise AssertionError()
+
+
+def test_daily_stats_template_usage_my_month_no_data():
+    daily_stats_template_usage_my_month()
+
+    results = db.session.query(StatsTemplateUsageByMonth).all()
+
+    assert len(results) == 0
+
+
+def test_daily_stats_template_usage_my_month_multiple_runs(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session)
+    template_two = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_my_month()
+
+    template_three = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_three)
+    notification_history(created_at=datetime(2017, 9, 1), sample_template=template_three)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_my_month()
+
+    results = db.session.query(StatsTemplateUsageByMonth).all()
+
+    assert len(results) == 4
+
+    for result in results:
+        if result.template_id == template_one.id:
+            assert result.template_id == template_one.id
+            assert result.month == 10
+            assert result.year == 2017
+            assert result.count == 1
+        elif result.template_id == template_two.id:
+            assert result.template_id == template_two.id
+            assert result.month == 4
+            assert result.year == 2016
+            assert result.count == 4
+        elif result.template_id == template_three.id:
+            if result.month == 10:
+                assert result.template_id == template_three.id
+                assert result.month == 10
+                assert result.year == 2017
+                assert result.count == 1
+            elif result.month == 9:
+                assert result.template_id == template_three.id
+                assert result.month == 9
+                assert result.year == 2017
+                assert result.count == 1
+            else:
+                raise AssertionError()
+        else:
+            raise AssertionError()

tests/app/dao/test_services_dao.py

@@ -30,8 +30,8 @@ from app.dao.services_dao import (
     dao_suspend_service,
     dao_resume_service,
     dao_fetch_active_users_for_service,
-    dao_fetch_service_by_inbound_number
-)
+    dao_fetch_service_by_inbound_number,
+    dao_fetch_monthly_historical_stats_by_template_for_service_without_status)
 from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission
 from app.dao.users_dao import save_model_user
 from app.models import (
@@ -1004,3 +1004,38 @@ def _assert_service_permissions(service_permissions, expected):
 
     assert len(service_permissions) == len(expected)
     assert set(expected) == set(p.permission for p in service_permissions)
+
+
+def test_dao_fetch_monthly_historical_stats_by_template(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session)
+    template_two = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    results = dao_fetch_monthly_historical_stats_by_template_for_service_without_status()
+
+    assert len(results) == 2
+
+    for result in results:
+        if result.template_id == template_one.id:
+            assert result.template_id == template_one.id
+            assert result.month.month == 10
+            assert result.year.year == 2017
+            assert result.count == 1
+        elif result.template_id == template_two.id:
+            assert result.template_id == template_two.id
+            assert result.month.month == 4
+            assert result.year.year == 2016
+            assert result.count == 2
+        else:
+            raise AssertionError()

tests/app/dao/test_stats_template_usage_by_month_dao.py

@@ -0,0 +1,45 @@
+import pytest
+
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
+from app.models import StatsTemplateUsageByMonth
+
+from tests.app.conftest import sample_notification, sample_email_template, sample_template, sample_job, sample_service
+
+
+def test_create_stats_for_template(notify_db_session, sample_template):
+    assert StatsTemplateUsageByMonth.query.count() == 0
+
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10)
+    stats_by_month = StatsTemplateUsageByMonth.query.filter(
+        StatsTemplateUsageByMonth.template_id == sample_template.id
+    ).all()
+
+    assert len(stats_by_month) == 1
+    assert stats_by_month[0].template_id == sample_template.id
+    assert stats_by_month[0].month == 1
+    assert stats_by_month[0].year == 2017
+    assert stats_by_month[0].count == 10
+
+
+def test_update_stats_for_template(notify_db_session, sample_template):
+    assert StatsTemplateUsageByMonth.query.count() == 0
+
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10)
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 20)
+    insert_or_update_stats_for_template(sample_template.id, 2, 2017, 30)
+
+    stats_by_month = StatsTemplateUsageByMonth.query.filter(
+        StatsTemplateUsageByMonth.template_id == sample_template.id
+    ).order_by(StatsTemplateUsageByMonth.template_id).all()
+
+    assert len(stats_by_month) == 2
+
+    assert stats_by_month[0].template_id == sample_template.id
+    assert stats_by_month[0].month == 1
+    assert stats_by_month[0].year == 2017
+    assert stats_by_month[0].count == 20
+
+    assert stats_by_month[1].template_id == sample_template.id
+    assert stats_by_month[1].month == 2
+    assert stats_by_month[1].year == 2017
+    assert stats_by_month[1].count == 30