Merge pull request #1377 from alphagov/rc-monthly-template-usage

Added scheduled task to get stats for template usage

app/celery/scheduled_tasks.py

@@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload
 
 from app.aws import s3
 from app import notify_celery
+from app.dao.services_dao import (
+    dao_fetch_monthly_historical_stats_by_template
+)
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
 from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
@@ -402,3 +406,17 @@ def check_job_status():
             queue=QueueNames.JOBS
         )
         raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
+
+
+@notify_celery.task(name='daily-stats-template_usage_by_month')
+@statsd(namespace="tasks")
+def daily_stats_template_usage_by_month():
+    results = dao_fetch_monthly_historical_stats_by_template()
+
+    for result in results:
+        insert_or_update_stats_for_template(
+            result.template_id,
+            result.month,
+            result.year,
+            result.count
+        )

app/config.py

@@ -241,6 +241,11 @@ class Config(object):
             'task': 'check-job-status',
             'schedule': crontab(),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'daily-stats-template_usage_by_month': {
+            'task': 'daily-stats-template_usage_by_month',
+            'schedule': crontab(hour=0, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/monthly_billing_dao.py

@@ -26,6 +26,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
     ).distinct().all()
 
 
+@statsd(namespace="dao")
 def create_or_update_monthly_billing(service_id, billing_month):
     start_date, end_date = get_month_start_and_end_date_in_utc(billing_month)
     _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE)
@@ -47,6 +48,7 @@ def _monthly_billing_data_to_json(billing_data):
     return results
 
 
+@statsd(namespace="dao")
 @transactional
 def _update_monthly_billing(service_id, start_date, end_date, notification_type):
     billing_data = get_billing_data_for_month(

app/dao/notification_usage_dao.py

@@ -105,6 +105,7 @@ def is_between(date, start_date, end_date):
     return start_date <= date <= end_date
 
 
+@statsd(namespace="dao")
 def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
     if notification_type == SMS_TYPE:

app/dao/services_dao.py

@@ -1,7 +1,7 @@
 import uuid
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, time
 
-from sqlalchemy import asc, func
+from sqlalchemy import asc, func, extract
 from sqlalchemy.orm import joinedload
 from flask import current_app
 
@@ -520,3 +520,25 @@ def dao_fetch_active_users_for_service(service_id):
     )
 
     return query.all()
+
+
+@statsd(namespace="dao")
+def dao_fetch_monthly_historical_stats_by_template():
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+    year = func.date_trunc("year", NotificationHistory.created_at)
+    end_date = datetime.combine(date.today(), time.min)
+
+    return db.session.query(
+        NotificationHistory.template_id,
+        extract('month', month).label('month'),
+        extract('year', year).label('year'),
+        func.count().label('count')
+    ).filter(
+        NotificationHistory.created_at < end_date
+    ).group_by(
+        NotificationHistory.template_id,
+        month,
+        year
+    ).order_by(
+        NotificationHistory.template_id
+    ).all()

app/dao/stats_template_usage_by_month_dao.py

@@ -0,0 +1,24 @@
+from app import db
+from app.models import StatsTemplateUsageByMonth
+
+
+def insert_or_update_stats_for_template(template_id, month, year, count):
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).filter(
+        StatsTemplateUsageByMonth.template_id == template_id,
+        StatsTemplateUsageByMonth.month == month,
+        StatsTemplateUsageByMonth.year == year
+    ).update(
+        {
+            'count': count
+        }
+    )
+    if result == 0:
+        monthly_stats = StatsTemplateUsageByMonth(
+            template_id=template_id,
+            month=month,
+            year=year,
+            count=count
+        )
+        db.session.add(monthly_stats)

app/models.py

@@ -1555,3 +1555,37 @@ class AuthType(db.Model):
     __tablename__ = 'auth_type'
 
     name = db.Column(db.String, primary_key=True)
+
+
+class StatsTemplateUsageByMonth(db.Model):
+    __tablename__ = "stats_template_usage_by_month"
+
+    template_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('templates.id'),
+        unique=False,
+        index=True,
+        nullable=False,
+        primary_key=True
+    )
+    month = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.month
+    )
+    year = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.year
+    )
+    count = db.Column(
+        db.Integer,
+        nullable=False,
+        default=0
+    )

migrations/versions/0135_stats_template_usage.py

@@ -0,0 +1,38 @@
+"""
+
+Revision ID: 0135_stats_template_usage
+Revises: 0134_add_email_2fa_template
+Create Date: 2017-11-07 14:35:04.798561
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0135_stats_template_usage'
+down_revision = '0134_add_email_2fa_template'
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('stats_template_usage_by_month',
+    sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False),
+    sa.Column('month', sa.Integer(), nullable=False),
+    sa.Column('year', sa.Integer(), nullable=False),
+    sa.Column('count', sa.Integer(), nullable=False),
+    sa.ForeignKeyConstraint(['template_id'], ['templates.id'], ),
+    sa.PrimaryKeyConstraint('template_id', 'month', 'year')
+    )
+    op.create_index(op.f('ix_stats_template_usage_by_month_month'), 'stats_template_usage_by_month', ['month'], unique=False)
+    op.create_index(op.f('ix_stats_template_usage_by_month_template_id'), 'stats_template_usage_by_month', ['template_id'], unique=False)
+    op.create_index(op.f('ix_stats_template_usage_by_month_year'), 'stats_template_usage_by_month', ['year'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_stats_template_usage_by_month_year'), table_name='stats_template_usage_by_month')
+    op.drop_index(op.f('ix_stats_template_usage_by_month_template_id'), table_name='stats_template_usage_by_month')
+    op.drop_index(op.f('ix_stats_template_usage_by_month_month'), table_name='stats_template_usage_by_month')
+    op.drop_table('stats_template_usage_by_month')
+    # ### end Alembic commands ###

tests/app/celery/test_scheduled_tasks.py

@@ -2,11 +2,13 @@ from datetime import datetime, timedelta
 from functools import partial
 from unittest.mock import call, patch, PropertyMock
 
+import functools
 from flask import current_app
 
 import pytest
 from freezegun import freeze_time
 
+from app import db
 from app.celery import scheduled_tasks
 from app.celery.scheduled_tasks import (
     check_job_status,
@@ -30,7 +32,8 @@ from app.celery.scheduled_tasks import (
     send_total_sent_notifications_to_performance_platform,
     switch_current_sms_provider_on_slow_delivery,
     timeout_job_statistics,
-    timeout_notifications
+    timeout_notifications,
+    daily_stats_template_usage_by_month
 )
 from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient
 from app.config import QueueNames, TaskNames
@@ -49,14 +52,16 @@ from app.models import (
     NOTIFICATION_PENDING,
     NOTIFICATION_CREATED,
     KEY_TYPE_TEST,
-    MonthlyBilling
-)
+    MonthlyBilling,
+    StatsTemplateUsageByMonth)
 from app.utils import get_london_midnight_in_utc
 from app.v2.errors import JobIncompleteError
 from tests.app.db import create_notification, create_service, create_template, create_job, create_rate
+
 from tests.app.conftest import (
     sample_job as create_sample_job,
     sample_notification_history as create_notification_history,
+    sample_template as create_sample_template,
     create_custom_template,
     datetime_in_past
 )
@@ -834,3 +839,107 @@ def test_check_job_status_task_raises_job_incomplete_error_for_multiple_jobs(moc
         args=([str(job.id), str(job_2.id)],),
         queue=QueueNames.JOBS
     )
+
+
+def test_daily_stats_template_usage_by_month(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session)
+    template_two = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_by_month()
+
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).order_by(
+        StatsTemplateUsageByMonth.year,
+        StatsTemplateUsageByMonth.month
+    ).all()
+
+    assert len(result) == 2
+
+    assert result[0].template_id == template_two.id
+    assert result[0].month == 4
+    assert result[0].year == 2016
+    assert result[0].count == 2
+
+    assert result[1].template_id == template_one.id
+    assert result[1].month == 10
+    assert result[1].year == 2017
+    assert result[1].count == 1
+
+
+def test_daily_stats_template_usage_by_month_no_data():
+    daily_stats_template_usage_by_month()
+
+    results = db.session.query(StatsTemplateUsageByMonth).all()
+
+    assert len(results) == 0
+
+
+def test_daily_stats_template_usage_by_month_multiple_runs(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session)
+    template_two = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 11, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_by_month()
+
+    template_three = create_sample_template(notify_db, notify_db_session)
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_three)
+    notification_history(created_at=datetime(2017, 9, 1), sample_template=template_three)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    daily_stats_template_usage_by_month()
+
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).order_by(
+        StatsTemplateUsageByMonth.year,
+        StatsTemplateUsageByMonth.month
+    ).all()
+
+    assert len(result) == 4
+
+    assert result[0].template_id == template_two.id
+    assert result[0].month == 4
+    assert result[0].year == 2016
+    assert result[0].count == 4
+
+    assert result[1].template_id == template_three.id
+    assert result[1].month == 9
+    assert result[1].year == 2017
+    assert result[1].count == 1
+
+    assert result[2].template_id == template_three.id
+    assert result[2].month == 10
+    assert result[2].year == 2017
+    assert result[2].count == 1
+
+    assert result[3].template_id == template_one.id
+    assert result[3].month == 11
+    assert result[3].year == 2017
+    assert result[3].count == 1

tests/app/dao/test_services_dao.py

@@ -30,7 +30,8 @@ from app.dao.services_dao import (
     dao_suspend_service,
     dao_resume_service,
     dao_fetch_active_users_for_service,
-    dao_fetch_service_by_inbound_number
+    dao_fetch_service_by_inbound_number,
+    dao_fetch_monthly_historical_stats_by_template
 )
 from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission
 from app.dao.users_dao import save_model_user
@@ -1004,3 +1005,34 @@ def _assert_service_permissions(service_permissions, expected):
 
     assert len(service_permissions) == len(expected)
     assert set(expected) == set(p.permission for p in service_permissions)
+
+
+def test_dao_fetch_monthly_historical_stats_by_template(notify_db, notify_db_session):
+    notification_history = functools.partial(
+        create_notification_history,
+        notify_db,
+        notify_db_session,
+        status='delivered'
+    )
+
+    template_one = create_sample_template(notify_db, notify_db_session, template_name='1')
+    template_two = create_sample_template(notify_db, notify_db_session, template_name='2')
+
+    notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two)
+    notification_history(created_at=datetime.now(), sample_template=template_two)
+
+    result = sorted(dao_fetch_monthly_historical_stats_by_template(), key=lambda x: (x.month, x.year))
+
+    assert len(result) == 2
+
+    assert result[0].template_id == template_two.id
+    assert result[0].month == 4
+    assert result[0].year == 2016
+    assert result[0].count == 2
+
+    assert result[1].template_id == template_one.id
+    assert result[1].month == 10
+    assert result[1].year == 2017
+    assert result[1].count == 1

tests/app/dao/test_stats_template_usage_by_month_dao.py

@@ -0,0 +1,45 @@
+import pytest
+
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
+from app.models import StatsTemplateUsageByMonth
+
+from tests.app.conftest import sample_notification, sample_email_template, sample_template, sample_job, sample_service
+
+
+def test_create_stats_for_template(notify_db_session, sample_template):
+    assert StatsTemplateUsageByMonth.query.count() == 0
+
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10)
+    stats_by_month = StatsTemplateUsageByMonth.query.filter(
+        StatsTemplateUsageByMonth.template_id == sample_template.id
+    ).all()
+
+    assert len(stats_by_month) == 1
+    assert stats_by_month[0].template_id == sample_template.id
+    assert stats_by_month[0].month == 1
+    assert stats_by_month[0].year == 2017
+    assert stats_by_month[0].count == 10
+
+
+def test_update_stats_for_template(notify_db_session, sample_template):
+    assert StatsTemplateUsageByMonth.query.count() == 0
+
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10)
+    insert_or_update_stats_for_template(sample_template.id, 1, 2017, 20)
+    insert_or_update_stats_for_template(sample_template.id, 2, 2017, 30)
+
+    stats_by_month = StatsTemplateUsageByMonth.query.filter(
+        StatsTemplateUsageByMonth.template_id == sample_template.id
+    ).order_by(StatsTemplateUsageByMonth.template_id).all()
+
+    assert len(stats_by_month) == 2
+
+    assert stats_by_month[0].template_id == sample_template.id
+    assert stats_by_month[0].month == 1
+    assert stats_by_month[0].year == 2017
+    assert stats_by_month[0].count == 20
+
+    assert stats_by_month[1].template_id == sample_template.id
+    assert stats_by_month[1].month == 2
+    assert stats_by_month[1].year == 2017
+    assert stats_by_month[1].count == 30