Merge pull request #1377 from alphagov/rc-monthly-template-usage

Added scheduled task to get stats for template usage

app/celery/scheduled_tasks.py

@@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload
 
 from app.aws import s3
 from app import notify_celery
+from app.dao.services_dao import (
+    dao_fetch_monthly_historical_stats_by_template
+)
+from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template
 from app.performance_platform import total_sent_notifications, processing_time
 from app import performance_platform_client
 from app.dao.date_util import get_month_start_and_end_date_in_utc
@@ -402,3 +406,17 @@ def check_job_status():
             queue=QueueNames.JOBS
         )
         raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids))
+
+
+@notify_celery.task(name='daily-stats-template_usage_by_month')
+@statsd(namespace="tasks")
+def daily_stats_template_usage_by_month():
+    results = dao_fetch_monthly_historical_stats_by_template()
+
+    for result in results:
+        insert_or_update_stats_for_template(
+            result.template_id,
+            result.month,
+            result.year,
+            result.count
+        )

app/config.py

@@ -241,6 +241,11 @@ class Config(object):
             'task': 'check-job-status',
             'schedule': crontab(),
             'options': {'queue': QueueNames.PERIODIC}
+        },
+        'daily-stats-template_usage_by_month': {
+            'task': 'daily-stats-template_usage_by_month',
+            'schedule': crontab(hour=0, minute=50),
+            'options': {'queue': QueueNames.PERIODIC}
         }
     }
     CELERY_QUEUES = []

app/dao/monthly_billing_dao.py

@@ -26,6 +26,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date):
     ).distinct().all()
 
 
+@statsd(namespace="dao")
 def create_or_update_monthly_billing(service_id, billing_month):
     start_date, end_date = get_month_start_and_end_date_in_utc(billing_month)
     _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE)
@@ -47,6 +48,7 @@ def _monthly_billing_data_to_json(billing_data):
     return results
 
 
+@statsd(namespace="dao")
 @transactional
 def _update_monthly_billing(service_id, start_date, end_date, notification_type):
     billing_data = get_billing_data_for_month(

app/dao/notification_usage_dao.py

@@ -105,6 +105,7 @@ def is_between(date, start_date, end_date):
     return start_date <= date <= end_date
 
 
+@statsd(namespace="dao")
 def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type):
     month = get_london_month_from_utc_column(NotificationHistory.created_at)
     if notification_type == SMS_TYPE:

app/dao/services_dao.py

@@ -1,7 +1,7 @@
 import uuid
-from datetime import date, datetime, timedelta
+from datetime import date, datetime, timedelta, time
 
-from sqlalchemy import asc, func
+from sqlalchemy import asc, func, extract
 from sqlalchemy.orm import joinedload
 from flask import current_app
 
@@ -520,3 +520,25 @@ def dao_fetch_active_users_for_service(service_id):
     )
 
     return query.all()
+
+
+@statsd(namespace="dao")
+def dao_fetch_monthly_historical_stats_by_template():
+    month = get_london_month_from_utc_column(NotificationHistory.created_at)
+    year = func.date_trunc("year", NotificationHistory.created_at)
+    end_date = datetime.combine(date.today(), time.min)
+
+    return db.session.query(
+        NotificationHistory.template_id,
+        extract('month', month).label('month'),
+        extract('year', year).label('year'),
+        func.count().label('count')
+    ).filter(
+        NotificationHistory.created_at < end_date
+    ).group_by(
+        NotificationHistory.template_id,
+        month,
+        year
+    ).order_by(
+        NotificationHistory.template_id
+    ).all()

app/dao/stats_template_usage_by_month_dao.py

@@ -0,0 +1,24 @@
+from app import db
+from app.models import StatsTemplateUsageByMonth
+
+
+def insert_or_update_stats_for_template(template_id, month, year, count):
+    result = db.session.query(
+        StatsTemplateUsageByMonth
+    ).filter(
+        StatsTemplateUsageByMonth.template_id == template_id,
+        StatsTemplateUsageByMonth.month == month,
+        StatsTemplateUsageByMonth.year == year
+    ).update(
+        {
+            'count': count
+        }
+    )
+    if result == 0:
+        monthly_stats = StatsTemplateUsageByMonth(
+            template_id=template_id,
+            month=month,
+            year=year,
+            count=count
+        )
+        db.session.add(monthly_stats)

app/models.py

@@ -1555,3 +1555,37 @@ class AuthType(db.Model):
     __tablename__ = 'auth_type'
 
     name = db.Column(db.String, primary_key=True)
+
+
+class StatsTemplateUsageByMonth(db.Model):
+    __tablename__ = "stats_template_usage_by_month"
+
+    template_id = db.Column(
+        UUID(as_uuid=True),
+        db.ForeignKey('templates.id'),
+        unique=False,
+        index=True,
+        nullable=False,
+        primary_key=True
+    )
+    month = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.month
+    )
+    year = db.Column(
+        db.Integer,
+        nullable=False,
+        index=True,
+        unique=False,
+        primary_key=True,
+        default=datetime.datetime.year
+    )
+    count = db.Column(
+        db.Integer,
+        nullable=False,
+        default=0
+    )