diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 2e0413fe8..e39876df0 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload from app.aws import s3 from app import notify_celery +from app.dao.services_dao import ( + dao_fetch_monthly_historical_stats_by_template +) +from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template from app.performance_platform import total_sent_notifications, processing_time from app import performance_platform_client from app.dao.date_util import get_month_start_and_end_date_in_utc @@ -402,3 +406,17 @@ def check_job_status(): queue=QueueNames.JOBS ) raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids)) + + +@notify_celery.task(name='daily-stats-template_usage_by_month') +@statsd(namespace="tasks") +def daily_stats_template_usage_by_month(): + results = dao_fetch_monthly_historical_stats_by_template() + + for result in results: + insert_or_update_stats_for_template( + result.template_id, + result.month, + result.year, + result.count + ) diff --git a/app/config.py b/app/config.py index 59be8ad02..16892ac28 100644 --- a/app/config.py +++ b/app/config.py @@ -241,6 +241,11 @@ class Config(object): 'task': 'check-job-status', 'schedule': crontab(), 'options': {'queue': QueueNames.PERIODIC} + }, + 'daily-stats-template_usage_by_month': { + 'task': 'daily-stats-template_usage_by_month', + 'schedule': crontab(hour=0, minute=50), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] diff --git a/app/dao/monthly_billing_dao.py b/app/dao/monthly_billing_dao.py index 606c02cc9..4528f414d 100644 --- a/app/dao/monthly_billing_dao.py +++ b/app/dao/monthly_billing_dao.py @@ -26,6 +26,7 @@ def get_service_ids_that_need_billing_populated(start_date, end_date): ).distinct().all() +@statsd(namespace="dao") def create_or_update_monthly_billing(service_id, billing_month): start_date, end_date = get_month_start_and_end_date_in_utc(billing_month) _update_monthly_billing(service_id, start_date, end_date, SMS_TYPE) @@ -47,6 +48,7 @@ def _monthly_billing_data_to_json(billing_data): return results +@statsd(namespace="dao") @transactional def _update_monthly_billing(service_id, start_date, end_date, notification_type): billing_data = get_billing_data_for_month( diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py index 7542fb0df..afb547287 100644 --- a/app/dao/notification_usage_dao.py +++ b/app/dao/notification_usage_dao.py @@ -105,6 +105,7 @@ def is_between(date, start_date, end_date): return start_date <= date <= end_date +@statsd(namespace="dao") def billing_data_per_month_query(rate, service_id, start_date, end_date, notification_type): month = get_london_month_from_utc_column(NotificationHistory.created_at) if notification_type == SMS_TYPE: diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 93905e7e6..d023fb297 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,7 +1,7 @@ import uuid -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, time -from sqlalchemy import asc, func +from sqlalchemy import asc, func, extract from sqlalchemy.orm import joinedload from flask import current_app @@ -520,3 +520,25 @@ def dao_fetch_active_users_for_service(service_id): ) return query.all() + + +@statsd(namespace="dao") +def dao_fetch_monthly_historical_stats_by_template(): + month = get_london_month_from_utc_column(NotificationHistory.created_at) + year = func.date_trunc("year", NotificationHistory.created_at) + end_date = datetime.combine(date.today(), time.min) + + return db.session.query( + NotificationHistory.template_id, + extract('month', month).label('month'), + extract('year', year).label('year'), + func.count().label('count') + ).filter( + NotificationHistory.created_at < end_date + ).group_by( + NotificationHistory.template_id, + month, + year + ).order_by( + NotificationHistory.template_id + ).all() diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py new file mode 100644 index 000000000..2884e2c53 --- /dev/null +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -0,0 +1,24 @@ +from app import db +from app.models import StatsTemplateUsageByMonth + + +def insert_or_update_stats_for_template(template_id, month, year, count): + result = db.session.query( + StatsTemplateUsageByMonth + ).filter( + StatsTemplateUsageByMonth.template_id == template_id, + StatsTemplateUsageByMonth.month == month, + StatsTemplateUsageByMonth.year == year + ).update( + { + 'count': count + } + ) + if result == 0: + monthly_stats = StatsTemplateUsageByMonth( + template_id=template_id, + month=month, + year=year, + count=count + ) + db.session.add(monthly_stats) diff --git a/app/models.py b/app/models.py index 74b0af79a..0c64a4581 100644 --- a/app/models.py +++ b/app/models.py @@ -1555,3 +1555,37 @@ class AuthType(db.Model): __tablename__ = 'auth_type' name = db.Column(db.String, primary_key=True) + + +class StatsTemplateUsageByMonth(db.Model): + __tablename__ = "stats_template_usage_by_month" + + template_id = db.Column( + UUID(as_uuid=True), + db.ForeignKey('templates.id'), + unique=False, + index=True, + nullable=False, + primary_key=True + ) + month = db.Column( + db.Integer, + nullable=False, + index=True, + unique=False, + primary_key=True, + default=datetime.datetime.month + ) + year = db.Column( + db.Integer, + nullable=False, + index=True, + unique=False, + primary_key=True, + default=datetime.datetime.year + ) + count = db.Column( + db.Integer, + nullable=False, + default=0 + ) diff --git a/migrations/versions/0135_stats_template_usage.py b/migrations/versions/0135_stats_template_usage.py new file mode 100644 index 000000000..5a8f5ef7a --- /dev/null +++ b/migrations/versions/0135_stats_template_usage.py @@ -0,0 +1,38 @@ +""" + +Revision ID: 0135_stats_template_usage +Revises: 0134_add_email_2fa_template +Create Date: 2017-11-07 14:35:04.798561 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0135_stats_template_usage' +down_revision = '0134_add_email_2fa_template' + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('stats_template_usage_by_month', + sa.Column('template_id', postgresql.UUID(as_uuid=True), nullable=False), + sa.Column('month', sa.Integer(), nullable=False), + sa.Column('year', sa.Integer(), nullable=False), + sa.Column('count', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['template_id'], ['templates.id'], ), + sa.PrimaryKeyConstraint('template_id', 'month', 'year') + ) + op.create_index(op.f('ix_stats_template_usage_by_month_month'), 'stats_template_usage_by_month', ['month'], unique=False) + op.create_index(op.f('ix_stats_template_usage_by_month_template_id'), 'stats_template_usage_by_month', ['template_id'], unique=False) + op.create_index(op.f('ix_stats_template_usage_by_month_year'), 'stats_template_usage_by_month', ['year'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_stats_template_usage_by_month_year'), table_name='stats_template_usage_by_month') + op.drop_index(op.f('ix_stats_template_usage_by_month_template_id'), table_name='stats_template_usage_by_month') + op.drop_index(op.f('ix_stats_template_usage_by_month_month'), table_name='stats_template_usage_by_month') + op.drop_table('stats_template_usage_by_month') + # ### end Alembic commands ### diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index a61c70807..d31e2b5a4 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -2,11 +2,13 @@ from datetime import datetime, timedelta from functools import partial from unittest.mock import call, patch, PropertyMock +import functools from flask import current_app import pytest from freezegun import freeze_time +from app import db from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( check_job_status, @@ -30,7 +32,8 @@ from app.celery.scheduled_tasks import ( send_total_sent_notifications_to_performance_platform, switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, - timeout_notifications + timeout_notifications, + daily_stats_template_usage_by_month ) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames @@ -49,14 +52,16 @@ from app.models import ( NOTIFICATION_PENDING, NOTIFICATION_CREATED, KEY_TYPE_TEST, - MonthlyBilling -) + MonthlyBilling, + StatsTemplateUsageByMonth) from app.utils import get_london_midnight_in_utc from app.v2.errors import JobIncompleteError from tests.app.db import create_notification, create_service, create_template, create_job, create_rate + from tests.app.conftest import ( sample_job as create_sample_job, sample_notification_history as create_notification_history, + sample_template as create_sample_template, create_custom_template, datetime_in_past ) @@ -834,3 +839,107 @@ def test_check_job_status_task_raises_job_incomplete_error_for_multiple_jobs(moc args=([str(job.id), str(job_2.id)],), queue=QueueNames.JOBS ) + + +def test_daily_stats_template_usage_by_month(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session) + template_two = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_by_month() + + result = db.session.query( + StatsTemplateUsageByMonth + ).order_by( + StatsTemplateUsageByMonth.year, + StatsTemplateUsageByMonth.month + ).all() + + assert len(result) == 2 + + assert result[0].template_id == template_two.id + assert result[0].month == 4 + assert result[0].year == 2016 + assert result[0].count == 2 + + assert result[1].template_id == template_one.id + assert result[1].month == 10 + assert result[1].year == 2017 + assert result[1].count == 1 + + +def test_daily_stats_template_usage_by_month_no_data(): + daily_stats_template_usage_by_month() + + results = db.session.query(StatsTemplateUsageByMonth).all() + + assert len(results) == 0 + + +def test_daily_stats_template_usage_by_month_multiple_runs(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session) + template_two = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 11, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_by_month() + + template_three = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_three) + notification_history(created_at=datetime(2017, 9, 1), sample_template=template_three) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_by_month() + + result = db.session.query( + StatsTemplateUsageByMonth + ).order_by( + StatsTemplateUsageByMonth.year, + StatsTemplateUsageByMonth.month + ).all() + + assert len(result) == 4 + + assert result[0].template_id == template_two.id + assert result[0].month == 4 + assert result[0].year == 2016 + assert result[0].count == 4 + + assert result[1].template_id == template_three.id + assert result[1].month == 9 + assert result[1].year == 2017 + assert result[1].count == 1 + + assert result[2].template_id == template_three.id + assert result[2].month == 10 + assert result[2].year == 2017 + assert result[2].count == 1 + + assert result[3].template_id == template_one.id + assert result[3].month == 11 + assert result[3].year == 2017 + assert result[3].count == 1 diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index c45ce0272..997301531 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -30,7 +30,8 @@ from app.dao.services_dao import ( dao_suspend_service, dao_resume_service, dao_fetch_active_users_for_service, - dao_fetch_service_by_inbound_number + dao_fetch_service_by_inbound_number, + dao_fetch_monthly_historical_stats_by_template ) from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.users_dao import save_model_user @@ -1004,3 +1005,34 @@ def _assert_service_permissions(service_permissions, expected): assert len(service_permissions) == len(expected) assert set(expected) == set(p.permission for p in service_permissions) + + +def test_dao_fetch_monthly_historical_stats_by_template(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session, template_name='1') + template_two = create_sample_template(notify_db, notify_db_session, template_name='2') + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + result = sorted(dao_fetch_monthly_historical_stats_by_template(), key=lambda x: (x.month, x.year)) + + assert len(result) == 2 + + assert result[0].template_id == template_two.id + assert result[0].month == 4 + assert result[0].year == 2016 + assert result[0].count == 2 + + assert result[1].template_id == template_one.id + assert result[1].month == 10 + assert result[1].year == 2017 + assert result[1].count == 1 diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py new file mode 100644 index 000000000..ae46c682b --- /dev/null +++ b/tests/app/dao/test_stats_template_usage_by_month_dao.py @@ -0,0 +1,45 @@ +import pytest + +from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template +from app.models import StatsTemplateUsageByMonth + +from tests.app.conftest import sample_notification, sample_email_template, sample_template, sample_job, sample_service + + +def test_create_stats_for_template(notify_db_session, sample_template): + assert StatsTemplateUsageByMonth.query.count() == 0 + + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) + stats_by_month = StatsTemplateUsageByMonth.query.filter( + StatsTemplateUsageByMonth.template_id == sample_template.id + ).all() + + assert len(stats_by_month) == 1 + assert stats_by_month[0].template_id == sample_template.id + assert stats_by_month[0].month == 1 + assert stats_by_month[0].year == 2017 + assert stats_by_month[0].count == 10 + + +def test_update_stats_for_template(notify_db_session, sample_template): + assert StatsTemplateUsageByMonth.query.count() == 0 + + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 20) + insert_or_update_stats_for_template(sample_template.id, 2, 2017, 30) + + stats_by_month = StatsTemplateUsageByMonth.query.filter( + StatsTemplateUsageByMonth.template_id == sample_template.id + ).order_by(StatsTemplateUsageByMonth.template_id).all() + + assert len(stats_by_month) == 2 + + assert stats_by_month[0].template_id == sample_template.id + assert stats_by_month[0].month == 1 + assert stats_by_month[0].year == 2017 + assert stats_by_month[0].count == 20 + + assert stats_by_month[1].template_id == sample_template.id + assert stats_by_month[1].month == 2 + assert stats_by_month[1].year == 2017 + assert stats_by_month[1].count == 30