diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 2e0413fe8..9b1a830d9 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -11,6 +11,10 @@ from notifications_utils.s3 import s3upload from app.aws import s3 from app import notify_celery +from app.dao.services_dao import ( + dao_fetch_monthly_historical_stats_by_template +) +from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template from app.performance_platform import total_sent_notifications, processing_time from app import performance_platform_client from app.dao.date_util import get_month_start_and_end_date_in_utc @@ -402,3 +406,17 @@ def check_job_status(): queue=QueueNames.JOBS ) raise JobIncompleteError("Job(s) {} have not completed.".format(job_ids)) + + +@notify_celery.task(name='daily-stats-template_usage_by_month') +@statsd(namespace="tasks") +def daily_stats_template_usage_my_month(): + results = dao_fetch_monthly_historical_stats_by_template() + + for result in results: + insert_or_update_stats_for_template( + result.template_id, + result.month.month, + result.year.year, + result.count + ) diff --git a/app/config.py b/app/config.py index 59be8ad02..afd70dade 100644 --- a/app/config.py +++ b/app/config.py @@ -241,6 +241,11 @@ class Config(object): 'task': 'check-job-status', 'schedule': crontab(), 'options': {'queue': QueueNames.PERIODIC} + }, + 'daily-stats-template_usage_by_month': { + 'task': 'daily-stats-template_usage_by_month', + 'schedule': crontab(hour=00, minute=50), + 'options': {'queue': QueueNames.PERIODIC} } } CELERY_QUEUES = [] diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 93905e7e6..684696d83 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -1,7 +1,7 @@ import uuid -from datetime import date, datetime, timedelta +from datetime import date, datetime, timedelta, time -from sqlalchemy import asc, func +from sqlalchemy import asc, func, extract from sqlalchemy.orm import joinedload from flask import current_app @@ -40,7 +40,7 @@ from app.models import ( ) from app.service.statistics import format_monthly_template_notification_stats from app.statsd_decorators import statsd -from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc +from app.utils import get_london_month_from_utc_column, get_london_midnight_in_utc, get_london_year_from_utc_column from app.dao.annual_billing_dao import dao_insert_annual_billing DEFAULT_SERVICE_PERMISSIONS = [ @@ -520,3 +520,25 @@ def dao_fetch_active_users_for_service(service_id): ) return query.all() + + +@statsd(namespace="dao") +def dao_fetch_monthly_historical_stats_by_template(): + month = get_london_month_from_utc_column(NotificationHistory.created_at) + year = get_london_year_from_utc_column(NotificationHistory.created_at) + end_date = datetime.combine(date.today(), time.min) + + return db.session.query( + NotificationHistory.template_id, + month.label('month'), + year.label('year'), + func.count().label('count') + ).filter( + NotificationHistory.created_at < end_date + ).group_by( + NotificationHistory.template_id, + month, + year + ).order_by( + NotificationHistory.template_id + ).all() diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py new file mode 100644 index 000000000..ca94dd83f --- /dev/null +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -0,0 +1,24 @@ +from app import db +from app.models import StatsTemplateUsageByMonth + + +def insert_or_update_stats_for_template(template_id, month, year, count): + result = db.session.query( + StatsTemplateUsageByMonth + ).filter( + StatsTemplateUsageByMonth.template_id == template_id, + StatsTemplateUsageByMonth.month == month, + StatsTemplateUsageByMonth.year == year + ).update( + { + 'count': count + } + ) + if result == 0: + new_sms_sender = StatsTemplateUsageByMonth( + template_id=template_id, + month=month, + year=year, + count=count + ) + db.session.add(new_sms_sender) diff --git a/app/utils.py b/app/utils.py index 49f6d61dc..7fe4bdf89 100644 --- a/app/utils.py +++ b/app/utils.py @@ -75,6 +75,22 @@ def get_london_month_from_utc_column(column): ) +def get_london_year_from_utc_column(column): + """ + Where queries need to count notifications by month it needs to be + the month in BST (British Summer Time). + The database stores all timestamps as UTC without the timezone. + - First set the timezone on created_at to UTC + - then convert the timezone to BST (or Europe/London) + - lastly truncate the datetime to month with which we can group + queries + """ + return func.date_trunc( + "month", + func.timezone("Europe/London", func.timezone("UTC", column)) + ) + + def cache_key_for_service_template_counter(service_id, limit_days=7): return "{}-template-counter-limit-{}-days".format(service_id, limit_days) diff --git a/tests/app/celery/test_scheduled_tasks.py b/tests/app/celery/test_scheduled_tasks.py index a61c70807..7f60318cc 100644 --- a/tests/app/celery/test_scheduled_tasks.py +++ b/tests/app/celery/test_scheduled_tasks.py @@ -2,11 +2,13 @@ from datetime import datetime, timedelta from functools import partial from unittest.mock import call, patch, PropertyMock +import functools from flask import current_app import pytest from freezegun import freeze_time +from app import db from app.celery import scheduled_tasks from app.celery.scheduled_tasks import ( check_job_status, @@ -30,8 +32,8 @@ from app.celery.scheduled_tasks import ( send_total_sent_notifications_to_performance_platform, switch_current_sms_provider_on_slow_delivery, timeout_job_statistics, - timeout_notifications -) + timeout_notifications, + daily_stats_template_usage_my_month) from app.clients.performance_platform.performance_platform_client import PerformancePlatformClient from app.config import QueueNames, TaskNames from app.dao.jobs_dao import dao_get_job_by_id @@ -49,19 +51,24 @@ from app.models import ( NOTIFICATION_PENDING, NOTIFICATION_CREATED, KEY_TYPE_TEST, - MonthlyBilling -) + MonthlyBilling, + StatsTemplateUsageByMonth) from app.utils import get_london_midnight_in_utc from app.v2.errors import JobIncompleteError from tests.app.db import create_notification, create_service, create_template, create_job, create_rate from tests.app.conftest import ( sample_job as create_sample_job, - sample_notification_history as create_notification_history, create_custom_template, datetime_in_past ) from tests.app.aws.test_s3 import single_s3_object_stub -from tests.conftest import set_config_values +from tests.conftest import set_config_values, notify_db, notify_db_session + +from tests.app.conftest import ( + sample_notification as create_notification, + sample_notification_history as create_notification_history, + sample_template as create_sample_template +) def _create_slow_delivery_notification(provider='mmg'): @@ -834,3 +841,108 @@ def test_check_job_status_task_raises_job_incomplete_error_for_multiple_jobs(moc args=([str(job.id), str(job_2.id)],), queue=QueueNames.JOBS ) + + +def test_daily_stats_template_usage_my_month(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session) + template_two = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_my_month() + + results = db.session.query(StatsTemplateUsageByMonth).all() + + assert len(results) == 2 + + for result in results: + if result.template_id == template_one.id: + assert result.template_id == template_one.id + assert result.month == 10 + assert result.year == 2017 + assert result.count == 1 + elif result.template_id == template_two.id: + assert result.template_id == template_two.id + assert result.month == 4 + assert result.year == 2016 + assert result.count == 2 + else: + raise AssertionError() + + +def test_daily_stats_template_usage_my_month_no_data(): + daily_stats_template_usage_my_month() + + results = db.session.query(StatsTemplateUsageByMonth).all() + + assert len(results) == 0 + + +def test_daily_stats_template_usage_my_month_multiple_runs(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session) + template_two = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_my_month() + + template_three = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_three) + notification_history(created_at=datetime(2017, 9, 1), sample_template=template_three) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + daily_stats_template_usage_my_month() + + results = db.session.query(StatsTemplateUsageByMonth).all() + + assert len(results) == 4 + + for result in results: + if result.template_id == template_one.id: + assert result.template_id == template_one.id + assert result.month == 10 + assert result.year == 2017 + assert result.count == 1 + elif result.template_id == template_two.id: + assert result.template_id == template_two.id + assert result.month == 4 + assert result.year == 2016 + assert result.count == 4 + elif result.template_id == template_three.id: + if result.month == 10: + assert result.template_id == template_three.id + assert result.month == 10 + assert result.year == 2017 + assert result.count == 1 + elif result.month == 9: + assert result.template_id == template_three.id + assert result.month == 9 + assert result.year == 2017 + assert result.count == 1 + else: + raise AssertionError() + else: + raise AssertionError() diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index c45ce0272..bc5a8af82 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -30,8 +30,8 @@ from app.dao.services_dao import ( dao_suspend_service, dao_resume_service, dao_fetch_active_users_for_service, - dao_fetch_service_by_inbound_number -) + dao_fetch_service_by_inbound_number, + dao_fetch_monthly_historical_stats_by_template_for_service_without_status) from app.dao.service_permissions_dao import dao_add_service_permission, dao_remove_service_permission from app.dao.users_dao import save_model_user from app.models import ( @@ -1004,3 +1004,38 @@ def _assert_service_permissions(service_permissions, expected): assert len(service_permissions) == len(expected) assert set(expected) == set(p.permission for p in service_permissions) + + +def test_dao_fetch_monthly_historical_stats_by_template(notify_db, notify_db_session): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session) + template_two = create_sample_template(notify_db, notify_db_session) + + notification_history(created_at=datetime(2017, 10, 1), sample_template=template_one) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime(2016, 4, 1), sample_template=template_two) + notification_history(created_at=datetime.now(), sample_template=template_two) + + results = dao_fetch_monthly_historical_stats_by_template_for_service_without_status() + + assert len(results) == 2 + + for result in results: + if result.template_id == template_one.id: + assert result.template_id == template_one.id + assert result.month.month == 10 + assert result.year.year == 2017 + assert result.count == 1 + elif result.template_id == template_two.id: + assert result.template_id == template_two.id + assert result.month.month == 4 + assert result.year.year == 2016 + assert result.count == 2 + else: + raise AssertionError() diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py new file mode 100644 index 000000000..ae46c682b --- /dev/null +++ b/tests/app/dao/test_stats_template_usage_by_month_dao.py @@ -0,0 +1,45 @@ +import pytest + +from app.dao.stats_template_usage_by_month_dao import insert_or_update_stats_for_template +from app.models import StatsTemplateUsageByMonth + +from tests.app.conftest import sample_notification, sample_email_template, sample_template, sample_job, sample_service + + +def test_create_stats_for_template(notify_db_session, sample_template): + assert StatsTemplateUsageByMonth.query.count() == 0 + + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) + stats_by_month = StatsTemplateUsageByMonth.query.filter( + StatsTemplateUsageByMonth.template_id == sample_template.id + ).all() + + assert len(stats_by_month) == 1 + assert stats_by_month[0].template_id == sample_template.id + assert stats_by_month[0].month == 1 + assert stats_by_month[0].year == 2017 + assert stats_by_month[0].count == 10 + + +def test_update_stats_for_template(notify_db_session, sample_template): + assert StatsTemplateUsageByMonth.query.count() == 0 + + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 10) + insert_or_update_stats_for_template(sample_template.id, 1, 2017, 20) + insert_or_update_stats_for_template(sample_template.id, 2, 2017, 30) + + stats_by_month = StatsTemplateUsageByMonth.query.filter( + StatsTemplateUsageByMonth.template_id == sample_template.id + ).order_by(StatsTemplateUsageByMonth.template_id).all() + + assert len(stats_by_month) == 2 + + assert stats_by_month[0].template_id == sample_template.id + assert stats_by_month[0].month == 1 + assert stats_by_month[0].year == 2017 + assert stats_by_month[0].count == 20 + + assert stats_by_month[1].template_id == sample_template.id + assert stats_by_month[1].month == 2 + assert stats_by_month[1].year == 2017 + assert stats_by_month[1].count == 30