diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 798b8197a..6b5bc2a1d 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,5 +1,6 @@ import math -from sqlalchemy import desc, func +from sqlalchemy import (desc, func, Integer) +from sqlalchemy.sql.expression import cast from datetime import ( datetime, @@ -51,6 +52,33 @@ def dao_get_notification_statistics_for_service_and_day(service_id, day): ).order_by(desc(NotificationStatistics.day)).first() +def dao_get_7_day_agg_notification_statistics_for_service(service_id, + date_from, + week_count=52): + doy = date_from.timetuple().tm_yday + return db.session.query( + cast(func.floor((func.extract('doy', NotificationStatistics.day) - doy) / 7), Integer), + cast(func.sum(NotificationStatistics.emails_requested), Integer), + cast(func.sum(NotificationStatistics.emails_delivered), Integer), + cast(func.sum(NotificationStatistics.emails_failed), Integer), + cast(func.sum(NotificationStatistics.sms_requested), Integer), + cast(func.sum(NotificationStatistics.sms_delivered), Integer), + cast(func.sum(NotificationStatistics.sms_failed), Integer) + ).filter( + NotificationStatistics.service_id == service_id + ).filter( + NotificationStatistics.day >= date_from + ).filter( + NotificationStatistics.day < date_from + timedelta(days=7 * week_count) + ).group_by( + func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7)) + ).order_by( + desc(func.floor(((func.extract('doy', NotificationStatistics.day) - doy) / 7))) + ).limit( + week_count + ) + + def dao_get_template_statistics_for_service(service_id, limit_days=None): filter = [TemplateStatistics.service_id == service_id] if limit_days is not None: diff --git a/app/notifications_statistics/rest.py b/app/notifications_statistics/rest.py index f4e0cea74..f864e473b 100644 --- a/app/notifications_statistics/rest.py +++ b/app/notifications_statistics/rest.py @@ -1,11 +1,18 @@ +from datetime import (date, timedelta) from flask import ( Blueprint, jsonify, request ) -from app.dao.notifications_dao import dao_get_notification_statistics_for_service -from app.schemas import notifications_statistics_schema +from app.dao.notifications_dao import ( + dao_get_notification_statistics_for_service, + dao_get_7_day_agg_notification_statistics_for_service +) +from app.schemas import ( + notifications_statistics_schema, + week_aggregate_notification_statistics_schema +) notifications_statistics = Blueprint( 'notifications-statistics', @@ -35,3 +42,32 @@ def get_all_notification_statistics_for_service(service_id): data, errors = notifications_statistics_schema.dump(statistics, many=True) return jsonify(data=data) + + +@notifications_statistics.route('/seven_day_aggregate') +def get_notification_statistics_for_service_seven_day_aggregate(service_id): + data, errors = week_aggregate_notification_statistics_schema.load(request.args) + if errors: + return jsonify(result='error', message=errors), 400 + date_from = data['date_from'] if 'date_from' in data else date(date.today().year, 4, 1) + week_count = data['week_count'] if 'week_count' in data else 52 + stats = dao_get_7_day_agg_notification_statistics_for_service( + service_id, + date_from, + week_count).all() + json_stats = [] + for x in range(week_count - 1, -1, -1): + week_stats = stats.pop(0) if len(stats) > 0 and stats[0][0] == x else [x, 0, 0, 0, 0, 0, 0] + week_start = (date_from + timedelta(days=week_stats[0] * 7)) + if week_start <= date.today(): + json_stats.append({ + 'week_start': week_start.strftime('%Y-%m-%d'), + 'week_end': (date_from + timedelta(days=(week_stats[0] * 7) + 6)).strftime('%Y-%m-%d'), + 'emails_requested': week_stats[1], + 'emails_delivered': week_stats[2], + 'emails_failed': week_stats[3], + 'sms_requested': week_stats[4], + 'sms_delivered': week_stats[5], + 'sms_failed': week_stats[6] + }) + return jsonify(data=json_stats) diff --git a/app/schemas.py b/app/schemas.py index 4cca7b66c..d01a4e978 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -24,6 +24,20 @@ from app import models from app.dao.permissions_dao import permission_dao +def _validate_positive_number(value, msg="Not a positive integer"): + try: + page_int = int(value) + if page_int < 1: + raise ValidationError(msg) + except: + raise ValidationError(msg) + + +def _validate_not_in_future(dte, msg="Date cannot be in the future"): + if dte > date.today(): + raise ValidationError(msg) + + # TODO I think marshmallow provides a better integration and error handling. # Would be better to replace functionality in dao with the marshmallow supported # functionality. @@ -265,21 +279,13 @@ class NotificationsFilterSchema(ma.Schema): in_data['status'] = [x.status for x in in_data['status']] return in_data - def _validate_positive_number(self, value): - try: - page_int = int(value) - if page_int < 1: - raise ValidationError("Not a positive integer") - except: - raise ValidationError("Not a positive integer") - @validates('page') def validate_page(self, value): - self._validate_positive_number(value) + _validate_positive_number(value) @validates('page_size') def validate_page_size(self, value): - self._validate_positive_number(value) + _validate_positive_number(value) class TemplateStatisticsSchema(BaseSchema): @@ -323,17 +329,13 @@ class FromToDateSchema(ma.Schema): date_from = fields.Date() date_to = fields.Date() - def _validate_not_in_future(self, dte): - if dte > date.today(): - raise ValidationError('Date cannot be in the future') - @validates('date_from') def validate_date_from(self, value): - self._validate_not_in_future(value) + _validate_not_in_future(value) @validates('date_to') def validate_date_to(self, value): - self._validate_not_in_future(value) + _validate_not_in_future(value) @validates_schema def validate_dates(self, data): @@ -343,6 +345,20 @@ class FromToDateSchema(ma.Schema): raise ValidationError("date_from needs to be greater than date_to") +class WeekAggregateNotificationStatisticsSchema(ma.Schema): + + date_from = fields.Date() + week_count = fields.Int() + + @validates('date_from') + def validate_date_from(self, value): + _validate_not_in_future(value) + + @validates('week_count') + def validate_week_count(self, value): + _validate_positive_number(value) + + user_schema = UserSchema() user_schema_load_json = UserSchema(load_json=True) service_schema = ServiceSchema() @@ -372,3 +388,4 @@ api_key_history_schema = ApiKeyHistorySchema() template_history_schema = TemplateHistorySchema() event_schema = EventSchema() from_to_date_schema = FromToDateSchema() +week_aggregate_notification_statistics_schema = WeekAggregateNotificationStatisticsSchema() diff --git a/tests/app/conftest.py b/tests/app/conftest.py index bcfe45dd5..04e38a9e6 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -13,7 +13,8 @@ from app.models import ( MMG_PROVIDER, SES_PROVIDER, TWILIO_PROVIDER, - ProviderStatistics) + ProviderStatistics, + NotificationStatistics) from app.dao.users_dao import (save_model_user, create_user_code, create_secret_code) from app.dao.services_dao import (dao_create_service, dao_add_user_to_service) from app.dao.templates_dao import dao_create_template @@ -493,3 +494,32 @@ def sample_provider_statistics(notify_db, notify_db.session.add(stats) notify_db.session.commit() return stats + + +@pytest.fixture(scope='function') +def sample_notification_statistics(notify_db, + notify_db_session, + service=None, + day=None, + emails_requested=2, + emails_delivered=1, + emails_failed=1, + sms_requested=2, + sms_delivered=1, + sms_failed=1): + if service is None: + service = sample_service(notify_db, notify_db_session) + if day is None: + day = date.today() + stats = NotificationStatistics( + service=service, + day=day, + emails_requested=emails_requested, + emails_delivered=emails_delivered, + emails_failed=emails_failed, + sms_requested=sms_requested, + sms_delivered=sms_delivered, + sms_failed=sms_failed) + notify_db.session.add(stats) + notify_db.session.commit() + return stats diff --git a/tests/app/dao/test_notification_statistics_dao.py b/tests/app/dao/test_notification_statistics_dao.py new file mode 100644 index 000000000..b8ffe8445 --- /dev/null +++ b/tests/app/dao/test_notification_statistics_dao.py @@ -0,0 +1,122 @@ +from datetime import (date, timedelta) + +from app.models import NotificationStatistics +from tests.app.conftest import sample_notification_statistics as create_sample_notification_statistics +from app.dao.notifications_dao import dao_get_7_day_agg_notification_statistics_for_service + + +def test_display_weekly_notification_statistics_sum_over_week(notify_db, + notify_db_session, + sample_service): + fools = date(2016, 4, 1) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + timedelta(days=1) + ) + assert dao_get_7_day_agg_notification_statistics_for_service( + sample_service.id, + fools + ).all() == [(0, 4, 2, 2, 4, 2, 2)] + + +def test_display_weekly_notification_statistics_separate_over_weeks(notify_db, + notify_db_session, + sample_service): + fools = date(2016, 4, 1) + next_week = fools + timedelta(days=7) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=next_week + ) + assert dao_get_7_day_agg_notification_statistics_for_service( + sample_service.id, + fools + ).all() == [(1, 2, 1, 1, 2, 1, 1), (0, 2, 1, 1, 2, 1, 1)] + + +def test_display_weekly_notification_statistics_7_days_from_date_from(notify_db, + notify_db_session, + sample_service): + fools = date(2016, 4, 1) + eow_fools = fools + timedelta(days=6) + next_week = fools + timedelta(days=7) + two_weeks_later = fools + timedelta(days=14) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=eow_fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=next_week + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=two_weeks_later + ) + assert dao_get_7_day_agg_notification_statistics_for_service( + sample_service.id, + fools + ).all() == [(2, 2, 1, 1, 2, 1, 1), (1, 2, 1, 1, 2, 1, 1), (0, 4, 2, 2, 4, 2, 2)] + + +def test_display_weekly_notification_statistics_week_number_misses_week(notify_db, + notify_db_session, + sample_service): + fools = date(2016, 4, 1) + two_weeks_later = fools + timedelta(days=14) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=two_weeks_later + ) + assert dao_get_7_day_agg_notification_statistics_for_service( + sample_service.id, + fools + ).all() == [(2, 2, 1, 1, 2, 1, 1), (0, 2, 1, 1, 2, 1, 1)] + + +def test_display_weekly_notification_statistics_week_limit(notify_db, + notify_db_session, + sample_service): + fools = date(2016, 4, 1) + two_weeks_later = fools + timedelta(days=14) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=fools + ) + create_sample_notification_statistics( + notify_db, + notify_db_session, + day=two_weeks_later + ) + assert dao_get_7_day_agg_notification_statistics_for_service( + sample_service.id, + fools, + 1 + ).all() == [(0, 2, 1, 1, 2, 1, 1)] diff --git a/tests/app/notifications/test_notification_statistics_rest.py b/tests/app/notifications/test_notification_statistics_rest.py new file mode 100644 index 000000000..f4212abe0 --- /dev/null +++ b/tests/app/notifications/test_notification_statistics_rest.py @@ -0,0 +1,122 @@ +import json +from datetime import (date, timedelta) +from flask import url_for +from tests import create_authorization_header +from tests.app.conftest import sample_notification_statistics as create_sample_notification_statistics + + +def test_get_notification_statistics( + notify_api, + notify_db, + notify_db_session, + sample_template, + sample_email_template +): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + + path = '/service/{}/notifications-statistics'.format(sample_email_template.service) + + auth_header = create_authorization_header( + service_id=sample_email_template.service_id) + + response = client.get(path, headers=[auth_header]) + assert response.status_code == 404 + + stats = json.loads(response.get_data(as_text=True)) + assert stats['result'] == 'error' + assert stats['message'] == 'No result found' + + +def test_get_week_aggregate_statistics(notify_api, + notify_db, + notify_db_session, + sample_service): + with notify_api.test_request_context(): + sample_notification_statistics = create_sample_notification_statistics( + notify_db, + notify_db_session, + day=date(date.today().year, 4, 1)) + with notify_api.test_client() as client: + endpoint = url_for( + 'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate', + service_id=sample_service.id) + auth_header = create_authorization_header( + service_id=sample_service.id) + + resp = client.get(endpoint, headers=[auth_header]) + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + week_len_index = len(json_resp['data']) - 1 + assert json_resp['data'][week_len_index]['emails_requested'] == 2 + assert json_resp['data'][week_len_index]['sms_requested'] == 2 + assert json_resp['data'][week_len_index]['week_start'] == date(date.today().year, 4, 1).strftime('%Y-%m-%d') + assert json_resp['data'][week_len_index]['week_end'] == date(date.today().year, 4, 7).strftime('%Y-%m-%d') + + +def test_get_week_aggregate_statistics_date_from(notify_api, + notify_db, + notify_db_session, + sample_service): + with notify_api.test_request_context(): + sample_notification_statistics = create_sample_notification_statistics( + notify_db, + notify_db_session, + day=date(date.today().year, 4, 1)) + date_from_str = date(date.today().year, 4, 1).strftime('%Y-%m-%d') + with notify_api.test_client() as client: + endpoint = url_for( + 'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate', + service_id=sample_service.id, + date_from=date_from_str) + auth_header = create_authorization_header( + service_id=sample_service.id) + + resp = client.get(endpoint, headers=[auth_header]) + assert resp.status_code == 200 + json_resp = json.loads(resp.get_data(as_text=True)) + week_len_index = len(json_resp['data']) - 1 + assert json_resp['data'][week_len_index]['emails_requested'] == 2 + assert json_resp['data'][week_len_index]['sms_requested'] == 2 + assert json_resp['data'][week_len_index]['week_start'] == date_from_str + assert json_resp['data'][week_len_index]['week_end'] == date(date.today().year, 4, 7).strftime('%Y-%m-%d') + + +def test_get_week_aggregate_statistics_date_in_future(notify_api, + notify_db, + notify_db_session, + sample_service): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + endpoint = url_for( + 'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate', + service_id=sample_service.id, + date_from=(date.today() + timedelta(days=1)).strftime('%Y-%m-%d')) + auth_header = create_authorization_header( + service_id=sample_service.id) + + resp = client.get(endpoint, headers=[auth_header]) + assert resp.status_code == 400 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp['result'] == 'error' + assert json_resp['message']['date_from'][0] == 'Date cannot be in the future' + + +def test_get_week_aggregate_statistics_invalid_week_count(notify_api, + notify_db, + notify_db_session, + sample_service): + with notify_api.test_request_context(): + with notify_api.test_client() as client: + endpoint = url_for( + 'notifications-statistics.get_notification_statistics_for_service_seven_day_aggregate', + service_id=sample_service.id, + week_count=-1) + auth_header = create_authorization_header( + service_id=sample_service.id) + + resp = client.get(endpoint, headers=[auth_header]) + assert resp.status_code == 400 + json_resp = json.loads(resp.get_data(as_text=True)) + assert json_resp['result'] == 'error' + assert json_resp['message']['week_count'][0] == 'Not a positive integer' diff --git a/tests/app/notifications/test_rest.py b/tests/app/notifications/test_rest.py index 95c407201..0f8567961 100644 --- a/tests/app/notifications/test_rest.py +++ b/tests/app/notifications/test_rest.py @@ -442,28 +442,6 @@ def test_filter_by_status_and_template_type(notify_api, assert notifications['notifications'][0]['status'] == 'delivered' -def test_get_notification_statistics( - notify_api, - notify_db, - notify_db_session, - sample_template, - sample_email_template -): - with notify_api.test_request_context(): - with notify_api.test_client() as client: - - path = '/service/{}/notifications-statistics'.format(sample_email_template.service) - - auth_header = create_authorization_header(service_id=sample_email_template.service_id) - - response = client.get(path, headers=[auth_header]) - assert response.status_code == 404 - - stats = json.loads(response.get_data(as_text=True)) - assert stats['result'] == 'error' - assert stats['message'] == 'No result found' - - def test_create_sms_should_reject_if_missing_required_fields(notify_api, sample_api_key, mocker): with notify_api.test_request_context(): with notify_api.test_client() as client: