From 5ba58031b18b9cb801ce45bcd9f55505abaf3ad1 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Thu, 16 Nov 2017 13:51:42 +0000 Subject: [PATCH 1/4] Fixed Bug in stats aggregation The check for aggregation was too broad and hence was adding together totals based on template_id and not the unqiue combination of template id, month and year. - Added test to test for the failure - Added check and a test to for template_id, mon and year matches - Celery process name did not match the task --- app/dao/services_dao.py | 6 ++- tests/app/dao/test_services_dao.py | 62 ++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 2 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 0fe8be7d0..396f2a762 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -571,7 +571,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) extract('year', year).label('year'), func.count().label('count') ).join( - Template, Notification.template_id == Template.id + Template, Notification.template_id == Template.id, ).filter( Notification.created_at >= start_date ).group_by( @@ -586,7 +586,9 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) for today_result in today_results: add_to_stats = True for stat in stats: - if today_result.template_id == stat.template_id: + if today_result.template_id == stat.template_id and \ + today_result.month == stat.month and \ + today_result.year == stat.year: stat.count = stat.count + today_result.count add_to_stats = False diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index f880ba4e3..0539ca1cb 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1292,3 +1292,65 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea ) assert len(result) == 2 + + +@freeze_time("2017-11-10 11:09:00.000000") +def test_dao_fetch_monthly_historical_usage_by_template_for_service_combined_historical_current( + notify_db, + notify_db_session, + sample_service +): + notification_history = functools.partial( + create_notification_history, + notify_db, + notify_db_session, + status='delivered' + ) + + template_one = create_sample_template(notify_db, notify_db_session, template_name='1') + + date = datetime.now() + day = date.day + month = date.month + year = date.year + + n = notification_history(created_at=datetime(year, month, day) - timedelta(days=30), sample_template=template_one) + + daily_stats_template_usage_by_month() + + result = sorted( + dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), + key=lambda x: (x.month, x.year) + ) + + assert len(result) == 1 + + assert result[0].template_id == template_one.id + assert result[0].month == 10 + assert result[0].year == 2017 + assert result[0].count == 1 + + create_notification( + notify_db, + notify_db_session, + service=sample_service, + template=template_one, + created_at=datetime.utcnow() + ) + + result = sorted( + dao_fetch_monthly_historical_usage_by_template_for_service(n.service_id, 2017), + key=lambda x: (x.month, x.year) + ) + + assert len(result) == 2 + + assert result[0].template_id == template_one.id + assert result[0].month == 10 + assert result[0].year == 2017 + assert result[0].count == 1 + + assert result[1].template_id == template_one.id + assert result[1].month == 11 + assert result[1].year == 2017 + assert result[1].count == 1 From e7de0c0900ccf2ab20c11ddc6e025c008bf8a422 Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Thu, 16 Nov 2017 14:41:07 +0000 Subject: [PATCH 2/4] Added a template_type to the rest call Added a template type which is required for the admin UI and updated the tests. The rest tests needed updated because of the bug fix for aggregation. --- app/config.py | 2 +- app/dao/services_dao.py | 4 +++ app/dao/stats_template_usage_by_month_dao.py | 1 + app/service/rest.py | 1 + tests/app/dao/test_services_dao.py | 26 ++++++++++++++++ .../test_stats_template_usage_by_month_dao.py | 1 + tests/app/service/test_rest.py | 31 ++++++++++++++++--- 7 files changed, 60 insertions(+), 6 deletions(-) diff --git a/app/config.py b/app/config.py index 86a635985..3ce5bca68 100644 --- a/app/config.py +++ b/app/config.py @@ -243,7 +243,7 @@ class Config(object): 'options': {'queue': QueueNames.PERIODIC} }, 'daily-stats-template-usage-by-month': { - 'task': 'daily-stats-template_usage_by_month', + 'task': 'daily-stats-template-usage-by-month', 'schedule': crontab(hour=0, minute=50), 'options': {'queue': QueueNames.PERIODIC} } diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 396f2a762..302cd8952 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -554,6 +554,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) for result in results: stat = type("", (), {})() stat.template_id = result.template_id + stat.template_type = result.template_type stat.name = str(result.name) stat.month = result.month stat.year = result.year @@ -567,6 +568,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) today_results = db.session.query( Notification.template_id, Template.name, + Template.template_type, extract('month', month).label('month'), extract('year', year).label('year'), func.count().label('count') @@ -577,6 +579,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) ).group_by( Notification.template_id, Template.name, + Template.template_type, month, year ).order_by( @@ -595,6 +598,7 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) if add_to_stats: new_stat = type("", (), {})() new_stat.template_id = today_result.template_id + new_stat.template_type = today_result.template_type new_stat.name = today_result.name new_stat.month = today_result.month new_stat.year = today_result.year diff --git a/app/dao/stats_template_usage_by_month_dao.py b/app/dao/stats_template_usage_by_month_dao.py index bacc7a900..84d2550a6 100644 --- a/app/dao/stats_template_usage_by_month_dao.py +++ b/app/dao/stats_template_usage_by_month_dao.py @@ -34,6 +34,7 @@ def dao_get_template_usage_stats_by_service(service_id, year): return db.session.query( StatsTemplateUsageByMonth.template_id, Template.name, + Template.template_type, StatsTemplateUsageByMonth.month, StatsTemplateUsageByMonth.year, StatsTemplateUsageByMonth.count diff --git a/app/service/rest.py b/app/service/rest.py index 250430d7f..b4f25287f 100644 --- a/app/service/rest.py +++ b/app/service/rest.py @@ -539,6 +539,7 @@ def get_monthly_template_usage(service_id): { 'template_id': str(i.template_id), 'name': i.name, + 'type': i.template_type, 'month': i.month, 'year': i.year, 'count': i.count diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 0539ca1cb..3f9418a78 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1069,11 +1069,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_no_stats_tod assert len(result) == 2 assert result[0].template_id == template_two.id + assert result[0].name == template_two.name + assert result[0].template_type == template_two.template_type assert result[0].month == 4 assert result[0].year == 2017 assert result[0].count == 2 assert result[1].template_id == template_one.id + assert result[1].name == template_one.name + assert result[1].template_type == template_two.template_type assert result[1].month == 10 assert result[1].year == 2017 assert result[1].count == 1 @@ -1115,11 +1119,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_add_to_histo assert len(result) == 2 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == 11 assert result[1].year == 2017 assert result[1].count == 2 @@ -1147,16 +1155,22 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_add_to_histo assert len(result) == 3 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == month assert result[1].year == year assert result[1].count == 3 assert result[2].template_id == template_three.id + assert result[2].name == template_three.name + assert result[2].template_type == template_three.template_type assert result[2].month == 11 assert result[2].year == 2017 assert result[2].count == 1 @@ -1198,11 +1212,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_add_old assert len(result) == 2 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 9 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_two.id + assert result[1].name == template_two.name + assert result[1].template_type == template_two.template_type assert result[1].month == 11 assert result[1].year == 2017 assert result[1].count == 2 @@ -1259,6 +1277,8 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea assert len(result) == 1 assert result[0].template_id == template_two.id + assert result[0].name == template_two.name + assert result[0].template_type == template_two.template_type assert result[0].month == 11 assert result[0].year == 2017 assert result[0].count == 2 @@ -1326,6 +1346,8 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_combined_his assert len(result) == 1 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 10 assert result[0].year == 2017 assert result[0].count == 1 @@ -1346,11 +1368,15 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_combined_his assert len(result) == 2 assert result[0].template_id == template_one.id + assert result[0].name == template_one.name + assert result[0].template_type == template_one.template_type assert result[0].month == 10 assert result[0].year == 2017 assert result[0].count == 1 assert result[1].template_id == template_one.id + assert result[1].name == template_one.name + assert result[1].template_type == template_one.template_type assert result[1].month == 11 assert result[1].year == 2017 assert result[1].count == 1 diff --git a/tests/app/dao/test_stats_template_usage_by_month_dao.py b/tests/app/dao/test_stats_template_usage_by_month_dao.py index cd21e3af6..99112c6ca 100644 --- a/tests/app/dao/test_stats_template_usage_by_month_dao.py +++ b/tests/app/dao/test_stats_template_usage_by_month_dao.py @@ -99,6 +99,7 @@ def test_dao_get_template_usage_stats_by_service_specific_year(sample_service): assert len(result) == 1 assert result[0].template_id == email_template.id assert result[0].name == email_template.name + assert result[0].template_type == email_template.template_type assert result[0].month == 2 assert result[0].year == 2017 assert result[0].count == 10 diff --git a/tests/app/service/test_rest.py b/tests/app/service/test_rest.py index f7655be72..fad550593 100644 --- a/tests/app/service/test_rest.py +++ b/tests/app/service/test_rest.py @@ -1865,6 +1865,7 @@ def test_get_template_stats_by_month_returns_correct_data(notify_db, notify_db_s assert resp_json["2016-05"][str(sample_template.id)]["counts"]["permanent-failure"] == 1 +@freeze_time('2017-11-11 02:00') def test_get_template_usage_by_month_returns_correct_data( notify_db, notify_db_session, @@ -1918,15 +1919,24 @@ def test_get_template_usage_by_month_returns_correct_data( resp_json = json.loads(resp.get_data(as_text=True)).get('stats') assert resp.status_code == 200 - assert len(resp_json) == 1 + assert len(resp_json) == 2 assert resp_json[0]["template_id"] == str(sample_template.id) assert resp_json[0]["name"] == sample_template.name + assert resp_json[0]["type"] == sample_template.template_type assert resp_json[0]["month"] == 4 assert resp_json[0]["year"] == 2016 - assert resp_json[0]["count"] == 5 + assert resp_json[0]["count"] == 4 + + assert resp_json[1]["template_id"] == str(sample_template.id) + assert resp_json[1]["name"] == sample_template.name + assert resp_json[1]["type"] == sample_template.template_type + assert resp_json[1]["month"] == 11 + assert resp_json[1]["year"] == 2017 + assert resp_json[1]["count"] == 1 +@freeze_time('2017-11-11 02:00') def test_get_template_usage_by_month_returns_two_templates( notify_db, notify_db_session, @@ -1983,19 +1993,30 @@ def test_get_template_usage_by_month_returns_two_templates( resp_json = json.loads(resp.get_data(as_text=True)).get('stats') assert resp.status_code == 200 - assert len(resp_json) == 2 + assert len(resp_json) == 3 - resp_json = sorted(resp_json, key=lambda k: k.get('count', 0)) + resp_json = sorted(resp_json, key=lambda k: (k.get('year', 0), k.get('month', 0), k.get('count', 0))) assert resp_json[0]["template_id"] == str(template_one.id) + assert resp_json[0]["name"] == template_one.name + assert resp_json[0]["type"] == template_one.template_type assert resp_json[0]["month"] == 4 assert resp_json[0]["year"] == 2016 assert resp_json[0]["count"] == 1 assert resp_json[1]["template_id"] == str(sample_template.id) + assert resp_json[1]["name"] == sample_template.name + assert resp_json[1]["type"] == sample_template.template_type assert resp_json[1]["month"] == 4 assert resp_json[1]["year"] == 2016 - assert resp_json[1]["count"] == 4 + assert resp_json[1]["count"] == 3 + + assert resp_json[2]["template_id"] == str(sample_template.id) + assert resp_json[2]["name"] == sample_template.name + assert resp_json[2]["type"] == sample_template.template_type + assert resp_json[2]["month"] == 11 + assert resp_json[2]["year"] == 2017 + assert resp_json[2]["count"] == 1 @pytest.mark.parametrize('query_string, expected_status, expected_json', [ From fd89a252e0897577464d6848706dd97261aad3da Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Thu, 16 Nov 2017 15:16:31 +0000 Subject: [PATCH 3/4] Code Style Updates Updated the to conform with the line ends expected by the code style in tests rather than PyCharm. --- app/dao/services_dao.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 302cd8952..0b91e2267 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -589,9 +589,8 @@ def dao_fetch_monthly_historical_usage_by_template_for_service(service_id, year) for today_result in today_results: add_to_stats = True for stat in stats: - if today_result.template_id == stat.template_id and \ - today_result.month == stat.month and \ - today_result.year == stat.year: + if today_result.template_id == stat.template_id and today_result.month == stat.month \ + and today_result.year == stat.year: stat.count = stat.count + today_result.count add_to_stats = False From b56825a680080c00f1eb6925883abfe14c710e0a Mon Sep 17 00:00:00 2001 From: Richard Chapman Date: Thu, 16 Nov 2017 15:43:21 +0000 Subject: [PATCH 4/4] Updated Tests to use a variety of templates All of the tests defaulted to sms templates, updated a few to use a variety of templates types so all types are tested. --- tests/app/dao/test_services_dao.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/app/dao/test_services_dao.py b/tests/app/dao/test_services_dao.py index 3f9418a78..05c564c02 100644 --- a/tests/app/dao/test_services_dao.py +++ b/tests/app/dao/test_services_dao.py @@ -1189,9 +1189,9 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_does_add_old status='delivered' ) - template_one = create_sample_template(notify_db, notify_db_session, template_name='1') - template_two = create_sample_template(notify_db, notify_db_session, template_name='2') - template_three = create_sample_template(notify_db, notify_db_session, template_name='3') + template_one = create_sample_template(notify_db, notify_db_session, template_name='1', template_type='email') + template_two = create_sample_template(notify_db, notify_db_session, template_name='2', template_type='sms') + template_three = create_sample_template(notify_db, notify_db_session, template_name='3', template_type='letter') date = datetime.now() day = date.day @@ -1254,9 +1254,9 @@ def test_dao_fetch_monthly_historical_usage_by_template_for_service_get_this_yea status='delivered' ) - template_one = create_sample_template(notify_db, notify_db_session, template_name='1') - template_two = create_sample_template(notify_db, notify_db_session, template_name='2') - template_three = create_sample_template(notify_db, notify_db_session, template_name='3') + template_one = create_sample_template(notify_db, notify_db_session, template_name='1', template_type='email') + template_two = create_sample_template(notify_db, notify_db_session, template_name='2', template_type='sms') + template_three = create_sample_template(notify_db, notify_db_session, template_name='3', template_type='letter') date = datetime.now() day = date.day