From 17767dfa627cef88ef06da66cc175763ce2abb2f Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Mon, 9 Jan 2017 11:13:24 +0000 Subject: [PATCH] Calculating the billing units per month was taking too long for a services with a lot of data. The query now does the sum per month. --- app/dao/notifications_dao.py | 22 +++++++------------- tests/app/dao/test_notification_dao.py | 28 ++++++++++++++------------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index d62e62607..b5a1b6591 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,14 +1,13 @@ + import pytz from datetime import ( datetime, timedelta, - date -) -from itertools import groupby + date) from flask import current_app from werkzeug.datastructures import MultiDict -from sqlalchemy import (desc, func, or_, and_, asc) +from sqlalchemy import (desc, func, or_, and_, asc, extract) from sqlalchemy.orm import joinedload from app import db, create_uuid @@ -217,23 +216,16 @@ def get_notification_billable_unit_count_per_month(service_id, year): start, end = get_financial_year(year) notifications = db.session.query( - NotificationHistory.created_at, - NotificationHistory.billable_units - ).order_by( - NotificationHistory.created_at + func.date_trunc("month", (NotificationHistory.created_at + extract("timezone", func.timezone("Europe/London", NotificationHistory.created_at)) * timedelta(seconds=1))), + func.sum(NotificationHistory.billable_units) ).filter( NotificationHistory.billable_units != 0, NotificationHistory.service_id == service_id, NotificationHistory.created_at >= start, NotificationHistory.created_at < end - ) + ).group_by(func.date_trunc("month", (NotificationHistory.created_at + extract("timezone", func.timezone("Europe/London", NotificationHistory.created_at)) * timedelta(seconds=1)))).all() - return [ - (month, sum(count for _, count in row)) - for month, row in groupby( - notifications, lambda row: get_bst_month(row[0]) - ) - ] + return [(datetime.strftime(x[0], "%B"), x[1]) for x in notifications] @statsd(namespace="dao") diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 8e60f58a3..029435775 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -700,21 +700,23 @@ def test_get_all_notifications_for_job_by_status(notify_db, notify_db_session, s def test_get_notification_billable_unit_count_per_month(notify_db, notify_db_session, sample_service): - for year, month, day in ( - (2017, 1, 15), # ↓ 2016 financial year - (2016, 8, 1), - (2016, 7, 15), - (2016, 4, 15), - (2016, 4, 15), - (2016, 4, 1), # ↓ 2015 financial year - (2016, 3, 31), - (2016, 1, 15) + for year, month, day, hour, minute, second in ( + (2017, 1, 15, 23, 59, 59), # ↓ 2016 financial year + (2016, 9, 30, 23, 59, 59), # counts in October (BST conversion) + (2016, 6, 30, 23, 50, 20), + (2016, 7, 15, 9, 20, 25), + (2016, 4, 15, 12, 30, 00), + (2016, 4, 1, 1, 1, 00), + (2016, 4, 1, 0, 0, 00), # ↓ 2015 financial year + (2016, 3, 20, 22, 40, 45), + (2015, 11, 20, 22, 40, 45), + (2016, 1, 15, 2, 30, 40) ): sample_notification( notify_db, notify_db_session, service=sample_service, created_at=datetime( - year, month, day, 0, 0, 0, 0 - ) - timedelta(hours=1, seconds=1) # one second before midnight + year, month, day, hour, minute, second, 0 + ) ) for financial_year, months in ( @@ -724,11 +726,11 @@ def test_get_notification_billable_unit_count_per_month(notify_db, notify_db_ses ), ( 2016, - [('April', 2), ('July', 2), ('January', 1)] + [('April', 2), ('July', 2), ('October', 1), ('January', 1)] ), ( 2015, - [('January', 1), ('March', 2)] + [('November', 1), ('January', 1), ('March', 1), ('April', 1)] ), ( 2014,