From 6dc336ad6c3834729bc78474a980bd3b12627726 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 26 Apr 2017 14:16:47 +0100 Subject: [PATCH] Created new queries to return the rate with the sum of billable units for the year totals. Once we have the new columns in notifications table, the query will need to include the rate multiplier and if the number is international. The monthly billing query will be built next. --- app/dao/date_util.py | 18 ++++ app/dao/notification_usage_dao.py | 69 ++++++++++++++ app/dao/notifications_dao.py | 21 +---- app/dao/rates_dao.py | 11 --- app/dao/services_dao.py | 9 +- tests/app/dao/test_date_utils.py | 13 +++ tests/app/dao/test_notification_dao.py | 18 +--- tests/app/dao/test_notification_usage_dao.py | 97 ++++++++++++++++++++ tests/app/dao/test_rates_dao.py | 18 ---- tests/conftest.py | 3 +- 10 files changed, 210 insertions(+), 67 deletions(-) create mode 100644 app/dao/date_util.py create mode 100644 app/dao/notification_usage_dao.py delete mode 100644 app/dao/rates_dao.py create mode 100644 tests/app/dao/test_date_utils.py create mode 100644 tests/app/dao/test_notification_usage_dao.py delete mode 100644 tests/app/dao/test_rates_dao.py diff --git a/app/dao/date_util.py b/app/dao/date_util.py new file mode 100644 index 000000000..92932bb4c --- /dev/null +++ b/app/dao/date_util.py @@ -0,0 +1,18 @@ +from datetime import datetime + +import pytz + + +def get_financial_year(year): + return get_april_fools(year), get_april_fools(year + 1) + + +def get_april_fools(year): + """ + This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, + the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. + :param year: the year to calculate the April 1, 00:00 BST for + :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 + """ + return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( + tzinfo=None) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py new file mode 100644 index 000000000..fbd53681c --- /dev/null +++ b/app/dao/notification_usage_dao.py @@ -0,0 +1,69 @@ +from decimal import Decimal + +from sqlalchemy import func + +from app import db +from app.dao.date_util import get_financial_year +from app.models import (NotificationHistory, + Rate, + NOTIFICATION_STATUS_TYPES_BILLABLE, + KEY_TYPE_TEST, + SMS_TYPE, + EMAIL_TYPE) +from app.statsd_decorators import statsd + + +@statsd(namespace="dao") +def get_yearly_billing_data(service_id, year): + start_date, end_date = get_financial_year(year) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + result = [] + for r, n in zip(rates, rates[1:]): + result.append( + sms_billing_data_query(str(r.rate), service_id, r.valid_from, n.valid_from)) + + result.append(sms_billing_data_query(str(rates[-1].rate), service_id, rates[-1].valid_from, end_date)) + + result.append(email_billing_data_query(service_id, start_date, end_date)) + + return result + + +def billing_data_filter(notification_type, start_date, end_date, service_id): + return [NotificationHistory.notification_type == notification_type, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.service_id == service_id, + NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE), + NotificationHistory.key_type != KEY_TYPE_TEST + ] + + +def email_billing_data_query(service_id, start_date, end_date): + result = db.session.query(func.count(NotificationHistory.id), + NotificationHistory.notification_type, + "0" + ).filter(*billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) + ).group_by(NotificationHistory.notification_type).first() + if not result: + return 0, EMAIL_TYPE, Decimal("0") + else: + return result + + +def sms_billing_data_query(rate, service_id, start_date, end_date): + result = db.session.query(func.sum(NotificationHistory.billable_units), + NotificationHistory.notification_type, + rate + ).filter(*billing_data_filter(SMS_TYPE, start_date, end_date, service_id) + ).group_by(NotificationHistory.notification_type).first() + if not result: + return 0, SMS_TYPE, Decimal("0") + else: + return result + + +def get_rates_for_year(start_date, end_date, notification_type): + return Rate.query.filter(Rate.valid_from >= start_date, Rate.valid_from < end_date, + Rate.notification_type == notification_type).order_by(Rate.valid_from).all() diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 377b47e3c..6bb5337f8 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,5 +1,4 @@ import functools -import pytz from datetime import ( datetime, timedelta, @@ -12,6 +11,7 @@ from sqlalchemy.orm import joinedload from app import db, create_uuid from app.dao import days_ago +from app.dao.date_util import get_financial_year from app.models import ( Service, Notification, @@ -243,13 +243,15 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page def get_notification_billable_unit_count_per_month(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) notifications = db.session.query( month, func.sum(NotificationHistory.billable_units) ).filter( NotificationHistory.billable_units != 0, NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( month ).order_by( @@ -410,21 +412,6 @@ def dao_timeout_notifications(timeout_period_in_seconds): return updated -def get_financial_year(year): - return get_april_fools(year), get_april_fools(year + 1) - - -def get_april_fools(year): - """ - This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. - :param year: the year to calculate the April 1, 00:00 BST for - :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 - """ - return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( - tzinfo=None) - - def get_total_sent_notifications_in_date_range(start_date, end_date, notification_type): result = db.session.query( func.count(NotificationHistory.id).label('count') diff --git a/app/dao/rates_dao.py b/app/dao/rates_dao.py deleted file mode 100644 index 47ed83a4e..000000000 --- a/app/dao/rates_dao.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import desc - -from app import db -from app.models import Rate - - -def get_rate_for_type_and_date(notification_type, date_sent): - return db.session.query(Rate).filter(Rate.notification_type == notification_type, - Rate.valid_from <= date_sent - ).order_by(Rate.valid_from.desc() - ).limit(1).first() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 928b3188b..fe25469da 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -229,6 +229,7 @@ def _stats_for_service_query(service_id): def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) sq = db.session.query( NotificationHistory.template_id, NotificationHistory.status, @@ -236,7 +237,9 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) func.count().label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)) + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date + ).group_by( month, NotificationHistory.template_id, @@ -262,6 +265,7 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) def dao_fetch_monthly_historical_stats_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) rows = db.session.query( NotificationHistory.notification_type, NotificationHistory.status, @@ -269,7 +273,8 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year): func.count(NotificationHistory.id).label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( NotificationHistory.notification_type, NotificationHistory.status, diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py new file mode 100644 index 000000000..d6be85da2 --- /dev/null +++ b/tests/app/dao/test_date_utils.py @@ -0,0 +1,13 @@ +from app.dao.date_util import get_financial_year, get_april_fools + + +def test_get_financial_year(): + start, end = get_financial_year(2000) + assert str(start) == '2000-03-31 23:00:00' + assert str(end) == '2001-03-31 23:00:00' + + +def test_get_april_fools(): + april_fools = get_april_fools(2016) + assert str(april_fools) == '2016-03-31 23:00:00' + assert april_fools.tzinfo is None diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 8f67d2335..b5461e036 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -3,7 +3,6 @@ import uuid from functools import partial import pytest - from freezegun import freeze_time from sqlalchemy.exc import SQLAlchemyError, IntegrityError @@ -18,8 +17,7 @@ from app.models import ( NOTIFICATION_SENT, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, - KEY_TYPE_TEST -) + KEY_TYPE_TEST) from app.dao.notifications_dao import ( dao_create_notification, @@ -40,8 +38,6 @@ from app.dao.notifications_dao import ( update_notification_status_by_reference, dao_delete_notifications_and_history_by_id, dao_timeout_notifications, - get_financial_year, - get_april_fools, is_delivery_slow_for_provider, dao_update_notifications_sent_to_dvla) @@ -1359,18 +1355,6 @@ def test_should_exclude_test_key_notifications_by_default( assert len(all_notifications) == 1 -def test_get_financial_year(): - start, end = get_financial_year(2000) - assert str(start) == '2000-03-31 23:00:00' - assert str(end) == '2001-03-31 23:00:00' - - -def test_get_april_fools(): - april_fools = get_april_fools(2016) - assert str(april_fools) == '2016-03-31 23:00:00' - assert april_fools.tzinfo is None - - @pytest.mark.parametrize('notification_type', ['sms', 'email']) def test_get_total_sent_notifications_in_date_range_returns_only_in_date_range( notify_db, diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py new file mode 100644 index 000000000..39525338b --- /dev/null +++ b/tests/app/dao/test_notification_usage_dao.py @@ -0,0 +1,97 @@ +import uuid +from datetime import datetime + +from decimal import Decimal + +from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data) +from app.models import Rate +from tests.app.db import create_notification + + +def test_get_rates_for_year(notify_db, notify_db_session): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.50) + set_up_rate(notify_db, datetime(2017, 6, 1), 1.75) + rates = get_rates_for_year(datetime(2016, 4, 1), datetime(2017, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" + assert rates[0].rate == Decimal("1.50") + rates = get_rates_for_year(datetime(2017, 4, 1), datetime(2018, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" + assert rates[0].rate == Decimal("1.75") + + +def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + set_up_rate(notify_db, datetime(2016, 6, 1), 1.58) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=5) + create_notification(template=sample_email_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=0) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=6) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 3 + assert results[0] == (3, 'sms', Decimal('1.4')) + assert results[1] == (12, 'sms', Decimal('1.58')) + assert results[2] == (2, 'email', Decimal("0")) + + +def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 22, 59, 59), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=5) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=7) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (15, 'sms', Decimal('1.4')) + assert results[1] == (0, 'email', Decimal('0')) + + +def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + create_notification(template=sample_email_template, created_at=datetime(2016, 7, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2016, 10, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=0) + + results = get_yearly_billing_data(sample_email_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (0, 'sms', Decimal('0')) + assert results[1] == (2, 'email', Decimal('0')) + + +def set_up_rate(notify_db, start_date, value): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') + notify_db.session.add(rate) diff --git a/tests/app/dao/test_rates_dao.py b/tests/app/dao/test_rates_dao.py deleted file mode 100644 index 33f68a671..000000000 --- a/tests/app/dao/test_rates_dao.py +++ /dev/null @@ -1,18 +0,0 @@ -from datetime import datetime - -from decimal import Decimal - -from app.dao.rates_dao import get_rate_for_type_and_date - - -def test_get_rate_for_type_and_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime.utcnow()) - assert rate.rate == Decimal("1.58") - - rate = get_rate_for_type_and_date('sms', datetime(2016, 6, 1)) - assert rate.rate == Decimal("1.65") - - -def test_get_rate_for_type_and_date_early_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime(2014, 6, 1)) - assert not rate diff --git a/tests/conftest.py b/tests/conftest.py index c7ec7ee3b..b08059725 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,7 @@ def notify_db_session(notify_db): "job_status", "provider_details_history", "template_process_type", - "dvla_organisation", - "rates"]: + "dvla_organisation"]: notify_db.engine.execute(tbl.delete()) notify_db.session.commit()