diff --git a/app/dao/date_util.py b/app/dao/date_util.py new file mode 100644 index 000000000..92932bb4c --- /dev/null +++ b/app/dao/date_util.py @@ -0,0 +1,18 @@ +from datetime import datetime + +import pytz + + +def get_financial_year(year): + return get_april_fools(year), get_april_fools(year + 1) + + +def get_april_fools(year): + """ + This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, + the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. + :param year: the year to calculate the April 1, 00:00 BST for + :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 + """ + return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( + tzinfo=None) diff --git a/app/dao/notification_usage_dao.py b/app/dao/notification_usage_dao.py new file mode 100644 index 000000000..fbd53681c --- /dev/null +++ b/app/dao/notification_usage_dao.py @@ -0,0 +1,69 @@ +from decimal import Decimal + +from sqlalchemy import func + +from app import db +from app.dao.date_util import get_financial_year +from app.models import (NotificationHistory, + Rate, + NOTIFICATION_STATUS_TYPES_BILLABLE, + KEY_TYPE_TEST, + SMS_TYPE, + EMAIL_TYPE) +from app.statsd_decorators import statsd + + +@statsd(namespace="dao") +def get_yearly_billing_data(service_id, year): + start_date, end_date = get_financial_year(year) + rates = get_rates_for_year(start_date, end_date, SMS_TYPE) + + result = [] + for r, n in zip(rates, rates[1:]): + result.append( + sms_billing_data_query(str(r.rate), service_id, r.valid_from, n.valid_from)) + + result.append(sms_billing_data_query(str(rates[-1].rate), service_id, rates[-1].valid_from, end_date)) + + result.append(email_billing_data_query(service_id, start_date, end_date)) + + return result + + +def billing_data_filter(notification_type, start_date, end_date, service_id): + return [NotificationHistory.notification_type == notification_type, + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date, + NotificationHistory.service_id == service_id, + NotificationHistory.status.in_(NOTIFICATION_STATUS_TYPES_BILLABLE), + NotificationHistory.key_type != KEY_TYPE_TEST + ] + + +def email_billing_data_query(service_id, start_date, end_date): + result = db.session.query(func.count(NotificationHistory.id), + NotificationHistory.notification_type, + "0" + ).filter(*billing_data_filter(EMAIL_TYPE, start_date, end_date, service_id) + ).group_by(NotificationHistory.notification_type).first() + if not result: + return 0, EMAIL_TYPE, Decimal("0") + else: + return result + + +def sms_billing_data_query(rate, service_id, start_date, end_date): + result = db.session.query(func.sum(NotificationHistory.billable_units), + NotificationHistory.notification_type, + rate + ).filter(*billing_data_filter(SMS_TYPE, start_date, end_date, service_id) + ).group_by(NotificationHistory.notification_type).first() + if not result: + return 0, SMS_TYPE, Decimal("0") + else: + return result + + +def get_rates_for_year(start_date, end_date, notification_type): + return Rate.query.filter(Rate.valid_from >= start_date, Rate.valid_from < end_date, + Rate.notification_type == notification_type).order_by(Rate.valid_from).all() diff --git a/app/dao/notifications_dao.py b/app/dao/notifications_dao.py index 377b47e3c..6bb5337f8 100644 --- a/app/dao/notifications_dao.py +++ b/app/dao/notifications_dao.py @@ -1,5 +1,4 @@ import functools -import pytz from datetime import ( datetime, timedelta, @@ -12,6 +11,7 @@ from sqlalchemy.orm import joinedload from app import db, create_uuid from app.dao import days_ago +from app.dao.date_util import get_financial_year from app.models import ( Service, Notification, @@ -243,13 +243,15 @@ def get_notifications_for_job(service_id, job_id, filter_dict=None, page=1, page def get_notification_billable_unit_count_per_month(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) notifications = db.session.query( month, func.sum(NotificationHistory.billable_units) ).filter( NotificationHistory.billable_units != 0, NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( month ).order_by( @@ -410,21 +412,6 @@ def dao_timeout_notifications(timeout_period_in_seconds): return updated -def get_financial_year(year): - return get_april_fools(year), get_april_fools(year + 1) - - -def get_april_fools(year): - """ - This function converts the start of the financial year April 1, 00:00 as BST (British Standard Time) to UTC, - the tzinfo is lastly removed from the datetime becasue the database stores the timestamps without timezone. - :param year: the year to calculate the April 1, 00:00 BST for - :return: the datetime of April 1 for the given year, for example 2016 = 2016-03-31 23:00:00 - """ - return pytz.timezone('Europe/London').localize(datetime(year, 4, 1, 0, 0, 0)).astimezone(pytz.UTC).replace( - tzinfo=None) - - def get_total_sent_notifications_in_date_range(start_date, end_date, notification_type): result = db.session.query( func.count(NotificationHistory.id).label('count') diff --git a/app/dao/rates_dao.py b/app/dao/rates_dao.py deleted file mode 100644 index 47ed83a4e..000000000 --- a/app/dao/rates_dao.py +++ /dev/null @@ -1,11 +0,0 @@ -from sqlalchemy import desc - -from app import db -from app.models import Rate - - -def get_rate_for_type_and_date(notification_type, date_sent): - return db.session.query(Rate).filter(Rate.notification_type == notification_type, - Rate.valid_from <= date_sent - ).order_by(Rate.valid_from.desc() - ).limit(1).first() diff --git a/app/dao/services_dao.py b/app/dao/services_dao.py index 928b3188b..fe25469da 100644 --- a/app/dao/services_dao.py +++ b/app/dao/services_dao.py @@ -229,6 +229,7 @@ def _stats_for_service_query(service_id): def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) sq = db.session.query( NotificationHistory.template_id, NotificationHistory.status, @@ -236,7 +237,9 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) func.count().label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)) + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date + ).group_by( month, NotificationHistory.template_id, @@ -262,6 +265,7 @@ def dao_fetch_monthly_historical_stats_by_template_for_service(service_id, year) def dao_fetch_monthly_historical_stats_for_service(service_id, year): month = get_london_month_from_utc_column(NotificationHistory.created_at) + start_date, end_date = get_financial_year(year) rows = db.session.query( NotificationHistory.notification_type, NotificationHistory.status, @@ -269,7 +273,8 @@ def dao_fetch_monthly_historical_stats_for_service(service_id, year): func.count(NotificationHistory.id).label('count') ).filter( NotificationHistory.service_id == service_id, - NotificationHistory.created_at.between(*get_financial_year(year)), + NotificationHistory.created_at >= start_date, + NotificationHistory.created_at < end_date ).group_by( NotificationHistory.notification_type, NotificationHistory.status, diff --git a/tests/app/dao/test_date_utils.py b/tests/app/dao/test_date_utils.py new file mode 100644 index 000000000..d6be85da2 --- /dev/null +++ b/tests/app/dao/test_date_utils.py @@ -0,0 +1,13 @@ +from app.dao.date_util import get_financial_year, get_april_fools + + +def test_get_financial_year(): + start, end = get_financial_year(2000) + assert str(start) == '2000-03-31 23:00:00' + assert str(end) == '2001-03-31 23:00:00' + + +def test_get_april_fools(): + april_fools = get_april_fools(2016) + assert str(april_fools) == '2016-03-31 23:00:00' + assert april_fools.tzinfo is None diff --git a/tests/app/dao/test_notification_dao.py b/tests/app/dao/test_notification_dao.py index 8f67d2335..b5461e036 100644 --- a/tests/app/dao/test_notification_dao.py +++ b/tests/app/dao/test_notification_dao.py @@ -3,7 +3,6 @@ import uuid from functools import partial import pytest - from freezegun import freeze_time from sqlalchemy.exc import SQLAlchemyError, IntegrityError @@ -18,8 +17,7 @@ from app.models import ( NOTIFICATION_SENT, KEY_TYPE_NORMAL, KEY_TYPE_TEAM, - KEY_TYPE_TEST -) + KEY_TYPE_TEST) from app.dao.notifications_dao import ( dao_create_notification, @@ -40,8 +38,6 @@ from app.dao.notifications_dao import ( update_notification_status_by_reference, dao_delete_notifications_and_history_by_id, dao_timeout_notifications, - get_financial_year, - get_april_fools, is_delivery_slow_for_provider, dao_update_notifications_sent_to_dvla) @@ -1359,18 +1355,6 @@ def test_should_exclude_test_key_notifications_by_default( assert len(all_notifications) == 1 -def test_get_financial_year(): - start, end = get_financial_year(2000) - assert str(start) == '2000-03-31 23:00:00' - assert str(end) == '2001-03-31 23:00:00' - - -def test_get_april_fools(): - april_fools = get_april_fools(2016) - assert str(april_fools) == '2016-03-31 23:00:00' - assert april_fools.tzinfo is None - - @pytest.mark.parametrize('notification_type', ['sms', 'email']) def test_get_total_sent_notifications_in_date_range_returns_only_in_date_range( notify_db, diff --git a/tests/app/dao/test_notification_usage_dao.py b/tests/app/dao/test_notification_usage_dao.py new file mode 100644 index 000000000..39525338b --- /dev/null +++ b/tests/app/dao/test_notification_usage_dao.py @@ -0,0 +1,97 @@ +import uuid +from datetime import datetime + +from decimal import Decimal + +from app.dao.notification_usage_dao import (get_rates_for_year, get_yearly_billing_data) +from app.models import Rate +from tests.app.db import create_notification + + +def test_get_rates_for_year(notify_db, notify_db_session): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.50) + set_up_rate(notify_db, datetime(2017, 6, 1), 1.75) + rates = get_rates_for_year(datetime(2016, 4, 1), datetime(2017, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2016-04-01 00:00:00" + assert rates[0].rate == Decimal("1.50") + rates = get_rates_for_year(datetime(2017, 4, 1), datetime(2018, 3, 31), 'sms') + assert len(rates) == 1 + assert datetime.strftime(rates[0].valid_from, '%Y-%m-%d %H:%M:%S') == "2017-06-01 00:00:00" + assert rates[0].rate == Decimal("1.75") + + +def test_get_yearly_billing_data(notify_db, notify_db_session, sample_template, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + set_up_rate(notify_db, datetime(2016, 6, 1), 1.58) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=5) + create_notification(template=sample_email_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2017, 3, 31), sent_at=datetime(2017, 3, 31), + status='sending', billable_units=0) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=6) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 3 + assert results[0] == (3, 'sms', Decimal('1.4')) + assert results[1] == (12, 'sms', Decimal('1.58')) + assert results[2] == (2, 'email', Decimal("0")) + + +def test_get_yearly_billing_data_with_one_rate(notify_db, notify_db_session, sample_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + # previous year + create_notification(template=sample_template, created_at=datetime(2016, 3, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=1) + # current year + create_notification(template=sample_template, created_at=datetime(2016, 4, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=1) + create_notification(template=sample_template, created_at=datetime(2016, 5, 18), sent_at=datetime(2016, 5, 18), + status='sending', billable_units=2) + create_notification(template=sample_template, created_at=datetime(2016, 7, 22), sent_at=datetime(2016, 7, 22), + status='sending', billable_units=3) + create_notification(template=sample_template, created_at=datetime(2016, 9, 15), sent_at=datetime(2016, 9, 15), + status='sending', billable_units=4) + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 22, 59, 59), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=5) + # next year + create_notification(template=sample_template, created_at=datetime(2017, 3, 31, 23, 00, 00), + sent_at=datetime(2017, 3, 31), status='sending', billable_units=6) + create_notification(template=sample_template, created_at=datetime(2017, 4, 1), sent_at=datetime(2017, 4, 1), + status='sending', billable_units=7) + results = get_yearly_billing_data(sample_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (15, 'sms', Decimal('1.4')) + assert results[1] == (0, 'email', Decimal('0')) + + +def test_get_yearly_billing_data_with_no_sms_notifications(notify_db, notify_db_session, sample_email_template): + set_up_rate(notify_db, datetime(2016, 4, 1), 1.40) + create_notification(template=sample_email_template, created_at=datetime(2016, 7, 31), sent_at=datetime(2016, 3, 31), + status='sending', billable_units=0) + create_notification(template=sample_email_template, created_at=datetime(2016, 10, 2), sent_at=datetime(2016, 4, 2), + status='sending', billable_units=0) + + results = get_yearly_billing_data(sample_email_template.service_id, 2016) + assert len(results) == 2 + assert results[0] == (0, 'sms', Decimal('0')) + assert results[1] == (2, 'email', Decimal('0')) + + +def set_up_rate(notify_db, start_date, value): + rate = Rate(id=uuid.uuid4(), valid_from=start_date, rate=value, notification_type='sms') + notify_db.session.add(rate) diff --git a/tests/app/dao/test_rates_dao.py b/tests/app/dao/test_rates_dao.py deleted file mode 100644 index 33f68a671..000000000 --- a/tests/app/dao/test_rates_dao.py +++ /dev/null @@ -1,18 +0,0 @@ -from datetime import datetime - -from decimal import Decimal - -from app.dao.rates_dao import get_rate_for_type_and_date - - -def test_get_rate_for_type_and_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime.utcnow()) - assert rate.rate == Decimal("1.58") - - rate = get_rate_for_type_and_date('sms', datetime(2016, 6, 1)) - assert rate.rate == Decimal("1.65") - - -def test_get_rate_for_type_and_date_early_date(notify_db): - rate = get_rate_for_type_and_date('sms', datetime(2014, 6, 1)) - assert not rate diff --git a/tests/conftest.py b/tests/conftest.py index c7ec7ee3b..b08059725 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,8 +76,7 @@ def notify_db_session(notify_db): "job_status", "provider_details_history", "template_process_type", - "dvla_organisation", - "rates"]: + "dvla_organisation"]: notify_db.engine.execute(tbl.delete()) notify_db.session.commit()