diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 605f1ac18..8bbc1d942 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -13,7 +13,6 @@ from app.dao.date_util import ( from app.models import ( FactBilling, - Notification, Service, KEY_TYPE_TEST, LETTER_TYPE, @@ -27,7 +26,7 @@ from app.models import ( AnnualBilling, Organisation, ) -from app.utils import get_london_midnight_in_utc +from app.utils import get_london_midnight_in_utc, get_notification_table_to_use def fetch_sms_free_allowance_remainder(start_date): @@ -227,9 +226,12 @@ def fetch_billing_totals_for_year(service_id, year): def fetch_monthly_billing_for_year(service_id, year): - year_start_date, year_end_date = get_financial_year(year) - utcnow = datetime.utcnow() - today = convert_utc_to_bst(utcnow) + year_start_datetime, year_end_datetime = get_financial_year(year) + + year_start_date = convert_utc_to_bst(year_start_datetime).date() + year_end_date = convert_utc_to_bst(year_end_datetime).date() + + today = convert_utc_to_bst(datetime.utcnow()).date() # if year end date is less than today, we are calculating for data in the past and have no need for deltas. if year_end_date >= today: yesterday = today - timedelta(days=1) @@ -303,27 +305,21 @@ def fetch_billing_data_for_day(process_day, service_id=None): current_app.logger.info("Populate ft_billing for {} to {}".format(start_date, end_date)) transit_data = [] if not service_id: - service_ids = [x.id for x in Service.query.all()] + services = Service.query.all() else: - service_ids = [service_id] - for id_of_service in service_ids: + services = [Service.query.get(service_id)] + + for service in services: for notification_type in (SMS_TYPE, EMAIL_TYPE, LETTER_TYPE): + table = get_notification_table_to_use(service, notification_type, process_day) + results = _query_for_billing_data( - table=Notification, + table=table, notification_type=notification_type, start_date=start_date, end_date=end_date, - service_id=id_of_service + service_id=service.id ) - # If data has been purged from Notification then use NotificationHistory - if len(results) == 0: - results = _query_for_billing_data( - table=NotificationHistory, - notification_type=notification_type, - start_date=start_date, - end_date=end_date, - service_id=id_of_service - ) transit_data += results diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 7c9b94439..ae805e39a 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -1,4 +1,3 @@ -from calendar import monthrange from decimal import Decimal from datetime import datetime, timedelta, date @@ -23,7 +22,6 @@ from app.dao.fact_billing_dao import ( from app.dao.organisation_dao import dao_add_service_to_organisation from app.models import ( FactBilling, - Notification, NOTIFICATION_STATUS_TYPES, ) from tests.app.db import ( @@ -36,7 +34,8 @@ from tests.app.db import ( create_notification_history, create_annual_billing, create_organisation, - set_up_usage_data + create_service_data_retention, + set_up_usage_data, ) @@ -45,33 +44,27 @@ def set_up_yearly_data(): sms_template = create_template(service=service, template_type="sms") email_template = create_template(service=service, template_type="email") letter_template = create_template(service=service, template_type="letter") - for year in (2016, 2017): - for month in range(1, 13): - mon = str(month).zfill(2) - for day in range(1, monthrange(year, month)[1] + 1): - d = str(day).zfill(2) - create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d), - service=service, - template=sms_template, - notification_type='sms', - rate=0.162) - create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d), - service=service, - template=email_template, - notification_type='email', - rate=0) - create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d), - service=service, - template=letter_template, - notification_type='letter', - rate=0.33, - postage='second') - create_ft_billing(bst_date='{}-{}-{}'.format(year, mon, d), - service=service, - template=letter_template, - notification_type='letter', - rate=0.30, - postage='second') + + start_date = date(2016, 3, 31) + end_date = date(2017, 4, 2) + + for n in range((end_date - start_date).days): + dt = start_date + timedelta(days=n) + + create_ft_billing(bst_date=dt, + template=sms_template, + rate=0.162) + create_ft_billing(bst_date=dt, + template=email_template, + rate=0) + create_ft_billing(bst_date=dt, + template=letter_template, + rate=0.33, + postage='second') + create_ft_billing(bst_date=dt, + template=letter_template, + rate=0.30, + postage='second') return service @@ -82,11 +75,11 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify create_notification(template=template, status=status) today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert results == [] for status in ['delivered', 'sending', 'temporary-failure']: create_notification(template=template, status=status) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 1 assert results[0].notifications_sent == 3 @@ -98,7 +91,7 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(noti create_notification(template=template, status='delivered', key_type=key_type) today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 1 assert results[0].notifications_sent == 2 @@ -115,7 +108,7 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_d create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1)) day_under_test = convert_utc_to_bst(process_day) - results = fetch_billing_data_for_day(day_under_test) + results = fetch_billing_data_for_day(day_under_test.date()) assert len(results) == 1 assert results[0].notifications_sent == 2 @@ -128,7 +121,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type create_notification(template=sms_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1 @@ -143,7 +136,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session): create_notification(template=sms_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1 @@ -156,7 +149,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session): create_notification(template=template, status='delivered', sent_by='firetext') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1 @@ -169,7 +162,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_rate_mulitplier(notify_db_sess create_notification(template=template, status='delivered', rate_multiplier=2) today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1 @@ -182,7 +175,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_international(notify_db_sessio create_notification(template=template, status='delivered', international=False) today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].notifications_sent == 1 assert results[1].notifications_sent == 1 @@ -201,7 +194,7 @@ def test_fetch_billing_data_for_day_is_grouped_by_notification_type(notify_db_se create_notification(template=letter_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 3 notification_types = [x[2] for x in results if x[2] in ['email', 'sms', 'letter']] assert len(notification_types) == 3 @@ -217,7 +210,7 @@ def test_fetch_billing_data_for_day_groups_by_postage(notify_db_session): create_notification(template=email_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 3 @@ -231,7 +224,7 @@ def test_fetch_billing_data_for_day_groups_by_sent_by(notify_db_session): create_notification(template=email_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 @@ -245,7 +238,7 @@ def test_fetch_billing_data_for_day_groups_by_page_count(notify_db_session): create_notification(template=email_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 3 @@ -257,7 +250,7 @@ def test_fetch_billing_data_for_day_sets_postage_for_emails_and_sms_to_none(noti create_notification(template=email_template, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert len(results) == 2 assert results[0].postage == 'none' assert results[1].postage == 'none' @@ -265,23 +258,26 @@ def test_fetch_billing_data_for_day_sets_postage_for_emails_and_sms_to_none(noti def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session): today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(today) + results = fetch_billing_data_for_day(today.date()) assert results == [] -def test_fetch_billing_data_for_day_uses_notification_history(notify_db_session): +def test_fetch_billing_data_for_day_uses_correct_table(notify_db_session): service = create_service() + create_service_data_retention(service, notification_type='email', days_of_retention=3) sms_template = create_template(service=service, template_type='sms') - create_notification_history(template=sms_template, status='delivered', - created_at=datetime.utcnow() - timedelta(days=8)) - create_notification_history(template=sms_template, status='delivered', - created_at=datetime.utcnow() - timedelta(days=8)) + email_template = create_template(service=service, template_type='email') - Notification.query.delete() - db.session.commit() - results = fetch_billing_data_for_day(process_day=datetime.utcnow() - timedelta(days=8), service_id=service.id) - assert len(results) == 1 - assert results[0].notifications_sent == 2 + five_days_ago = datetime.utcnow() - timedelta(days=5) + create_notification(template=sms_template, status='delivered', created_at=five_days_ago) + create_notification_history(template=email_template, status='delivered', created_at=five_days_ago) + + results = fetch_billing_data_for_day(process_day=five_days_ago.date(), service_id=service.id) + assert len(results) == 2 + assert results[0].notification_type == 'sms' + assert results[0].notifications_sent == 1 + assert results[1].notification_type == 'email' + assert results[1].notifications_sent == 1 def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_session): @@ -293,7 +289,7 @@ def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_ses create_notification(template=template_2, status='delivered') today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(process_day=today, service_id=service.id) + results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id) assert len(results) == 1 assert results[0].service_id == service.id @@ -308,7 +304,7 @@ def test_fetch_billing_data_for_day_bills_correctly_for_status(notify_db_session create_notification(template=email_template, status=status) create_notification(template=letter_template, status=status) today = convert_utc_to_bst(datetime.utcnow()) - results = fetch_billing_data_for_day(process_day=today, service_id=service.id) + results = fetch_billing_data_for_day(process_day=today.date(), service_id=service.id) sms_results = [x for x in results if x[2] == 'sms'] email_results = [x for x in results if x[2] == 'email'] diff --git a/tests/app/db.py b/tests/app/db.py index 341670b90..080ad41bc 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -675,7 +675,7 @@ def create_daily_sorted_letter(billing_day=date(2018, 1, 18), def create_ft_billing(bst_date, - notification_type, + notification_type=None, template=None, service=None, provider='test', @@ -686,10 +686,9 @@ def create_ft_billing(bst_date, notifications_sent=1, postage='none', ): - if not service: - service = create_service() - if not template: - template = create_template(service=service, template_type=notification_type) + if template: + service = template.service + notification_type = template.template_type data = FactBilling(bst_date=bst_date, service_id=service.id,