From 8cd4e1be2970a4be54b42163f3f535188862b61a Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Fri, 6 Apr 2018 11:55:49 +0100 Subject: [PATCH 1/7] Query ft_billing aggregate for a month --- app/dao/fact_billing_dao.py | 34 ++++++++++++++++++++++++++++ tests/app/dao/test_ft_billing_dao.py | 24 ++++++++++++++++++++ tests/app/db.py | 32 ++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) create mode 100644 app/dao/fact_billing_dao.py create mode 100644 tests/app/dao/test_ft_billing_dao.py diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py new file mode 100644 index 000000000..e0173ab88 --- /dev/null +++ b/app/dao/fact_billing_dao.py @@ -0,0 +1,34 @@ +from sqlalchemy import func + +from app import db +from app.dao.date_util import get_month_start_and_end_date_in_utc +from app.models import FactBilling +from app.utils import convert_utc_to_bst + + +def fetch_annual_billing_by_month(service_id, billing_month, notification_type): + billing_month_in_bst = convert_utc_to_bst(billing_month) + start_date, end_date = get_month_start_and_end_date_in_utc(billing_month_in_bst) + + monthly_data = db.session.query( + func.sum(FactBilling.notifications_sent).label('notifications_sent'), + func.sum(FactBilling.billable_units).label('billing_units'), + FactBilling.service_id, + FactBilling.notification_type, + FactBilling.rate, + FactBilling.rate_multiplier, + FactBilling.international + ).filter( + FactBilling.notification_type == notification_type, + FactBilling.service_id == service_id, + FactBilling.bst_date >= start_date, + FactBilling.bst_date <= end_date + ).group_by( + FactBilling.service_id, + FactBilling.notification_type, + FactBilling.rate, + FactBilling.rate_multiplier, + FactBilling.international + ).all() + + return monthly_data, start_date diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py new file mode 100644 index 000000000..a5282fe1b --- /dev/null +++ b/tests/app/dao/test_ft_billing_dao.py @@ -0,0 +1,24 @@ +from datetime import datetime +from decimal import Decimal + +from app.dao.fact_billing_dao import fetch_annual_billing_by_month +from tests.app.db import create_ft_billing, create_service, create_template + + +def test_fetch_annual_billing_by_month(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for i in range(1, 32): + record = create_ft_billing(bst_date='2018-01-{}'.format(i), + service=service, + template=template, + notification_type='email') + + results, month = fetch_annual_billing_by_month(service_id=record.service_id, + billing_month=datetime(2018, 1, 1), + notification_type='email') + + assert len(results) == 1 + assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False) + assert month == datetime(2018, 1, 1) + diff --git a/tests/app/db.py b/tests/app/db.py index 553e3c188..cdea79eb9 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -34,6 +34,7 @@ from app.models import ( AnnualBilling, LetterRate, InvitedOrganisationUser, + FactBilling ) from app.dao.users_dao import save_model_user from app.dao.notifications_dao import ( @@ -524,3 +525,34 @@ def create_daily_sorted_letter(billing_day=date(2018, 1, 18), db.session.commit() return daily_sorted_letter + + +def create_ft_billing(bst_date, + notification_type, + template = None, + service = None, + provider='test', + rate_multiplier=1, + international=False, + rate=0, + billable_unit=1, + notifications_sent=1 + ): + if not service: + service = create_service() + if not template: + template = create_template(service=service, template_type=notification_type) + + data = FactBilling(bst_date=bst_date, + service_id=service.id, + template_id=template.id, + notification_type=notification_type, + provider=provider, + rate_multiplier=rate_multiplier, + international=international, + rate=rate, + billable_units=billable_unit, + notifications_sent=notifications_sent) + db.session.add(data) + db.session.commit() + return data From bd91aac763a27b521b650f95b7b7d2f761250ed1 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Mon, 9 Apr 2018 11:38:00 +0100 Subject: [PATCH 2/7] Write some pseudo code for ft billing --- app/dao/fact_billing_dao.py | 52 +++++++++++++++++++++++++++- tests/app/dao/test_ft_billing_dao.py | 22 ++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index e0173ab88..44791d595 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,7 +1,8 @@ +from datetime import datetime, timedelta from sqlalchemy import func from app import db -from app.dao.date_util import get_month_start_and_end_date_in_utc +from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year from app.models import FactBilling from app.utils import convert_utc_to_bst @@ -32,3 +33,52 @@ def fetch_annual_billing_by_month(service_id, billing_month, notification_type): ).all() return monthly_data, start_date + + +def need_deltas(start_date, end_date, service_id, notification_type): + max_fact_billing_date = db.session.query( + func.max(FactBilling.bst_date) + ).filter( + FactBilling.notification_type == notification_type, + FactBilling.service_id == service_id, + FactBilling.bst_date >= start_date, + FactBilling.bst_date <= end_date + ).one() + print(max_fact_billing_date) + return max_fact_billing_date < end_date + + +def fetch_annual_billing_for_year(service_id, year): + year_start_date, year_end_date = get_financial_year(year) + utcnow = datetime.utcnow() + today = convert_utc_to_bst(utcnow) + last_2_days = utcnow - timedelta(days=2) + last_bst_date_for_ft = convert_utc_to_bst(last_2_days) + # if year end date is less than today, we are calculating for data in the past and have no need for deltas. + if year_end_date >= today: + todays_data = get_deltas(service_id, last_2_days, today) + year_end_date = last_bst_date_for_ft + + + yearly_data = db.session.query( + FactBilling.notifications_sent, + FactBilling.billable_units, + FactBilling.service_id, + FactBilling.notification_type, + FactBilling.rate, + FactBilling.rate_multiplier, + FactBilling.international + ).filter( + FactBilling.service_id == service_id, + FactBilling.bst_date >= year_start_date, + FactBilling.bst_date <= last_bst_date_for_ft + ).all() + + # today_data + yearly_data and aggregate by month + + +def get_deltas(service_id, start_date_end_date): + # query ft_billing data using queries from create_nightly_billing + return [] + + diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index a5282fe1b..3efe0f7f2 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -1,8 +1,11 @@ from datetime import datetime from decimal import Decimal -from app.dao.fact_billing_dao import fetch_annual_billing_by_month -from tests.app.db import create_ft_billing, create_service, create_template +from freezegun import freeze_time + +from app.dao.date_util import get_month_start_and_end_date_in_utc +from app.dao.fact_billing_dao import fetch_annual_billing_by_month, need_deltas +from tests.app.db import create_ft_billing, create_service, create_template, create_notification def test_fetch_annual_billing_by_month(notify_db_session): @@ -22,3 +25,18 @@ def test_fetch_annual_billing_by_month(notify_db_session): assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False) assert month == datetime(2018, 1, 1) + +@freeze_time("2018-01-21 13:00:00") +def test_need_deltas(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for i in range(1, 21): + record = create_ft_billing(bst_date='2018-01-{}'.format(i), + service=service, + template=template, + notification_type='email') + start_date, end_date = get_month_start_and_end_date_in_utc(datetime.utcnow()) + + result = need_deltas(start_date=start_date, end_date=end_date, + service_id=service.id, notification_type='email') + assert result From 16ef133aa5aa6b991c8fa5ae0d171e134d1e84b5 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Tue, 24 Apr 2018 17:37:04 +0100 Subject: [PATCH 3/7] Refactor reporting task so that methods can be used for the annual usage page. Still a work in progress, tests are coming. --- app/celery/reporting_tasks.py | 111 ++------------ app/dao/fact_billing_dao.py | 158 ++++++++++++++++---- tests/app/celery/test_reporting_tasks.py | 19 +-- tests/app/dao/test_ft_billing_dao.py | 32 ++-- tests/app/db.py | 180 +++++++++++------------ 5 files changed, 257 insertions(+), 243 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index f3cf640cf..aca6b1ee2 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,31 +1,16 @@ from datetime import datetime, timedelta, time -from app.models import (Notification, - Rate, - NOTIFICATION_CREATED, - NOTIFICATION_TECHNICAL_FAILURE, - KEY_TYPE_TEST, - LetterRate, - FactBilling, - Service, - LETTER_TYPE, SMS_TYPE) -from app import db -from sqlalchemy import func, desc, case -from notifications_utils.statsd_decorators import statsd -from app import notify_celery + from flask import current_app +from notifications_utils.statsd_decorators import statsd + +from app import notify_celery +from app.dao.fact_billing_dao import ( + fetch_billing_data, + update_fact_billing +) from app.utils import convert_bst_to_utc -def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): - - if notification_type == LETTER_TYPE: - return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2]) - elif notification_type == SMS_TYPE: - return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1]) - else: - return 0 - - @notify_celery.task(name="create-nightly-billing") @statsd(namespace="tasks") def create_nightly_billing(day_start=None): @@ -34,91 +19,21 @@ def create_nightly_billing(day_start=None): if day_start is None: day_start = datetime.today() - timedelta(days=1) - non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in - Rate.query.order_by(desc(Rate.valid_from)).all()] - letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in - LetterRate.query.order_by(desc(LetterRate.start_date)).all()] - for i in range(0, 3): process_day = day_start - timedelta(days=i) ds = convert_bst_to_utc(datetime.combine(process_day, time.min)) de = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) - transit_data = db.session.query( - Notification.template_id, - Notification.service_id, - Notification.notification_type, - func.coalesce(Notification.sent_by, - case( - [ - (Notification.notification_type == 'letter', 'dvla'), - (Notification.notification_type == 'sms', 'unknown'), - (Notification.notification_type == 'email', 'ses') - ]), - ).label('sent_by'), - func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'), - func.coalesce(Notification.international, False).label('international'), - func.sum(Notification.billable_units).label('billable_units'), - func.count().label('notifications_sent'), - Service.crown, - ).filter( - Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available - Notification.status != NOTIFICATION_TECHNICAL_FAILURE, - Notification.key_type != KEY_TYPE_TEST, - Notification.created_at >= ds, - Notification.created_at < de - ).group_by( - Notification.template_id, - Notification.service_id, - Notification.notification_type, - 'sent_by', - Notification.rate_multiplier, - Notification.international, - Service.crown - ).join( - Service - ).all() + transit_data = fetch_billing_data(start_date=ds, end_date=de) updated_records = 0 inserted_records = 0 for data in transit_data: - update_count = FactBilling.query.filter( - FactBilling.bst_date == datetime.date(process_day), - FactBilling.template_id == data.template_id, - FactBilling.service_id == data.service_id, - FactBilling.provider == data.sent_by, # This could be zero - this is a bug that needs to be fixed. - FactBilling.rate_multiplier == data.rate_multiplier, - FactBilling.notification_type == data.notification_type, - FactBilling.international == data.international - ).update( - {"notifications_sent": data.notifications_sent, - "billable_units": data.billable_units}, - synchronize_session=False) - - if update_count == 0: - billing_record = FactBilling( - bst_date=process_day, - template_id=data.template_id, - service_id=data.service_id, - notification_type=data.notification_type, - provider=data.sent_by, - rate_multiplier=data.rate_multiplier, - international=data.international, - billable_units=data.billable_units, - notifications_sent=data.notifications_sent, - rate=get_rate(non_letter_rates, - letter_rates, - data.notification_type, - process_day, - data.crown, - data.rate_multiplier) - ) - db.session.add(billing_record) - inserted_records += 1 - - updated_records += update_count - db.session.commit() + inserted_records, updated_records = update_fact_billing(data, + inserted_records, + process_day, + updated_records) current_app.logger.info('ft_billing {} to {}: {} rows updated, {} rows inserted' .format(ds, de, updated_records, inserted_records)) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 44791d595..eeda5d485 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,9 +1,16 @@ from datetime import datetime, timedelta -from sqlalchemy import func +from sqlalchemy import func, case, desc, extract from app import db from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year -from app.models import FactBilling +from app.models import ( + FactBilling, Notification, Service, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE, + KEY_TYPE_TEST, + LETTER_TYPE, + SMS_TYPE, + Rate, + LetterRate +) from app.utils import convert_utc_to_bst @@ -35,50 +42,143 @@ def fetch_annual_billing_by_month(service_id, billing_month, notification_type): return monthly_data, start_date -def need_deltas(start_date, end_date, service_id, notification_type): - max_fact_billing_date = db.session.query( - func.max(FactBilling.bst_date) - ).filter( - FactBilling.notification_type == notification_type, - FactBilling.service_id == service_id, - FactBilling.bst_date >= start_date, - FactBilling.bst_date <= end_date - ).one() - print(max_fact_billing_date) - return max_fact_billing_date < end_date - - def fetch_annual_billing_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) utcnow = datetime.utcnow() today = convert_utc_to_bst(utcnow) - last_2_days = utcnow - timedelta(days=2) - last_bst_date_for_ft = convert_utc_to_bst(last_2_days) # if year end date is less than today, we are calculating for data in the past and have no need for deltas. if year_end_date >= today: - todays_data = get_deltas(service_id, last_2_days, today) - year_end_date = last_bst_date_for_ft - - + last_2_days = utcnow - timedelta(days=2) + data = fetch_billing_data(start_date=last_2_days, end_date=today, service_id=service_id) + inserted_records = 0 + updated_records = 0 + for d in data: + update_fact_billing(data=data, + inserted_records=inserted_records, + process_day=d.created_at, + updated_records=updated_records) yearly_data = db.session.query( - FactBilling.notifications_sent, - FactBilling.billable_units, + extract('month', FactBilling.bst_date).label("Month"), + func.sum(FactBilling.notifications_sent).label("notifications_sent"), + func.sum(FactBilling.billable_units).label("billable_units"), FactBilling.service_id, - FactBilling.notification_type, FactBilling.rate, FactBilling.rate_multiplier, FactBilling.international ).filter( FactBilling.service_id == service_id, FactBilling.bst_date >= year_start_date, - FactBilling.bst_date <= last_bst_date_for_ft + FactBilling.bst_date <= year_end_date + ).group_by( + extract('month', FactBilling.bst_date), + FactBilling.service_id, + FactBilling.rate, + FactBilling.rate_multiplier, + FactBilling.international ).all() - # today_data + yearly_data and aggregate by month + return yearly_data -def get_deltas(service_id, start_date_end_date): - # query ft_billing data using queries from create_nightly_billing - return [] +def fetch_billing_data(start_date, end_date, service_id=None): + transit_data = db.session.query( + Notification.template_id, + Notification.service_id, + Notification.notification_type, + func.coalesce(Notification.sent_by, + case( + [ + (Notification.notification_type == 'letter', 'dvla'), + (Notification.notification_type == 'sms', 'unknown'), + (Notification.notification_type == 'email', 'ses') + ]), + ).label('sent_by'), + func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'), + func.coalesce(Notification.international, False).label('international'), + func.sum(Notification.billable_units).label('billable_units'), + func.count().label('notifications_sent'), + Service.crown, + ).filter( + Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available + Notification.status != NOTIFICATION_TECHNICAL_FAILURE, + Notification.key_type != KEY_TYPE_TEST, + Notification.created_at >= start_date, + Notification.created_at < end_date + ).group_by( + Notification.template_id, + Notification.service_id, + Notification.notification_type, + 'sent_by', + Notification.rate_multiplier, + Notification.international, + Service.crown + ).join( + Service + ) + if service_id: + transit_data.filter(Notification.service_id == service_id) + return transit_data.all() +def get_rates_for_billing(): + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + return non_letter_rates, letter_rates + + +def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): + if notification_type == LETTER_TYPE: + return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2]) + elif notification_type == SMS_TYPE: + return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1]) + else: + return 0 + + +def update_fact_billing(data, inserted_records, process_day, updated_records): + non_letter_rates, letter_rates = get_rates_for_billing() + + update_count = FactBilling.query.filter( + FactBilling.bst_date == datetime.date(process_day), + FactBilling.template_id == data.template_id, + FactBilling.service_id == data.service_id, + FactBilling.provider == data.sent_by, # This could be zero - this is a bug that needs to be fixed. + FactBilling.rate_multiplier == data.rate_multiplier, + FactBilling.notification_type == data.notification_type, + FactBilling.international == data.international + ).update( + {"notifications_sent": data.notifications_sent, + "billable_units": data.billable_units}, + synchronize_session=False) + + if update_count == 0: + rate = get_rate(non_letter_rates, + letter_rates, + data.notification_type, + process_day, + data.crown, + data.rate_multiplier) + billing_record = create_billing_record(data, rate, process_day) + db.session.add(billing_record) + inserted_records += 1 + updated_records += update_count + db.session.commit() + return inserted_records, updated_records + + +def create_billing_record(data, rate, process_day): + billing_record = FactBilling( + bst_date=process_day, + template_id=data.template_id, + service_id=data.service_id, + notification_type=data.notification_type, + provider=data.sent_by, + rate_multiplier=data.rate_multiplier, + international=data.international, + billable_units=data.billable_units, + notifications_sent=data.notifications_sent, + rate=rate + ) + return billing_record diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index 9bf2daa99..2e33fcc45 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -1,6 +1,7 @@ from datetime import datetime, timedelta, date from tests.app.conftest import sample_notification -from app.celery.reporting_tasks import create_nightly_billing, get_rate +from app.celery.reporting_tasks import create_nightly_billing +from app.dao.fact_billing_dao import get_rate from app.models import (FactBilling, Notification, LETTER_TYPE, @@ -44,7 +45,7 @@ def test_create_nightly_billing_sms_rate_multiplier( yesterday = datetime.now() - timedelta(days=1) - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) # These are sms notifications sample_notification( @@ -94,7 +95,7 @@ def test_create_nightly_billing_different_templates( mocker): yesterday = datetime.now() - timedelta(days=1) - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) sample_notification( notify_db, @@ -147,7 +148,7 @@ def test_create_nightly_billing_different_sent_by( mocker): yesterday = datetime.now() - timedelta(days=1) - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) # These are sms notifications sample_notification( @@ -197,7 +198,7 @@ def test_create_nightly_billing_letter( mocker): yesterday = datetime.now() - timedelta(days=1) - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) sample_notification( notify_db, @@ -234,7 +235,7 @@ def test_create_nightly_billing_null_sent_by_sms( mocker): yesterday = datetime.now() - timedelta(days=1) - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) sample_notification( notify_db, @@ -272,7 +273,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta( sample_template, mocker): - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) # create records from 11th to 15th for i in range(0, 5): @@ -365,7 +366,7 @@ def test_create_nightly_billing_use_BST( sample_template, mocker): - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) sample_notification( notify_db, @@ -414,7 +415,7 @@ def test_create_nightly_billing_update_when_record_exists( sample_template, mocker): - mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate) sample_notification( notify_db, diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 3efe0f7f2..23c552da1 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -1,11 +1,12 @@ from datetime import datetime from decimal import Decimal -from freezegun import freeze_time - -from app.dao.date_util import get_month_start_and_end_date_in_utc -from app.dao.fact_billing_dao import fetch_annual_billing_by_month, need_deltas -from tests.app.db import create_ft_billing, create_service, create_template, create_notification +from app.dao.fact_billing_dao import fetch_annual_billing_by_month, fetch_annual_billing_for_year +from tests.app.db import ( + create_ft_billing, + create_service, + create_template +) def test_fetch_annual_billing_by_month(notify_db_session): @@ -20,23 +21,20 @@ def test_fetch_annual_billing_by_month(notify_db_session): results, month = fetch_annual_billing_by_month(service_id=record.service_id, billing_month=datetime(2018, 1, 1), notification_type='email') - assert len(results) == 1 assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False) assert month == datetime(2018, 1, 1) -@freeze_time("2018-01-21 13:00:00") -def test_need_deltas(notify_db_session): +def test_fetch_annual_billing_for_year(notify_db_session): service = create_service() template = create_template(service=service, template_type="email") - for i in range(1, 21): - record = create_ft_billing(bst_date='2018-01-{}'.format(i), - service=service, - template=template, - notification_type='email') - start_date, end_date = get_month_start_and_end_date_in_utc(datetime.utcnow()) + for i in range(1, 31): + create_ft_billing(bst_date='2018-06-{}'.format(i), + service=service, + template=template, + notification_type='email') + results = fetch_annual_billing_for_year(service_id=service.id, + year=2018) - result = need_deltas(start_date=start_date, end_date=end_date, - service_id=service.id, notification_type='email') - assert result + assert results diff --git a/tests/app/db.py b/tests/app/db.py index cdea79eb9..94bd40e26 100644 --- a/tests/app/db.py +++ b/tests/app/db.py @@ -66,17 +66,17 @@ def create_user(mobile_number="+447700900986", email="notify@digital.cabinet-off def create_service( - user=None, - service_name="Sample service", - service_id=None, - restricted=False, - service_permissions=[EMAIL_TYPE, SMS_TYPE], - research_mode=False, - active=True, - email_from=None, - prefix_sms=True, - message_limit=1000, - organisation_type='central' + user=None, + service_name="Sample service", + service_id=None, + restricted=False, + service_permissions=[EMAIL_TYPE, SMS_TYPE], + research_mode=False, + active=True, + email_from=None, + prefix_sms=True, + message_limit=1000, + organisation_type='central' ): service = Service( name=service_name, @@ -97,8 +97,8 @@ def create_service( def create_service_with_inbound_number( - inbound_number='1234567', - *args, **kwargs + inbound_number='1234567', + *args, **kwargs ): service = create_service(*args, **kwargs) @@ -112,8 +112,8 @@ def create_service_with_inbound_number( def create_service_with_defined_sms_sender( - sms_sender_value='1234567', - *args, **kwargs + sms_sender_value='1234567', + *args, **kwargs ): service = create_service(*args, **kwargs) @@ -127,13 +127,13 @@ def create_service_with_defined_sms_sender( def create_template( - service, - template_type=SMS_TYPE, - template_name=None, - subject='Template subject', - content='Dear Sir/Madam, Hello. Yours Truly, The Government.', - reply_to=None, - hidden=False + service, + template_type=SMS_TYPE, + template_name=None, + subject='Template subject', + content='Dear Sir/Madam, Hello. Yours Truly, The Government.', + reply_to=None, + hidden=False ): data = { 'name': template_name or '{} Template Name'.format(template_type), @@ -152,29 +152,29 @@ def create_template( def create_notification( - template, - job=None, - job_row_number=None, - to_field=None, - status='created', - reference=None, - created_at=None, - sent_at=None, - updated_at=None, - billable_units=1, - personalisation=None, - api_key=None, - key_type=KEY_TYPE_NORMAL, - sent_by=None, - client_reference=None, - rate_multiplier=None, - international=False, - phone_prefix=None, - scheduled_for=None, - normalised_to=None, - one_off=False, - sms_sender_id=None, - reply_to_text=None + template, + job=None, + job_row_number=None, + to_field=None, + status='created', + reference=None, + created_at=None, + sent_at=None, + updated_at=None, + billable_units=1, + personalisation=None, + api_key=None, + key_type=KEY_TYPE_NORMAL, + sent_by=None, + client_reference=None, + rate_multiplier=None, + international=False, + phone_prefix=None, + scheduled_for=None, + normalised_to=None, + one_off=False, + sms_sender_id=None, + reply_to_text=None ): if created_at is None: created_at = datetime.utcnow() @@ -236,13 +236,13 @@ def create_notification( def create_job( - template, - notification_count=1, - created_at=None, - job_status='pending', - scheduled_for=None, - processing_started=None, - original_file_name='some.csv' + template, + notification_count=1, + created_at=None, + job_status='pending', + scheduled_for=None, + processing_started=None, + original_file_name='some.csv' ): data = { 'id': uuid.uuid4(), @@ -273,14 +273,14 @@ def create_service_permission(service_id, permission=EMAIL_TYPE): def create_inbound_sms( - service, - notify_number=None, - user_number='447700900111', - provider_date=None, - provider_reference=None, - content='Hello', - provider="mmg", - created_at=None + service, + notify_number=None, + user_number='447700900111', + provider_date=None, + provider_reference=None, + content='Hello', + provider="mmg", + created_at=None ): inbound = InboundSms( service=service, @@ -297,9 +297,9 @@ def create_inbound_sms( def create_service_inbound_api( - service, - url="https://something.com", - bearer_token="some_super_secret", + service, + url="https://something.com", + bearer_token="some_super_secret", ): service_inbound_api = ServiceInboundApi(service_id=service.id, url=url, @@ -311,9 +311,9 @@ def create_service_inbound_api( def create_service_callback_api( - service, - url="https://something.com", - bearer_token="some_super_secret", + service, + url="https://something.com", + bearer_token="some_super_secret", ): service_callback_api = ServiceCallbackApi(service_id=service.id, url=url, @@ -377,11 +377,11 @@ def create_inbound_number(number, provider='mmg', active=True, service_id=None): def create_monthly_billing_entry( - service, - start_date, - end_date, - notification_type, - monthly_totals=[] + service, + start_date, + end_date, + notification_type, + monthly_totals=[] ): entry = MonthlyBilling( service_id=service.id, @@ -398,9 +398,9 @@ def create_monthly_billing_entry( def create_reply_to_email( - service, - email_address, - is_default=True + service, + email_address, + is_default=True ): data = { 'service': service, @@ -416,10 +416,10 @@ def create_reply_to_email( def create_service_sms_sender( - service, - sms_sender, - is_default=True, - inbound_number_id=None + service, + sms_sender, + is_default=True, + inbound_number_id=None ): data = { 'service_id': service.id, @@ -436,9 +436,9 @@ def create_service_sms_sender( def create_letter_contact( - service, - contact_block, - is_default=True + service, + contact_block, + is_default=True ): data = { 'service': service, @@ -454,7 +454,7 @@ def create_letter_contact( def create_annual_billing( - service_id, free_sms_fragment_limit, financial_year_start + service_id, free_sms_fragment_limit, financial_year_start ): annual_billing = AnnualBilling( service_id=service_id, @@ -468,12 +468,12 @@ def create_annual_billing( def create_letter_rate( - start_date=datetime(2017, 1, 1, 00, 00, 00), - end_date=None, - sheet_count=1, - rate=0.31, - crown=True, - post_class='second' + start_date=datetime(2017, 1, 1, 00, 00, 00), + end_date=None, + sheet_count=1, + rate=0.31, + crown=True, + post_class='second' ): rate = LetterRate( start_date=start_date, @@ -529,8 +529,8 @@ def create_daily_sorted_letter(billing_day=date(2018, 1, 18), def create_ft_billing(bst_date, notification_type, - template = None, - service = None, + template=None, + service=None, provider='test', rate_multiplier=1, international=False, From d00614205e8fa5cecb9b2be3d2d4c4894c663e77 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 25 Apr 2018 14:24:47 +0100 Subject: [PATCH 4/7] Adding test for fetch_biling_data_for_day. --- app/celery/reporting_tasks.py | 21 +---- app/dao/fact_billing_dao.py | 73 ++++++---------- tests/app/dao/test_ft_billing_dao.py | 119 +++++++++++++++++++++++---- 3 files changed, 133 insertions(+), 80 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index aca6b1ee2..f9c336638 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,14 +1,12 @@ -from datetime import datetime, timedelta, time +from datetime import datetime, timedelta -from flask import current_app from notifications_utils.statsd_decorators import statsd from app import notify_celery from app.dao.fact_billing_dao import ( - fetch_billing_data, + fetch_billing_data_for_day, update_fact_billing ) -from app.utils import convert_bst_to_utc @notify_celery.task(name="create-nightly-billing") @@ -21,19 +19,8 @@ def create_nightly_billing(day_start=None): for i in range(0, 3): process_day = day_start - timedelta(days=i) - ds = convert_bst_to_utc(datetime.combine(process_day, time.min)) - de = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) - transit_data = fetch_billing_data(start_date=ds, end_date=de) - - updated_records = 0 - inserted_records = 0 + transit_data = fetch_billing_data_for_day(process_day=process_day) for data in transit_data: - inserted_records, updated_records = update_fact_billing(data, - inserted_records, - process_day, - updated_records) - - current_app.logger.info('ft_billing {} to {}: {} rows updated, {} rows inserted' - .format(ds, de, updated_records, inserted_records)) + update_fact_billing(data, process_day) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index eeda5d485..a0d9fc922 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,62 +1,37 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, time + +from flask import current_app from sqlalchemy import func, case, desc, extract from app import db -from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year +from app.dao.date_util import get_financial_year from app.models import ( - FactBilling, Notification, Service, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE, + FactBilling, + Notification, + Service, + NOTIFICATION_CREATED, + NOTIFICATION_TECHNICAL_FAILURE, KEY_TYPE_TEST, LETTER_TYPE, SMS_TYPE, Rate, LetterRate ) -from app.utils import convert_utc_to_bst - - -def fetch_annual_billing_by_month(service_id, billing_month, notification_type): - billing_month_in_bst = convert_utc_to_bst(billing_month) - start_date, end_date = get_month_start_and_end_date_in_utc(billing_month_in_bst) - - monthly_data = db.session.query( - func.sum(FactBilling.notifications_sent).label('notifications_sent'), - func.sum(FactBilling.billable_units).label('billing_units'), - FactBilling.service_id, - FactBilling.notification_type, - FactBilling.rate, - FactBilling.rate_multiplier, - FactBilling.international - ).filter( - FactBilling.notification_type == notification_type, - FactBilling.service_id == service_id, - FactBilling.bst_date >= start_date, - FactBilling.bst_date <= end_date - ).group_by( - FactBilling.service_id, - FactBilling.notification_type, - FactBilling.rate, - FactBilling.rate_multiplier, - FactBilling.international - ).all() - - return monthly_data, start_date +from app.utils import convert_utc_to_bst, convert_bst_to_utc def fetch_annual_billing_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) utcnow = datetime.utcnow() - today = convert_utc_to_bst(utcnow) + today = convert_utc_to_bst(utcnow).date() # if year end date is less than today, we are calculating for data in the past and have no need for deltas. - if year_end_date >= today: - last_2_days = utcnow - timedelta(days=2) - data = fetch_billing_data(start_date=last_2_days, end_date=today, service_id=service_id) - inserted_records = 0 - updated_records = 0 - for d in data: - update_fact_billing(data=data, - inserted_records=inserted_records, - process_day=d.created_at, - updated_records=updated_records) + if year_end_date.date() >= today: + yesterday = today - timedelta(days=1) + for day in [yesterday, today]: + data = fetch_billing_data_for_day(process_day=day, service_id=service_id) + for d in data: + update_fact_billing(data=d, process_day=day) + yearly_data = db.session.query( extract('month', FactBilling.bst_date).label("Month"), func.sum(FactBilling.notifications_sent).label("notifications_sent"), @@ -80,7 +55,10 @@ def fetch_annual_billing_for_year(service_id, year): return yearly_data -def fetch_billing_data(start_date, end_date, service_id=None): +def fetch_billing_data_for_day(process_day, service_id=None): + start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) + end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) + transit_data = db.session.query( Notification.template_id, Notification.service_id, @@ -137,7 +115,9 @@ def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None return 0 -def update_fact_billing(data, inserted_records, process_day, updated_records): +def update_fact_billing(data, process_day): + inserted_records = 0 + updated_records = 0 non_letter_rates, letter_rates = get_rates_for_billing() update_count = FactBilling.query.filter( @@ -165,7 +145,8 @@ def update_fact_billing(data, inserted_records, process_day, updated_records): inserted_records += 1 updated_records += update_count db.session.commit() - return inserted_records, updated_records + current_app.logger.info('ft_billing for {}: {} rows updated, {} rows inserted' + .format(process_day, updated_records, inserted_records)) def create_billing_record(data, rate, process_day): diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 23c552da1..d7512a972 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -1,40 +1,125 @@ -from datetime import datetime from decimal import Decimal -from app.dao.fact_billing_dao import fetch_annual_billing_by_month, fetch_annual_billing_for_year +from datetime import datetime, timedelta +from freezegun import freeze_time + +from app import db +from app.dao.fact_billing_dao import fetch_annual_billing_for_year, fetch_billing_data_for_day +from app.models import FactBilling +from app.utils import convert_utc_to_bst from tests.app.db import ( create_ft_billing, create_service, - create_template + create_template, + create_notification ) -def test_fetch_annual_billing_by_month(notify_db_session): +def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify_db_session): service = create_service() template = create_template(service=service, template_type="email") - for i in range(1, 32): - record = create_ft_billing(bst_date='2018-01-{}'.format(i), - service=service, - template=template, - notification_type='email') + for status in ['delivered', 'sending', 'temporary-failure', 'created', 'technical-failure']: + create_notification(template=template, status=status) - results, month = fetch_annual_billing_by_month(service_id=record.service_id, - billing_month=datetime(2018, 1, 1), - notification_type='email') + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) assert len(results) == 1 - assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False) - assert month == datetime(2018, 1, 1) + assert results[0].notifications_sent == 3 + + +def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for key_type in ['normal', 'test', 'team']: + create_notification(template=template, status='delivered', key_type=key_type) + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 1 + assert results[0].notifications_sent == 2 + + +def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_db_session): + process_day = datetime(2018, 4, 1, 13, 30, 00) + service = create_service() + template = create_template(service=service, template_type="email") + create_notification(template=template, status='delivered', created_at=process_day) + create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 23, 23, 23)) + + create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 20, 23, 23)) + create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1)) + + day_under_test = convert_utc_to_bst(process_day) + results = fetch_billing_data_for_day(day_under_test) + assert len(results) == 1 + assert results[0].notifications_sent == 2 + + +def test_fetch_billing_data_for_day_is_grouped_by_template(notify_db_session): + service = create_service() + email_template = create_template(service=service, template_type="email") + sms_template = create_template(service=service, template_type="email") + create_notification(template=email_template, status='delivered') + create_notification(template=sms_template, status='delivered') + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 def test_fetch_annual_billing_for_year(notify_db_session): service = create_service() - template = create_template(service=service, template_type="email") + template = create_template(service=service, template_type="sms") for i in range(1, 31): create_ft_billing(bst_date='2018-06-{}'.format(i), service=service, template=template, - notification_type='email') + notification_type='sms', + rate=0.162) + for i in range(1, 32): + create_ft_billing(bst_date='2018-07-{}'.format(i), + service=service, + template=template, + notification_type='sms', + rate=0.158) + results = fetch_annual_billing_for_year(service_id=service.id, year=2018) - assert results + assert len(results) == 2 + assert results[0][0] == 6.0 + assert results[0][1] == 30 + assert results[0][2] == Decimal('30') + assert results[0][3] == service.id + assert results[0][4] == Decimal('0.162') + assert results[0][5] == Decimal('1') + assert results[0][6] is False + + assert results[1][0] == 7.0 + assert results[1][1] == 31 + assert results[1][2] == Decimal('31') + assert results[1][3] == service.id + assert results[1][4] == Decimal('0.158') + assert results[1][5] == Decimal('1') + assert results[1][6] is False + + +@freeze_time('2018-08-01 13:30:00') +def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for i in range(1, 32): + create_ft_billing(bst_date='2018-07-{}'.format(i), + service=service, + template=template, + notification_type='email', + rate=0.162) + create_notification(template=template, status='delivered') + + assert db.session.query(FactBilling.bst_date).count() == 31 + results = fetch_annual_billing_for_year(service_id=service.id, + year=2018) + assert db.session.query(FactBilling.bst_date).count() == 32 + assert len(results) == 2 From e32debbca1e751c838ac4cede28f430aff8b3fa2 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 25 Apr 2018 15:40:44 +0100 Subject: [PATCH 5/7] Add more tests --- tests/app/dao/test_ft_billing_dao.py | 115 ++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 4 deletions(-) diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index d7512a972..091924f29 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -4,14 +4,19 @@ from datetime import datetime, timedelta from freezegun import freeze_time from app import db -from app.dao.fact_billing_dao import fetch_annual_billing_for_year, fetch_billing_data_for_day +from app.dao.fact_billing_dao import ( + fetch_annual_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing, + get_rate +) from app.models import FactBilling from app.utils import convert_utc_to_bst from tests.app.db import ( create_ft_billing, create_service, create_template, - create_notification + create_notification, + create_rate, + create_letter_rate ) @@ -55,10 +60,10 @@ def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_d assert results[0].notifications_sent == 2 -def test_fetch_billing_data_for_day_is_grouped_by_template(notify_db_session): +def test_fetch_billing_data_for_day_is_grouped_by_template_and_notification_type(notify_db_session): service = create_service() email_template = create_template(service=service, template_type="email") - sms_template = create_template(service=service, template_type="email") + sms_template = create_template(service=service, template_type="sms") create_notification(template=email_template, status='delivered') create_notification(template=sms_template, status='delivered') @@ -69,6 +74,108 @@ def test_fetch_billing_data_for_day_is_grouped_by_template(notify_db_session): assert results[1].notifications_sent == 1 +def test_fetch_billing_data_for_day_is_grouped_by_service(notify_db_session): + service_1 = create_service() + service_2 = create_service(service_name='Service 2') + email_template = create_template(service=service_1) + sms_template = create_template(service=service_2) + create_notification(template=email_template, status='delivered') + create_notification(template=sms_template, status='delivered') + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 + + +def test_fetch_billing_data_for_day_is_grouped_by_provider(notify_db_session): + service = create_service() + template = create_template(service=service) + create_notification(template=template, status='delivered', sent_by='mmg') + create_notification(template=template, status='delivered', sent_by='firetext') + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 + + +def test_fetch_billing_data_for_day_is_grouped_by_rate_mulitplier(notify_db_session): + service = create_service() + template = create_template(service=service) + create_notification(template=template, status='delivered', rate_multiplier=1) + create_notification(template=template, status='delivered', rate_multiplier=2) + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 + + +def test_fetch_billing_data_for_day_is_grouped_by_international(notify_db_session): + service = create_service() + template = create_template(service=service) + create_notification(template=template, status='delivered', international=True) + create_notification(template=template, status='delivered', international=False) + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 + + +def test_fetch_billing_data_for_day_returns_empty_list(notify_db_session): + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert results == [] + + +def test_fetch_billing_data_for_day_returns_list_for_given_service(notify_db_session): + service = create_service() + service_2 = create_service(service_name='Service 2') + template = create_template(service=service) + template_2 = create_template(service=service_2) + create_notification(template=template, status='delivered') + create_notification(template=template_2, status='delivered') + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(process_day=today, service_id=service.id) + assert len(results) == 1 + assert results[0].service_id == service.id + + +def test_get_rates_for_billing(notify_db_session): + create_rate(start_date=datetime.utcnow(), value=12, notification_type='email') + create_rate(start_date=datetime.utcnow(), value=22, notification_type='sms') + create_rate(start_date=datetime.utcnow(), value=33, notification_type='email') + create_letter_rate(start_date=datetime.utcnow()) + non_letter_rates, letter_rates = get_rates_for_billing() + + assert len(non_letter_rates) == 3 + assert len(letter_rates) == 1 + + +def test_get_rate(notify_db_session): + create_rate(start_date=datetime.utcnow(), value=1.2, notification_type='email') + create_rate(start_date=datetime.utcnow(), value=2.2, notification_type='sms') + create_rate(start_date=datetime.utcnow(), value=3.3, notification_type='email') + create_letter_rate(start_date=datetime.utcnow(), rate=4.4) + non_letter_rates, letter_rates = get_rates_for_billing() + rate = get_rate(non_letter_rates=non_letter_rates, letter_rates=letter_rates, notification_type='sms', date=datetime.utcnow()) + letter_rate = get_rate(non_letter_rates=non_letter_rates, letter_rates=letter_rates, + notification_type='letter', + crown=True, + rate_multiplier=1, + date=datetime.utcnow()) + + assert rate == 2.2 + assert letter_rate == Decimal('4.4') + + def test_fetch_annual_billing_for_year(notify_db_session): service = create_service() template = create_template(service=service, template_type="sms") From cff731c52d20f2ad14fac3ee0c279770a96546ac Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 25 Apr 2018 16:17:56 +0100 Subject: [PATCH 6/7] Set filter on query --- app/dao/fact_billing_dao.py | 2 +- tests/app/dao/test_ft_billing_dao.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index a0d9fc922..58c62d2a6 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -94,7 +94,7 @@ def fetch_billing_data_for_day(process_day, service_id=None): Service ) if service_id: - transit_data.filter(Notification.service_id == service_id) + transit_data = transit_data.filter(Notification.service_id == service_id) return transit_data.all() diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 091924f29..e0a69715a 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -165,7 +165,8 @@ def test_get_rate(notify_db_session): create_rate(start_date=datetime.utcnow(), value=3.3, notification_type='email') create_letter_rate(start_date=datetime.utcnow(), rate=4.4) non_letter_rates, letter_rates = get_rates_for_billing() - rate = get_rate(non_letter_rates=non_letter_rates, letter_rates=letter_rates, notification_type='sms', date=datetime.utcnow()) + rate = get_rate(non_letter_rates=non_letter_rates, letter_rates=letter_rates, notification_type='sms', + date=datetime.utcnow()) letter_rate = get_rate(non_letter_rates=non_letter_rates, letter_rates=letter_rates, notification_type='letter', crown=True, From 51af6b27a01125be1e5f32f59b05aa9bc7789c63 Mon Sep 17 00:00:00 2001 From: Rebecca Law Date: Wed, 25 Apr 2018 17:00:19 +0100 Subject: [PATCH 7/7] Update name of method to clarify the meaning. Update group by to use label. Update test. --- app/dao/fact_billing_dao.py | 4 ++-- tests/app/dao/test_ft_billing_dao.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index 58c62d2a6..bc6f7d357 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -20,7 +20,7 @@ from app.models import ( from app.utils import convert_utc_to_bst, convert_bst_to_utc -def fetch_annual_billing_for_year(service_id, year): +def fetch_montly_billing_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) utcnow = datetime.utcnow() today = convert_utc_to_bst(utcnow).date() @@ -45,7 +45,7 @@ def fetch_annual_billing_for_year(service_id, year): FactBilling.bst_date >= year_start_date, FactBilling.bst_date <= year_end_date ).group_by( - extract('month', FactBilling.bst_date), + 'Month', FactBilling.service_id, FactBilling.rate, FactBilling.rate_multiplier, diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index e0a69715a..4ed43ffc5 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -5,7 +5,7 @@ from freezegun import freeze_time from app import db from app.dao.fact_billing_dao import ( - fetch_annual_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing, + fetch_montly_billing_for_year, fetch_billing_data_for_day, get_rates_for_billing, get_rate ) from app.models import FactBilling @@ -23,11 +23,15 @@ from tests.app.db import ( def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify_db_session): service = create_service() template = create_template(service=service, template_type="email") - for status in ['delivered', 'sending', 'temporary-failure', 'created', 'technical-failure']: + for status in ['created', 'technical-failure']: create_notification(template=template, status=status) today = convert_utc_to_bst(datetime.utcnow()) results = fetch_billing_data_for_day(today) + assert results == [] + for status in ['delivered', 'sending', 'temporary-failure']: + create_notification(template=template, status=status) + results = fetch_billing_data_for_day(today) assert len(results) == 1 assert results[0].notifications_sent == 3 @@ -193,8 +197,7 @@ def test_fetch_annual_billing_for_year(notify_db_session): notification_type='sms', rate=0.158) - results = fetch_annual_billing_for_year(service_id=service.id, - year=2018) + results = fetch_montly_billing_for_year(service_id=service.id, year=2018) assert len(results) == 2 assert results[0][0] == 6.0 @@ -227,7 +230,7 @@ def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session): create_notification(template=template, status='delivered') assert db.session.query(FactBilling.bst_date).count() == 31 - results = fetch_annual_billing_for_year(service_id=service.id, + results = fetch_montly_billing_for_year(service_id=service.id, year=2018) assert db.session.query(FactBilling.bst_date).count() == 32 assert len(results) == 2