diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index aca6b1ee2..f9c336638 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,14 +1,12 @@ -from datetime import datetime, timedelta, time +from datetime import datetime, timedelta -from flask import current_app from notifications_utils.statsd_decorators import statsd from app import notify_celery from app.dao.fact_billing_dao import ( - fetch_billing_data, + fetch_billing_data_for_day, update_fact_billing ) -from app.utils import convert_bst_to_utc @notify_celery.task(name="create-nightly-billing") @@ -21,19 +19,8 @@ def create_nightly_billing(day_start=None): for i in range(0, 3): process_day = day_start - timedelta(days=i) - ds = convert_bst_to_utc(datetime.combine(process_day, time.min)) - de = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) - transit_data = fetch_billing_data(start_date=ds, end_date=de) - - updated_records = 0 - inserted_records = 0 + transit_data = fetch_billing_data_for_day(process_day=process_day) for data in transit_data: - inserted_records, updated_records = update_fact_billing(data, - inserted_records, - process_day, - updated_records) - - current_app.logger.info('ft_billing {} to {}: {} rows updated, {} rows inserted' - .format(ds, de, updated_records, inserted_records)) + update_fact_billing(data, process_day) diff --git a/app/dao/fact_billing_dao.py b/app/dao/fact_billing_dao.py index eeda5d485..a0d9fc922 100644 --- a/app/dao/fact_billing_dao.py +++ b/app/dao/fact_billing_dao.py @@ -1,62 +1,37 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, time + +from flask import current_app from sqlalchemy import func, case, desc, extract from app import db -from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year +from app.dao.date_util import get_financial_year from app.models import ( - FactBilling, Notification, Service, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE, + FactBilling, + Notification, + Service, + NOTIFICATION_CREATED, + NOTIFICATION_TECHNICAL_FAILURE, KEY_TYPE_TEST, LETTER_TYPE, SMS_TYPE, Rate, LetterRate ) -from app.utils import convert_utc_to_bst - - -def fetch_annual_billing_by_month(service_id, billing_month, notification_type): - billing_month_in_bst = convert_utc_to_bst(billing_month) - start_date, end_date = get_month_start_and_end_date_in_utc(billing_month_in_bst) - - monthly_data = db.session.query( - func.sum(FactBilling.notifications_sent).label('notifications_sent'), - func.sum(FactBilling.billable_units).label('billing_units'), - FactBilling.service_id, - FactBilling.notification_type, - FactBilling.rate, - FactBilling.rate_multiplier, - FactBilling.international - ).filter( - FactBilling.notification_type == notification_type, - FactBilling.service_id == service_id, - FactBilling.bst_date >= start_date, - FactBilling.bst_date <= end_date - ).group_by( - FactBilling.service_id, - FactBilling.notification_type, - FactBilling.rate, - FactBilling.rate_multiplier, - FactBilling.international - ).all() - - return monthly_data, start_date +from app.utils import convert_utc_to_bst, convert_bst_to_utc def fetch_annual_billing_for_year(service_id, year): year_start_date, year_end_date = get_financial_year(year) utcnow = datetime.utcnow() - today = convert_utc_to_bst(utcnow) + today = convert_utc_to_bst(utcnow).date() # if year end date is less than today, we are calculating for data in the past and have no need for deltas. - if year_end_date >= today: - last_2_days = utcnow - timedelta(days=2) - data = fetch_billing_data(start_date=last_2_days, end_date=today, service_id=service_id) - inserted_records = 0 - updated_records = 0 - for d in data: - update_fact_billing(data=data, - inserted_records=inserted_records, - process_day=d.created_at, - updated_records=updated_records) + if year_end_date.date() >= today: + yesterday = today - timedelta(days=1) + for day in [yesterday, today]: + data = fetch_billing_data_for_day(process_day=day, service_id=service_id) + for d in data: + update_fact_billing(data=d, process_day=day) + yearly_data = db.session.query( extract('month', FactBilling.bst_date).label("Month"), func.sum(FactBilling.notifications_sent).label("notifications_sent"), @@ -80,7 +55,10 @@ def fetch_annual_billing_for_year(service_id, year): return yearly_data -def fetch_billing_data(start_date, end_date, service_id=None): +def fetch_billing_data_for_day(process_day, service_id=None): + start_date = convert_bst_to_utc(datetime.combine(process_day, time.min)) + end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min)) + transit_data = db.session.query( Notification.template_id, Notification.service_id, @@ -137,7 +115,9 @@ def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None return 0 -def update_fact_billing(data, inserted_records, process_day, updated_records): +def update_fact_billing(data, process_day): + inserted_records = 0 + updated_records = 0 non_letter_rates, letter_rates = get_rates_for_billing() update_count = FactBilling.query.filter( @@ -165,7 +145,8 @@ def update_fact_billing(data, inserted_records, process_day, updated_records): inserted_records += 1 updated_records += update_count db.session.commit() - return inserted_records, updated_records + current_app.logger.info('ft_billing for {}: {} rows updated, {} rows inserted' + .format(process_day, updated_records, inserted_records)) def create_billing_record(data, rate, process_day): diff --git a/tests/app/dao/test_ft_billing_dao.py b/tests/app/dao/test_ft_billing_dao.py index 23c552da1..d7512a972 100644 --- a/tests/app/dao/test_ft_billing_dao.py +++ b/tests/app/dao/test_ft_billing_dao.py @@ -1,40 +1,125 @@ -from datetime import datetime from decimal import Decimal -from app.dao.fact_billing_dao import fetch_annual_billing_by_month, fetch_annual_billing_for_year +from datetime import datetime, timedelta +from freezegun import freeze_time + +from app import db +from app.dao.fact_billing_dao import fetch_annual_billing_for_year, fetch_billing_data_for_day +from app.models import FactBilling +from app.utils import convert_utc_to_bst from tests.app.db import ( create_ft_billing, create_service, - create_template + create_template, + create_notification ) -def test_fetch_annual_billing_by_month(notify_db_session): +def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify_db_session): service = create_service() template = create_template(service=service, template_type="email") - for i in range(1, 32): - record = create_ft_billing(bst_date='2018-01-{}'.format(i), - service=service, - template=template, - notification_type='email') + for status in ['delivered', 'sending', 'temporary-failure', 'created', 'technical-failure']: + create_notification(template=template, status=status) - results, month = fetch_annual_billing_by_month(service_id=record.service_id, - billing_month=datetime(2018, 1, 1), - notification_type='email') + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) assert len(results) == 1 - assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False) - assert month == datetime(2018, 1, 1) + assert results[0].notifications_sent == 3 + + +def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for key_type in ['normal', 'test', 'team']: + create_notification(template=template, status='delivered', key_type=key_type) + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 1 + assert results[0].notifications_sent == 2 + + +def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_db_session): + process_day = datetime(2018, 4, 1, 13, 30, 00) + service = create_service() + template = create_template(service=service, template_type="email") + create_notification(template=template, status='delivered', created_at=process_day) + create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 23, 23, 23)) + + create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 20, 23, 23)) + create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1)) + + day_under_test = convert_utc_to_bst(process_day) + results = fetch_billing_data_for_day(day_under_test) + assert len(results) == 1 + assert results[0].notifications_sent == 2 + + +def test_fetch_billing_data_for_day_is_grouped_by_template(notify_db_session): + service = create_service() + email_template = create_template(service=service, template_type="email") + sms_template = create_template(service=service, template_type="email") + create_notification(template=email_template, status='delivered') + create_notification(template=sms_template, status='delivered') + + today = convert_utc_to_bst(datetime.utcnow()) + results = fetch_billing_data_for_day(today) + assert len(results) == 2 + assert results[0].notifications_sent == 1 + assert results[1].notifications_sent == 1 def test_fetch_annual_billing_for_year(notify_db_session): service = create_service() - template = create_template(service=service, template_type="email") + template = create_template(service=service, template_type="sms") for i in range(1, 31): create_ft_billing(bst_date='2018-06-{}'.format(i), service=service, template=template, - notification_type='email') + notification_type='sms', + rate=0.162) + for i in range(1, 32): + create_ft_billing(bst_date='2018-07-{}'.format(i), + service=service, + template=template, + notification_type='sms', + rate=0.158) + results = fetch_annual_billing_for_year(service_id=service.id, year=2018) - assert results + assert len(results) == 2 + assert results[0][0] == 6.0 + assert results[0][1] == 30 + assert results[0][2] == Decimal('30') + assert results[0][3] == service.id + assert results[0][4] == Decimal('0.162') + assert results[0][5] == Decimal('1') + assert results[0][6] is False + + assert results[1][0] == 7.0 + assert results[1][1] == 31 + assert results[1][2] == Decimal('31') + assert results[1][3] == service.id + assert results[1][4] == Decimal('0.158') + assert results[1][5] == Decimal('1') + assert results[1][6] is False + + +@freeze_time('2018-08-01 13:30:00') +def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session): + service = create_service() + template = create_template(service=service, template_type="email") + for i in range(1, 32): + create_ft_billing(bst_date='2018-07-{}'.format(i), + service=service, + template=template, + notification_type='email', + rate=0.162) + create_notification(template=template, status='delivered') + + assert db.session.query(FactBilling.bst_date).count() == 31 + results = fetch_annual_billing_for_year(service_id=service.id, + year=2018) + assert db.session.query(FactBilling.bst_date).count() == 32 + assert len(results) == 2