Adding test for fetch_biling_data_for_day.

app/celery/reporting_tasks.py

@@ -1,14 +1,12 @@
-from datetime import datetime, timedelta, time
+from datetime import datetime, timedelta
 
-from flask import current_app
 from notifications_utils.statsd_decorators import statsd
 
 from app import notify_celery
 from app.dao.fact_billing_dao import (
-    fetch_billing_data,
+    fetch_billing_data_for_day,
     update_fact_billing
 )
-from app.utils import convert_bst_to_utc
 
 
 @notify_celery.task(name="create-nightly-billing")
@@ -21,19 +19,8 @@ def create_nightly_billing(day_start=None):
 
     for i in range(0, 3):
         process_day = day_start - timedelta(days=i)
-        ds = convert_bst_to_utc(datetime.combine(process_day, time.min))
-        de = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
 
-        transit_data = fetch_billing_data(start_date=ds, end_date=de)
+        transit_data = fetch_billing_data_for_day(process_day=process_day)
-
-        updated_records = 0
-        inserted_records = 0
 
         for data in transit_data:
-            inserted_records, updated_records = update_fact_billing(data,
+            update_fact_billing(data, process_day)
-                                                                    inserted_records,
-                                                                    process_day,
-                                                                    updated_records)
-
-        current_app.logger.info('ft_billing {} to {}: {} rows updated, {} rows inserted'
-                                .format(ds, de, updated_records, inserted_records))

app/dao/fact_billing_dao.py

@@ -1,62 +1,37 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, time
+
+from flask import current_app
 from sqlalchemy import func, case, desc, extract
 
 from app import db
-from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year
+from app.dao.date_util import get_financial_year
 from app.models import (
-    FactBilling, Notification, Service, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE,
+    FactBilling,
+    Notification,
+    Service,
+    NOTIFICATION_CREATED,
+    NOTIFICATION_TECHNICAL_FAILURE,
     KEY_TYPE_TEST,
     LETTER_TYPE,
     SMS_TYPE,
     Rate,
     LetterRate
 )
-from app.utils import convert_utc_to_bst
+from app.utils import convert_utc_to_bst, convert_bst_to_utc
-
-
-def fetch_annual_billing_by_month(service_id, billing_month, notification_type):
-    billing_month_in_bst = convert_utc_to_bst(billing_month)
-    start_date, end_date = get_month_start_and_end_date_in_utc(billing_month_in_bst)
-
-    monthly_data = db.session.query(
-        func.sum(FactBilling.notifications_sent).label('notifications_sent'),
-        func.sum(FactBilling.billable_units).label('billing_units'),
-        FactBilling.service_id,
-        FactBilling.notification_type,
-        FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international
-    ).filter(
-        FactBilling.notification_type == notification_type,
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).group_by(
-        FactBilling.service_id,
-        FactBilling.notification_type,
-        FactBilling.rate,
-        FactBilling.rate_multiplier,
-        FactBilling.international
-    ).all()
-
-    return monthly_data, start_date
 
 
 def fetch_annual_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
-    today = convert_utc_to_bst(utcnow)
+    today = convert_utc_to_bst(utcnow).date()
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
-    if year_end_date >= today:
+    if year_end_date.date() >= today:
-        last_2_days = utcnow - timedelta(days=2)
+        yesterday = today - timedelta(days=1)
-        data = fetch_billing_data(start_date=last_2_days, end_date=today, service_id=service_id)
+        for day in [yesterday, today]:
-        inserted_records = 0
+            data = fetch_billing_data_for_day(process_day=day, service_id=service_id)
-        updated_records = 0
+            for d in data:
-        for d in data:
+                update_fact_billing(data=d, process_day=day)
-            update_fact_billing(data=data,
+
-                                inserted_records=inserted_records,
-                                process_day=d.created_at,
-                                updated_records=updated_records)
     yearly_data = db.session.query(
         extract('month', FactBilling.bst_date).label("Month"),
         func.sum(FactBilling.notifications_sent).label("notifications_sent"),
@@ -80,7 +55,10 @@ def fetch_annual_billing_for_year(service_id, year):
     return yearly_data
 
 
-def fetch_billing_data(start_date, end_date, service_id=None):
+def fetch_billing_data_for_day(process_day, service_id=None):
+    start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
+    end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
+
     transit_data = db.session.query(
         Notification.template_id,
         Notification.service_id,
@@ -137,7 +115,9 @@ def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None
         return 0
 
 
-def update_fact_billing(data, inserted_records, process_day, updated_records):
+def update_fact_billing(data, process_day):
+    inserted_records = 0
+    updated_records = 0
     non_letter_rates, letter_rates = get_rates_for_billing()
 
     update_count = FactBilling.query.filter(
@@ -165,7 +145,8 @@ def update_fact_billing(data, inserted_records, process_day, updated_records):
         inserted_records += 1
     updated_records += update_count
     db.session.commit()
-    return inserted_records, updated_records
+    current_app.logger.info('ft_billing for {}: {} rows updated, {} rows inserted'
+                            .format(process_day, updated_records, inserted_records))
 
 
 def create_billing_record(data, rate, process_day):

tests/app/dao/test_ft_billing_dao.py

@@ -1,40 +1,125 @@
-from datetime import datetime
 from decimal import Decimal
 
-from app.dao.fact_billing_dao import fetch_annual_billing_by_month, fetch_annual_billing_for_year
+from datetime import datetime, timedelta
+from freezegun import freeze_time
+
+from app import db
+from app.dao.fact_billing_dao import fetch_annual_billing_for_year, fetch_billing_data_for_day
+from app.models import FactBilling
+from app.utils import convert_utc_to_bst
 from tests.app.db import (
     create_ft_billing,
     create_service,
-    create_template
+    create_template,
+    create_notification
 )
 
 
-def test_fetch_annual_billing_by_month(notify_db_session):
+def test_fetch_billing_data_for_today_includes_data_with_the_right_status(notify_db_session):
     service = create_service()
     template = create_template(service=service, template_type="email")
-    for i in range(1, 32):
+    for status in ['delivered', 'sending', 'temporary-failure', 'created', 'technical-failure']:
-        record = create_ft_billing(bst_date='2018-01-{}'.format(i),
+        create_notification(template=template, status=status)
-                                   service=service,
-                                   template=template,
-                                   notification_type='email')
 
-    results, month = fetch_annual_billing_by_month(service_id=record.service_id,
+    today = convert_utc_to_bst(datetime.utcnow())
-                                                   billing_month=datetime(2018, 1, 1),
+    results = fetch_billing_data_for_day(today)
-                                                   notification_type='email')
     assert len(results) == 1
-    assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False)
+    assert results[0].notifications_sent == 3
-    assert month == datetime(2018, 1, 1)
+
+
+def test_fetch_billing_data_for_today_includes_data_with_the_right_key_type(notify_db_session):
+    service = create_service()
+    template = create_template(service=service, template_type="email")
+    for key_type in ['normal', 'test', 'team']:
+        create_notification(template=template, status='delivered', key_type=key_type)
+
+    today = convert_utc_to_bst(datetime.utcnow())
+    results = fetch_billing_data_for_day(today)
+    assert len(results) == 1
+    assert results[0].notifications_sent == 2
+
+
+def test_fetch_billing_data_for_today_includes_data_with_the_right_date(notify_db_session):
+    process_day = datetime(2018, 4, 1, 13, 30, 00)
+    service = create_service()
+    template = create_template(service=service, template_type="email")
+    create_notification(template=template, status='delivered', created_at=process_day)
+    create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 23, 23, 23))
+
+    create_notification(template=template, status='delivered', created_at=datetime(2018, 3, 31, 20, 23, 23))
+    create_notification(template=template, status='sending', created_at=process_day + timedelta(days=1))
+
+    day_under_test = convert_utc_to_bst(process_day)
+    results = fetch_billing_data_for_day(day_under_test)
+    assert len(results) == 1
+    assert results[0].notifications_sent == 2
+
+
+def test_fetch_billing_data_for_day_is_grouped_by_template(notify_db_session):
+    service = create_service()
+    email_template = create_template(service=service, template_type="email")
+    sms_template = create_template(service=service, template_type="email")
+    create_notification(template=email_template, status='delivered')
+    create_notification(template=sms_template, status='delivered')
+
+    today = convert_utc_to_bst(datetime.utcnow())
+    results = fetch_billing_data_for_day(today)
+    assert len(results) == 2
+    assert results[0].notifications_sent == 1
+    assert results[1].notifications_sent == 1
 
 
 def test_fetch_annual_billing_for_year(notify_db_session):
     service = create_service()
-    template = create_template(service=service, template_type="email")
+    template = create_template(service=service, template_type="sms")
     for i in range(1, 31):
         create_ft_billing(bst_date='2018-06-{}'.format(i),
                           service=service,
                           template=template,
-                          notification_type='email')
+                          notification_type='sms',
+                          rate=0.162)
+    for i in range(1, 32):
+        create_ft_billing(bst_date='2018-07-{}'.format(i),
+                          service=service,
+                          template=template,
+                          notification_type='sms',
+                          rate=0.158)
+
     results = fetch_annual_billing_for_year(service_id=service.id,
                                             year=2018)
 
-    assert results
+    assert len(results) == 2
+    assert results[0][0] == 6.0
+    assert results[0][1] == 30
+    assert results[0][2] == Decimal('30')
+    assert results[0][3] == service.id
+    assert results[0][4] == Decimal('0.162')
+    assert results[0][5] == Decimal('1')
+    assert results[0][6] is False
+
+    assert results[1][0] == 7.0
+    assert results[1][1] == 31
+    assert results[1][2] == Decimal('31')
+    assert results[1][3] == service.id
+    assert results[1][4] == Decimal('0.158')
+    assert results[1][5] == Decimal('1')
+    assert results[1][6] is False
+
+
+@freeze_time('2018-08-01 13:30:00')
+def test_fetch_annual_billing_for_year_adds_data_for_today(notify_db_session):
+    service = create_service()
+    template = create_template(service=service, template_type="email")
+    for i in range(1, 32):
+        create_ft_billing(bst_date='2018-07-{}'.format(i),
+                          service=service,
+                          template=template,
+                          notification_type='email',
+                          rate=0.162)
+    create_notification(template=template, status='delivered')
+
+    assert db.session.query(FactBilling.bst_date).count() == 31
+    results = fetch_annual_billing_for_year(service_id=service.id,
+                                            year=2018)
+    assert db.session.query(FactBilling.bst_date).count() == 32
+    assert len(results) == 2