Refactor reporting task so that methods can be used for the annual usage page.

Still a work in progress, tests are coming.

app/celery/reporting_tasks.py

@@ -1,31 +1,16 @@
 from datetime import datetime, timedelta, time
-from app.models import (Notification,
+
-                        Rate,
-                        NOTIFICATION_CREATED,
-                        NOTIFICATION_TECHNICAL_FAILURE,
-                        KEY_TYPE_TEST,
-                        LetterRate,
-                        FactBilling,
-                        Service,
-                        LETTER_TYPE, SMS_TYPE)
-from app import db
-from sqlalchemy import func, desc, case
-from notifications_utils.statsd_decorators import statsd
-from app import notify_celery
 from flask import current_app
+from notifications_utils.statsd_decorators import statsd
+
+from app import notify_celery
+from app.dao.fact_billing_dao import (
+    fetch_billing_data,
+    update_fact_billing
+)
 from app.utils import convert_bst_to_utc
 
 
-def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
-
-    if notification_type == LETTER_TYPE:
-        return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2])
-    elif notification_type == SMS_TYPE:
-        return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1])
-    else:
-        return 0
-
-
 @notify_celery.task(name="create-nightly-billing")
 @statsd(namespace="tasks")
 def create_nightly_billing(day_start=None):
@@ -34,91 +19,21 @@ def create_nightly_billing(day_start=None):
     if day_start is None:
         day_start = datetime.today() - timedelta(days=1)
 
-    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
-                        Rate.query.order_by(desc(Rate.valid_from)).all()]
-    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
-                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
-
     for i in range(0, 3):
         process_day = day_start - timedelta(days=i)
         ds = convert_bst_to_utc(datetime.combine(process_day, time.min))
         de = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
 
-        transit_data = db.session.query(
+        transit_data = fetch_billing_data(start_date=ds, end_date=de)
-            Notification.template_id,
-            Notification.service_id,
-            Notification.notification_type,
-            func.coalesce(Notification.sent_by,
-                          case(
-                              [
-                                  (Notification.notification_type == 'letter', 'dvla'),
-                                  (Notification.notification_type == 'sms', 'unknown'),
-                                  (Notification.notification_type == 'email', 'ses')
-                              ]),
-                          ).label('sent_by'),
-            func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'),
-            func.coalesce(Notification.international, False).label('international'),
-            func.sum(Notification.billable_units).label('billable_units'),
-            func.count().label('notifications_sent'),
-            Service.crown,
-        ).filter(
-            Notification.status != NOTIFICATION_CREATED,     # at created status, provider information is not available
-            Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
-            Notification.key_type != KEY_TYPE_TEST,
-            Notification.created_at >= ds,
-            Notification.created_at < de
-        ).group_by(
-            Notification.template_id,
-            Notification.service_id,
-            Notification.notification_type,
-            'sent_by',
-            Notification.rate_multiplier,
-            Notification.international,
-            Service.crown
-        ).join(
-            Service
-        ).all()
 
         updated_records = 0
         inserted_records = 0
 
         for data in transit_data:
-            update_count = FactBilling.query.filter(
+            inserted_records, updated_records = update_fact_billing(data,
-                FactBilling.bst_date == datetime.date(process_day),
+                                                                    inserted_records,
-                FactBilling.template_id == data.template_id,
-                FactBilling.service_id == data.service_id,
-                FactBilling.provider == data.sent_by,  # This could be zero - this is a bug that needs to be fixed.
-                FactBilling.rate_multiplier == data.rate_multiplier,
-                FactBilling.notification_type == data.notification_type,
-                FactBilling.international == data.international
-            ).update(
-                {"notifications_sent": data.notifications_sent,
-                 "billable_units": data.billable_units},
-                synchronize_session=False)
-
-            if update_count == 0:
-                billing_record = FactBilling(
-                    bst_date=process_day,
-                    template_id=data.template_id,
-                    service_id=data.service_id,
-                    notification_type=data.notification_type,
-                    provider=data.sent_by,
-                    rate_multiplier=data.rate_multiplier,
-                    international=data.international,
-                    billable_units=data.billable_units,
-                    notifications_sent=data.notifications_sent,
-                    rate=get_rate(non_letter_rates,
-                                  letter_rates,
-                                  data.notification_type,
                                                                     process_day,
-                                  data.crown,
+                                                                    updated_records)
-                                  data.rate_multiplier)
-                )
-                db.session.add(billing_record)
-                inserted_records += 1
-
-            updated_records += update_count
-            db.session.commit()
 
         current_app.logger.info('ft_billing {} to {}: {} rows updated, {} rows inserted'
                                 .format(ds, de, updated_records, inserted_records))

app/dao/fact_billing_dao.py

@@ -1,9 +1,16 @@
 from datetime import datetime, timedelta
-from sqlalchemy import func
+from sqlalchemy import func, case, desc, extract
 
 from app import db
 from app.dao.date_util import get_month_start_and_end_date_in_utc, get_financial_year
-from app.models import FactBilling
+from app.models import (
+    FactBilling, Notification, Service, NOTIFICATION_CREATED, NOTIFICATION_TECHNICAL_FAILURE,
+    KEY_TYPE_TEST,
+    LETTER_TYPE,
+    SMS_TYPE,
+    Rate,
+    LetterRate
+)
 from app.utils import convert_utc_to_bst
 
 
@@ -35,50 +42,143 @@ def fetch_annual_billing_by_month(service_id, billing_month, notification_type):
     return monthly_data, start_date
 
 
-def need_deltas(start_date, end_date, service_id, notification_type):
-    max_fact_billing_date = db.session.query(
-        func.max(FactBilling.bst_date)
-    ).filter(
-        FactBilling.notification_type == notification_type,
-        FactBilling.service_id == service_id,
-        FactBilling.bst_date >= start_date,
-        FactBilling.bst_date <= end_date
-    ).one()
-    print(max_fact_billing_date)
-    return max_fact_billing_date < end_date
-
-
 def fetch_annual_billing_for_year(service_id, year):
     year_start_date, year_end_date = get_financial_year(year)
     utcnow = datetime.utcnow()
     today = convert_utc_to_bst(utcnow)
-    last_2_days = utcnow - timedelta(days=2)
-    last_bst_date_for_ft = convert_utc_to_bst(last_2_days)
     # if year end date is less than today, we are calculating for data in the past and have no need for deltas.
     if year_end_date >= today:
-        todays_data = get_deltas(service_id, last_2_days, today)
+        last_2_days = utcnow - timedelta(days=2)
-        year_end_date = last_bst_date_for_ft
+        data = fetch_billing_data(start_date=last_2_days, end_date=today, service_id=service_id)
-
+        inserted_records = 0
-
+        updated_records = 0
+        for d in data:
+            update_fact_billing(data=data,
+                                inserted_records=inserted_records,
+                                process_day=d.created_at,
+                                updated_records=updated_records)
     yearly_data = db.session.query(
-        FactBilling.notifications_sent,
+        extract('month', FactBilling.bst_date).label("Month"),
-        FactBilling.billable_units,
+        func.sum(FactBilling.notifications_sent).label("notifications_sent"),
+        func.sum(FactBilling.billable_units).label("billable_units"),
         FactBilling.service_id,
-        FactBilling.notification_type,
         FactBilling.rate,
         FactBilling.rate_multiplier,
         FactBilling.international
     ).filter(
         FactBilling.service_id == service_id,
         FactBilling.bst_date >= year_start_date,
-        FactBilling.bst_date <= last_bst_date_for_ft
+        FactBilling.bst_date <= year_end_date
+    ).group_by(
+        extract('month', FactBilling.bst_date),
+        FactBilling.service_id,
+        FactBilling.rate,
+        FactBilling.rate_multiplier,
+        FactBilling.international
     ).all()
 
-    # today_data + yearly_data and aggregate by month
+    return yearly_data
 
 
-def get_deltas(service_id, start_date_end_date):
+def fetch_billing_data(start_date, end_date, service_id=None):
-    # query ft_billing data using queries from create_nightly_billing
+    transit_data = db.session.query(
-    return []
+        Notification.template_id,
+        Notification.service_id,
+        Notification.notification_type,
+        func.coalesce(Notification.sent_by,
+                      case(
+                          [
+                              (Notification.notification_type == 'letter', 'dvla'),
+                              (Notification.notification_type == 'sms', 'unknown'),
+                              (Notification.notification_type == 'email', 'ses')
+                          ]),
+                      ).label('sent_by'),
+        func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'),
+        func.coalesce(Notification.international, False).label('international'),
+        func.sum(Notification.billable_units).label('billable_units'),
+        func.count().label('notifications_sent'),
+        Service.crown,
+    ).filter(
+        Notification.status != NOTIFICATION_CREATED,  # at created status, provider information is not available
+        Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.created_at >= start_date,
+        Notification.created_at < end_date
+    ).group_by(
+        Notification.template_id,
+        Notification.service_id,
+        Notification.notification_type,
+        'sent_by',
+        Notification.rate_multiplier,
+        Notification.international,
+        Service.crown
+    ).join(
+        Service
+    )
+    if service_id:
+        transit_data.filter(Notification.service_id == service_id)
+    return transit_data.all()
 
 
+def get_rates_for_billing():
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+    return non_letter_rates, letter_rates
+
+
+def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
+    if notification_type == LETTER_TYPE:
+        return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2])
+    elif notification_type == SMS_TYPE:
+        return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1])
+    else:
+        return 0
+
+
+def update_fact_billing(data, inserted_records, process_day, updated_records):
+    non_letter_rates, letter_rates = get_rates_for_billing()
+
+    update_count = FactBilling.query.filter(
+        FactBilling.bst_date == datetime.date(process_day),
+        FactBilling.template_id == data.template_id,
+        FactBilling.service_id == data.service_id,
+        FactBilling.provider == data.sent_by,  # This could be zero - this is a bug that needs to be fixed.
+        FactBilling.rate_multiplier == data.rate_multiplier,
+        FactBilling.notification_type == data.notification_type,
+        FactBilling.international == data.international
+    ).update(
+        {"notifications_sent": data.notifications_sent,
+         "billable_units": data.billable_units},
+        synchronize_session=False)
+
+    if update_count == 0:
+        rate = get_rate(non_letter_rates,
+                        letter_rates,
+                        data.notification_type,
+                        process_day,
+                        data.crown,
+                        data.rate_multiplier)
+        billing_record = create_billing_record(data, rate, process_day)
+        db.session.add(billing_record)
+        inserted_records += 1
+    updated_records += update_count
+    db.session.commit()
+    return inserted_records, updated_records
+
+
+def create_billing_record(data, rate, process_day):
+    billing_record = FactBilling(
+        bst_date=process_day,
+        template_id=data.template_id,
+        service_id=data.service_id,
+        notification_type=data.notification_type,
+        provider=data.sent_by,
+        rate_multiplier=data.rate_multiplier,
+        international=data.international,
+        billable_units=data.billable_units,
+        notifications_sent=data.notifications_sent,
+        rate=rate
+    )
+    return billing_record

tests/app/celery/test_reporting_tasks.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta, date
 from tests.app.conftest import sample_notification
-from app.celery.reporting_tasks import create_nightly_billing, get_rate
+from app.celery.reporting_tasks import create_nightly_billing
+from app.dao.fact_billing_dao import get_rate
 from app.models import (FactBilling,
                         Notification,
                         LETTER_TYPE,
@@ -44,7 +45,7 @@ def test_create_nightly_billing_sms_rate_multiplier(
 
     yesterday = datetime.now() - timedelta(days=1)
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     # These are sms notifications
     sample_notification(
@@ -94,7 +95,7 @@ def test_create_nightly_billing_different_templates(
         mocker):
     yesterday = datetime.now() - timedelta(days=1)
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     sample_notification(
         notify_db,
@@ -147,7 +148,7 @@ def test_create_nightly_billing_different_sent_by(
         mocker):
     yesterday = datetime.now() - timedelta(days=1)
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     # These are sms notifications
     sample_notification(
@@ -197,7 +198,7 @@ def test_create_nightly_billing_letter(
         mocker):
     yesterday = datetime.now() - timedelta(days=1)
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     sample_notification(
         notify_db,
@@ -234,7 +235,7 @@ def test_create_nightly_billing_null_sent_by_sms(
         mocker):
     yesterday = datetime.now() - timedelta(days=1)
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     sample_notification(
         notify_db,
@@ -272,7 +273,7 @@ def test_create_nightly_billing_consolidate_from_3_days_delta(
         sample_template,
         mocker):
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     # create records from 11th to 15th
     for i in range(0, 5):
@@ -365,7 +366,7 @@ def test_create_nightly_billing_use_BST(
         sample_template,
         mocker):
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     sample_notification(
         notify_db,
@@ -414,7 +415,7 @@ def test_create_nightly_billing_update_when_record_exists(
         sample_template,
         mocker):
 
-    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+    mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
 
     sample_notification(
         notify_db,

tests/app/dao/test_ft_billing_dao.py

@@ -1,11 +1,12 @@
 from datetime import datetime
 from decimal import Decimal
 
-from freezegun import freeze_time
+from app.dao.fact_billing_dao import fetch_annual_billing_by_month, fetch_annual_billing_for_year
-
+from tests.app.db import (
-from app.dao.date_util import get_month_start_and_end_date_in_utc
+    create_ft_billing,
-from app.dao.fact_billing_dao import fetch_annual_billing_by_month, need_deltas
+    create_service,
-from tests.app.db import create_ft_billing, create_service, create_template, create_notification
+    create_template
+)
 
 
 def test_fetch_annual_billing_by_month(notify_db_session):
@@ -20,23 +21,20 @@ def test_fetch_annual_billing_by_month(notify_db_session):
     results, month = fetch_annual_billing_by_month(service_id=record.service_id,
                                                    billing_month=datetime(2018, 1, 1),
                                                    notification_type='email')
-
     assert len(results) == 1
     assert results[0] == (31, Decimal('31'), service.id, 'email', Decimal('0'), Decimal('1'), False)
     assert month == datetime(2018, 1, 1)
 
 
-@freeze_time("2018-01-21 13:00:00")
+def test_fetch_annual_billing_for_year(notify_db_session):
-def test_need_deltas(notify_db_session):
     service = create_service()
     template = create_template(service=service, template_type="email")
-    for i in range(1, 21):
+    for i in range(1, 31):
-        record = create_ft_billing(bst_date='2018-01-{}'.format(i),
+        create_ft_billing(bst_date='2018-06-{}'.format(i),
                           service=service,
                           template=template,
                           notification_type='email')
-    start_date, end_date = get_month_start_and_end_date_in_utc(datetime.utcnow())
+    results = fetch_annual_billing_for_year(service_id=service.id,
+                                            year=2018)
 
-    result = need_deltas(start_date=start_date, end_date=end_date,
+    assert results
-                         service_id=service.id, notification_type='email')
-    assert result

tests/app/db.py

@@ -529,8 +529,8 @@ def create_daily_sorted_letter(billing_day=date(2018, 1, 18),
 
 def create_ft_billing(bst_date,
                       notification_type,
-                      template = None,
+                      template=None,
-                      service = None,
+                      service=None,
                       provider='test',
                       rate_multiplier=1,
                       international=False,