Create schedueled job, fixed tests

app/celery/reporting_tasks.py

@@ -1,6 +1,7 @@
 from app import notify_celery
 from notifications_utils.statsd_decorators import statsd
 import random
+from datetime import datetime, timedelta
 from app.models import (Notification,
                         Rate,
                         NOTIFICATION_CREATED,
@@ -15,11 +16,7 @@ from sqlalchemy import func, desc, case
 from app.dao.dao_utils import transactional
 
 
-def get_rate(notification_type, date, crown=None, rate_multiplier=None):
+def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
-    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
-                        Rate.query.order_by(desc(Rate.valid_from)).all()]
-    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
-                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
 
     if notification_type == LETTER_TYPE:
         return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2])
@@ -32,7 +29,15 @@ def get_rate(notification_type, date, crown=None, rate_multiplier=None):
 @notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300)
 @statsd(namespace="tasks")
 @transactional
-def create_nightly_billing(self, day_start):
+def create_nightly_billing(self, day_start=None):
+    if day_start is None:
+        day_start = datetime.date(datetime.utcnow()) - timedelta(days=3)   # Nightly jobs consolidating last 3 days
+        # Task to be run after mid-night
+
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
 
     transit_data = db.session.query(
         func.date_trunc('day', Notification.created_at).label('day_created'),
@@ -56,7 +61,7 @@ def create_nightly_billing(self, day_start):
         Notification.status != NOTIFICATION_CREATED,        # at created status, provider information is not available
         Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
         Notification.key_type != KEY_TYPE_TEST,
-        Notification.created_at >= '2018-01-01;'
+        Notification.created_at >= day_start
     ).group_by(
         'day_created',
         Notification.template_id,
@@ -94,6 +99,11 @@ def create_nightly_billing(self, day_start):
                 international=data.international,
                 billable_units=data.billable_units,
                 notifications_sent=data.notifications_sent,
-                rate=get_rate(data.notification_type, data.day_created, data.crown, data.rate_multiplier)
+                rate=get_rate(non_letter_rates,
+                              letter_rates,
+                              data.notification_type,
+                              data.day_created,
+                              data.crown,
+                              data.rate_multiplier)
             )
             db.session.add(billing_record)

app/config.py

@@ -213,6 +213,11 @@ class Config(object):
             'schedule': crontab(hour=3, minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=3, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
         'remove_sms_email_jobs': {
             'task': 'remove_csv_files',
             'schedule': crontab(hour=4, minute=0),

tests/app/celery/test_reporting_tasks.py

@@ -1,28 +1,33 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 from tests.app.conftest import sample_notification
 from app.celery.reporting_tasks import create_nightly_billing, get_rate
-from app.models import FactBilling
+from app.models import (FactBilling,
+                        Notification,
+                        LETTER_TYPE,
+                        EMAIL_TYPE,
+                        SMS_TYPE)
 from decimal import Decimal
 import pytest
 from app.dao.letter_rate_dao import dao_create_letter_rate
 from app.models import LetterRate, Rate
 from app import db
+from freezegun import freeze_time
+from sqlalchemy import desc
 
 
 def test_reporting_should_have_decorated_tasks_functions():
     assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing'
 
 
-def mocker_get_rate(notification_type, date, crown=None, rate_multiplier=None):
+def mocker_get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
-    if notification_type == 'letter':
+    if notification_type == LETTER_TYPE:
         return Decimal(2.1)
-    elif notification_type == 'sms':
+    elif notification_type == SMS_TYPE:
         return Decimal(1.33)
-    elif notification_type == 'email':
+    elif notification_type == EMAIL_TYPE:
         return Decimal(0)
 
 
-# Test when notifications with all dimensions are the same, and when the rate is different
 @pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
                          [(1.0, 1, 2, [1]),
                           (2.0, 2, 1, [1, 2])])
@@ -40,7 +45,6 @@ def test_create_nightly_billing_sms_rate_multiplier(
     yesterday = datetime.now() - timedelta(days=1)
 
     mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
-    # two_days_ago = datetime.now() - timedelta(days=6)
 
     # These are sms notifications
     sample_notification(
@@ -67,10 +71,10 @@ def test_create_nightly_billing_sms_rate_multiplier(
         rate_multiplier=second_rate,
         billable_units=1,
     )
-    # before running migration no records
+
     records = FactBilling.query.all()
     assert len(records) == 0
-    # before record data after migration
+
     create_nightly_billing(yesterday)
     records = FactBilling.query.all()
     assert len(records) == records_num
@@ -91,9 +95,7 @@ def test_create_nightly_billing_different_templates(
     yesterday = datetime.now() - timedelta(days=1)
 
     mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
-    # two_days_ago = datetime.now() - timedelta(days=6)
 
-    # These are sms notifications
     sample_notification(
         notify_db,
         notify_db_session,
@@ -118,13 +120,13 @@ def test_create_nightly_billing_different_templates(
         rate_multiplier=0,
         billable_units=0,
     )
-    # before running migration no records
+
     records = FactBilling.query.all()
     assert len(records) == 0
-    # before record data after migration
+
     create_nightly_billing(yesterday)
     records = FactBilling.query.order_by('rate_multiplier').all()
-    # First record is ses and second record is sms
+
     assert len(records) == 2
     multiplier = [0, 1]
     billable_units = [0, 1]
@@ -146,7 +148,6 @@ def test_create_nightly_billing_different_sent_by(
     yesterday = datetime.now() - timedelta(days=1)
 
     mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
-    # two_days_ago = datetime.now() - timedelta(days=6)
 
     # These are sms notifications
     sample_notification(
@@ -173,13 +174,13 @@ def test_create_nightly_billing_different_sent_by(
         rate_multiplier=1.0,
         billable_units=1,
     )
-    # before running migration no records
+
     records = FactBilling.query.all()
     assert len(records) == 0
-    # before record data after migration
+
     create_nightly_billing(yesterday)
     records = FactBilling.query.order_by('rate_multiplier').all()
-    # First record is ses and second record is sms
+
     assert len(records) == 2
     for i, record in enumerate(records):
         assert record.bst_date == datetime.date(yesterday)
@@ -197,9 +198,7 @@ def test_create_nightly_billing_letter(
     yesterday = datetime.now() - timedelta(days=1)
 
     mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
-    # two_days_ago = datetime.now() - timedelta(days=6)
 
-    # These are sms notifications
     sample_notification(
         notify_db,
         notify_db_session,
@@ -213,15 +212,14 @@ def test_create_nightly_billing_letter(
         billable_units=2,
     )
 
-    # before running migration no records
     records = FactBilling.query.all()
     assert len(records) == 0
-    # before record data after migration
+
     create_nightly_billing(yesterday)
     records = FactBilling.query.order_by('rate_multiplier').all()
     assert len(records) == 1
     record = records[0]
-    assert record.notification_type == 'letter'
+    assert record.notification_type == LETTER_TYPE
     assert record.bst_date == datetime.date(yesterday)
     assert record.rate == Decimal(2.1)
     assert record.billable_units == 2
@@ -237,9 +235,7 @@ def test_create_nightly_billing_null_sent_by_sms(
     yesterday = datetime.now() - timedelta(days=1)
 
     mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
-    # two_days_ago = datetime.now() - timedelta(days=6)
 
-    # These are sms notifications
     sample_notification(
         notify_db,
         notify_db_session,
@@ -253,13 +249,12 @@ def test_create_nightly_billing_null_sent_by_sms(
         billable_units=1,
     )
 
-    # before running migration no records
     records = FactBilling.query.all()
     assert len(records) == 0
-    # before record data after migration
+
     create_nightly_billing(yesterday)
     records = FactBilling.query.all()
-    # First record is ses and second record is sms
+
     assert len(records) == 1
     record = records[0]
     assert record.bst_date == datetime.date(yesterday)
@@ -269,6 +264,45 @@ def test_create_nightly_billing_null_sent_by_sms(
     assert record.provider in ['mmg', 'firetext']
 
 
+@freeze_time('2018-01-15T03:30:00')
+def test_create_nightly_billing_consolidate_from_3_days_delta(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        mocker):
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    # create records from 11th to 15th
+    for i in range(0, 5):
+        sample_notification(
+            notify_db,
+            notify_db_session,
+            created_at=datetime.now() - timedelta(days=i),
+            service=sample_service,
+            template=sample_template,
+            status='delivered',
+            sent_by=None,
+            international=False,
+            rate_multiplier=1.0,
+            billable_units=1,
+        )
+
+    notification = Notification.query.order_by(Notification.created_at).all()
+    assert datetime.date(notification[0].created_at) == date(2018, 1, 11)
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing()
+    records = FactBilling.query.order_by(FactBilling.bst_date).all()
+
+    assert len(records) == 4
+    assert records[0].bst_date == date(2018, 1, 12)
+    assert records[-1].bst_date == date(2018, 1, 15)
+
+
 def test_get_rate_for_letter_latest(notify_db, notify_db_session):
     letter_rate = LetterRate(start_date=datetime(2017, 12, 1),
                              rate=Decimal(0.33),
@@ -285,7 +319,12 @@ def test_get_rate_for_letter_latest(notify_db, notify_db_session):
                              post_class='second')
     dao_create_letter_rate(letter_rate)
 
-    rate = get_rate('letter', datetime(2018, 1, 1), True, 1)
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+
+    rate = get_rate(non_letter_rates, letter_rates, LETTER_TYPE, datetime(2018, 1, 1), True, 1)
     assert rate == Decimal(0.33)
 
 
@@ -298,15 +337,20 @@ def test_get_rate_for_sms_and_email(notify_db, notify_db_session):
     dao_create_letter_rate(letter_rate)
     sms_rate = Rate(valid_from=datetime(2017, 12, 1),
                     rate=Decimal(0.15),
-                    notification_type='sms')
+                    notification_type=SMS_TYPE)
     db.session.add(sms_rate)
     email_rate = Rate(valid_from=datetime(2017, 12, 1),
                       rate=Decimal(0),
-                      notification_type='email')
+                      notification_type=EMAIL_TYPE)
     db.session.add(email_rate)
 
-    rate = get_rate('sms', datetime(2018, 1, 1))
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+
+    rate = get_rate(non_letter_rates, letter_rates, SMS_TYPE, datetime(2018, 1, 1))
     assert rate == Decimal(0.15)
 
-    rate = get_rate('email', datetime(2018, 1, 1))
+    rate = get_rate(non_letter_rates, letter_rates, EMAIL_TYPE, datetime(2018, 1, 1))
     assert rate == Decimal(0)