Merge pull request #1771 from alphagov/vb-billing-periodic

Fine tune reporting model and create task for migrating data [#155511547]

app/celery/reporting_tasks.py

@@ -0,0 +1,109 @@
+from app import notify_celery
+from notifications_utils.statsd_decorators import statsd
+import random
+from datetime import datetime, timedelta
+from app.models import (Notification,
+                        Rate,
+                        NOTIFICATION_CREATED,
+                        NOTIFICATION_TECHNICAL_FAILURE,
+                        KEY_TYPE_TEST,
+                        LetterRate,
+                        FactBilling,
+                        Service,
+                        LETTER_TYPE, SMS_TYPE)
+from app import db
+from sqlalchemy import func, desc, case
+from app.dao.dao_utils import transactional
+
+
+def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
+
+    if notification_type == LETTER_TYPE:
+        return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2])
+    elif notification_type == SMS_TYPE:
+        return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1])
+    else:
+        return 0
+
+
+@notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300)
+@statsd(namespace="tasks")
+@transactional
+def create_nightly_billing(self, day_start=None):
+    if day_start is None:
+        day_start = datetime.date(datetime.utcnow()) - timedelta(days=3)   # Nightly jobs consolidating last 3 days
+        # Task to be run after mid-night
+
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+
+    transit_data = db.session.query(
+        func.date_trunc('day', Notification.created_at).label('day_created'),
+        Notification.template_id,
+        Notification.service_id,
+        Notification.notification_type,
+        case(
+            [
+                (Notification.notification_type == 'letter', func.coalesce(Notification.sent_by, 'dvla')),
+                (Notification.notification_type == 'sms',
+                 func.coalesce(Notification.sent_by, random.choice(['mmg', 'firetext'])))
+            ],
+            else_='ses'
+        ).label('sent_by'),  # This could be null - this is a bug to be fixed.
+        func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'),
+        func.coalesce(Notification.international, False).label('international'),
+        func.sum(Notification.billable_units).label('billable_units'),
+        func.count().label('notifications_sent'),
+        Service.crown,
+    ).filter(
+        Notification.status != NOTIFICATION_CREATED,        # at created status, provider information is not available
+        Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
+        Notification.key_type != KEY_TYPE_TEST,
+        Notification.created_at >= day_start
+    ).group_by(
+        'day_created',
+        Notification.template_id,
+        Notification.service_id,
+        Notification.notification_type,
+        'sent_by',
+        Notification.rate_multiplier,
+        Notification.international,
+        Service.crown
+    ).join(
+        Service
+    ).order_by(
+        'day_created'
+    ).all()
+
+    for data in transit_data:
+        update_count = FactBilling.query.filter(
+            FactBilling.bst_date == data.day_created,
+            FactBilling.template_id == data.template_id,
+            FactBilling.provider == data.sent_by,  # This could be zero - this is a bug that needs to be fixed.
+            FactBilling.rate_multiplier == data.rate_multiplier,
+            FactBilling.international == data.international,
+        ).update(
+            {"notifications_sent": data.notifications_sent,
+             "billable_units": data.billable_units},
+            synchronize_session=False)
+        if update_count == 0:
+            billing_record = FactBilling(
+                bst_date=data.day_created,
+                template_id=data.template_id,
+                service_id=data.service_id,
+                notification_type=data.notification_type,
+                provider=data.sent_by,
+                rate_multiplier=data.rate_multiplier,
+                international=data.international,
+                billable_units=data.billable_units,
+                notifications_sent=data.notifications_sent,
+                rate=get_rate(non_letter_rates,
+                              letter_rates,
+                              data.notification_type,
+                              data.day_created,
+                              data.crown,
+                              data.rate_multiplier)
+            )
+            db.session.add(billing_record)

app/config.py

@@ -213,6 +213,11 @@ class Config(object):
             'schedule': crontab(hour=3, minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'create-nightly-billing': {
+            'task': 'create-nightly-billing',
+            'schedule': crontab(hour=3, minute=30),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
         'remove_sms_email_jobs': {
             'task': 'remove_csv_files',
             'schedule': crontab(hour=4, minute=0),

app/models.py

@@ -1772,15 +1772,12 @@ class FactBilling(db.Model):
     __tablename__ = "ft_billing"
 
     bst_date = db.Column(db.Date, nullable=False, primary_key=True, index=True)
-    template_id = db.Column(UUID(as_uuid=True), nullable=True, primary_key=True, index=True)
-    service_id = db.Column(UUID(as_uuid=True), nullable=True, index=True)
-    organisation_id = db.Column(UUID(as_uuid=True), nullable=True)
-    annual_billing_id = db.Column(UUID(as_uuid=True), nullable=True)
+    template_id = db.Column(UUID(as_uuid=True), nullable=False, primary_key=True, index=True)
+    service_id = db.Column(UUID(as_uuid=True), nullable=False, index=True)
     notification_type = db.Column(db.Text, nullable=True)
-    provider = db.Column(db.Text, nullable=True)
-    crown = db.Column(db.Text, nullable=True)
-    rate_multiplier = db.Column(db.Numeric(), nullable=True)
-    international = db.Column(db.Boolean, nullable=True)
+    provider = db.Column(db.Text, nullable=True, primary_key=True)
+    rate_multiplier = db.Column(db.Numeric(), nullable=True, primary_key=True)
+    international = db.Column(db.Boolean, nullable=True, primary_key=True)
     rate = db.Column(db.Numeric(), nullable=True)
     billable_units = db.Column(db.Numeric(), nullable=True)
     notifications_sent = db.Column(db.Integer(), nullable=True)

migrations/versions/0179_billing_primary_const.py

@@ -0,0 +1,36 @@
+"""
+
+Revision ID: 0179_billing_primary_const
+Revises: 0178_add_filename
+Create Date: 2018-03-13 14:52:40.413474
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = '0179_billing_primary_const'
+down_revision = '0178_add_filename'
+
+
+def upgrade():
+    op.drop_column('ft_billing', 'crown')
+    op.drop_column('ft_billing', 'annual_billing_id')
+    op.drop_column('ft_billing', 'organisation_id')
+    op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary')
+    # These are the orthogonal dimensions that define a row (except international).
+    # These entries define a unique record.
+    op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date',
+                                                            'template_id',
+                                                            'rate_multiplier',
+                                                            'provider',
+                                                            'international'])
+
+
+def downgrade():
+    op.add_column('ft_billing', sa.Column('organisation_id', postgresql.UUID(), autoincrement=False, nullable=True))
+    op.add_column('ft_billing', sa.Column('annual_billing_id', postgresql.UUID(), autoincrement=False, nullable=True))
+    op.add_column('ft_billing', sa.Column('crown', sa.TEXT(), autoincrement=False, nullable=True))
+    op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary')
+    op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date',
+                                                            'template_id'])

tests/app/celery/test_reporting_tasks.py

@@ -0,0 +1,356 @@
+from datetime import datetime, timedelta, date
+from tests.app.conftest import sample_notification
+from app.celery.reporting_tasks import create_nightly_billing, get_rate
+from app.models import (FactBilling,
+                        Notification,
+                        LETTER_TYPE,
+                        EMAIL_TYPE,
+                        SMS_TYPE)
+from decimal import Decimal
+import pytest
+from app.dao.letter_rate_dao import dao_create_letter_rate
+from app.models import LetterRate, Rate
+from app import db
+from freezegun import freeze_time
+from sqlalchemy import desc
+
+
+def test_reporting_should_have_decorated_tasks_functions():
+    assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing'
+
+
+def mocker_get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
+    if notification_type == LETTER_TYPE:
+        return Decimal(2.1)
+    elif notification_type == SMS_TYPE:
+        return Decimal(1.33)
+    elif notification_type == EMAIL_TYPE:
+        return Decimal(0)
+
+
+@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
+                         [(1.0, 1, 2, [1]),
+                          (2.0, 2, 1, [1, 2])])
+def test_create_nightly_billing_sms_rate_multiplier(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        mocker,
+        second_rate,
+        records_num,
+        billable_units,
+        multiplier):
+
+    yesterday = datetime.now() - timedelta(days=1)
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    # These are sms notifications
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by='mmg',
+        international=False,
+        rate_multiplier=1.0,
+        billable_units=1,
+    )
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by='mmg',
+        international=False,
+        rate_multiplier=second_rate,
+        billable_units=1,
+    )
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing(yesterday)
+    records = FactBilling.query.all()
+    assert len(records) == records_num
+    for i, record in enumerate(records):
+        assert record.bst_date == datetime.date(yesterday)
+        assert record.rate == Decimal(1.33)
+        assert record.billable_units == billable_units
+        assert record.rate_multiplier == multiplier[i]
+
+
+def test_create_nightly_billing_different_templates(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        sample_email_template,
+        mocker):
+    yesterday = datetime.now() - timedelta(days=1)
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by='mmg',
+        international=False,
+        rate_multiplier=1.0,
+        billable_units=1,
+    )
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_email_template,
+        status='delivered',
+        sent_by='ses',
+        international=False,
+        rate_multiplier=0,
+        billable_units=0,
+    )
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing(yesterday)
+    records = FactBilling.query.order_by('rate_multiplier').all()
+
+    assert len(records) == 2
+    multiplier = [0, 1]
+    billable_units = [0, 1]
+    rate = [0, Decimal(1.33)]
+    for i, record in enumerate(records):
+        assert record.bst_date == datetime.date(yesterday)
+        assert record.rate == rate[i]
+        assert record.billable_units == billable_units[i]
+        assert record.rate_multiplier == multiplier[i]
+
+
+def test_create_nightly_billing_different_sent_by(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        sample_email_template,
+        mocker):
+    yesterday = datetime.now() - timedelta(days=1)
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    # These are sms notifications
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by='mmg',
+        international=False,
+        rate_multiplier=1.0,
+        billable_units=1,
+    )
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by='firetext',
+        international=False,
+        rate_multiplier=1.0,
+        billable_units=1,
+    )
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing(yesterday)
+    records = FactBilling.query.order_by('rate_multiplier').all()
+
+    assert len(records) == 2
+    for i, record in enumerate(records):
+        assert record.bst_date == datetime.date(yesterday)
+        assert record.rate == Decimal(1.33)
+        assert record.billable_units == 1
+        assert record.rate_multiplier == 1.0
+
+
+def test_create_nightly_billing_letter(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_letter_template,
+        mocker):
+    yesterday = datetime.now() - timedelta(days=1)
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_letter_template,
+        status='delivered',
+        sent_by='dvla',
+        international=False,
+        rate_multiplier=2.0,
+        billable_units=2,
+    )
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing(yesterday)
+    records = FactBilling.query.order_by('rate_multiplier').all()
+    assert len(records) == 1
+    record = records[0]
+    assert record.notification_type == LETTER_TYPE
+    assert record.bst_date == datetime.date(yesterday)
+    assert record.rate == Decimal(2.1)
+    assert record.billable_units == 2
+    assert record.rate_multiplier == 2.0
+
+
+def test_create_nightly_billing_null_sent_by_sms(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        mocker):
+    yesterday = datetime.now() - timedelta(days=1)
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    sample_notification(
+        notify_db,
+        notify_db_session,
+        created_at=yesterday,
+        service=sample_service,
+        template=sample_template,
+        status='delivered',
+        sent_by=None,
+        international=False,
+        rate_multiplier=1.0,
+        billable_units=1,
+    )
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing(yesterday)
+    records = FactBilling.query.all()
+
+    assert len(records) == 1
+    record = records[0]
+    assert record.bst_date == datetime.date(yesterday)
+    assert record.rate == Decimal(1.33)
+    assert record.billable_units == 1
+    assert record.rate_multiplier == 1
+    assert record.provider in ['mmg', 'firetext']
+
+
+@freeze_time('2018-01-15T03:30:00')
+def test_create_nightly_billing_consolidate_from_3_days_delta(
+        notify_db,
+        notify_db_session,
+        sample_service,
+        sample_template,
+        mocker):
+
+    mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
+
+    # create records from 11th to 15th
+    for i in range(0, 5):
+        sample_notification(
+            notify_db,
+            notify_db_session,
+            created_at=datetime.now() - timedelta(days=i),
+            service=sample_service,
+            template=sample_template,
+            status='delivered',
+            sent_by=None,
+            international=False,
+            rate_multiplier=1.0,
+            billable_units=1,
+        )
+
+    notification = Notification.query.order_by(Notification.created_at).all()
+    assert datetime.date(notification[0].created_at) == date(2018, 1, 11)
+
+    records = FactBilling.query.all()
+    assert len(records) == 0
+
+    create_nightly_billing()
+    records = FactBilling.query.order_by(FactBilling.bst_date).all()
+
+    assert len(records) == 4
+    assert records[0].bst_date == date(2018, 1, 12)
+    assert records[-1].bst_date == date(2018, 1, 15)
+
+
+def test_get_rate_for_letter_latest(notify_db, notify_db_session):
+    letter_rate = LetterRate(start_date=datetime(2017, 12, 1),
+                             rate=Decimal(0.33),
+                             crown=True,
+                             sheet_count=1,
+                             post_class='second')
+
+    dao_create_letter_rate(letter_rate)
+    letter_rate = LetterRate(start_date=datetime(2016, 12, 1),
+                             end_date=datetime(2017, 12, 1),
+                             rate=Decimal(0.30),
+                             crown=True,
+                             sheet_count=1,
+                             post_class='second')
+    dao_create_letter_rate(letter_rate)
+
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+
+    rate = get_rate(non_letter_rates, letter_rates, LETTER_TYPE, datetime(2018, 1, 1), True, 1)
+    assert rate == Decimal(0.33)
+
+
+def test_get_rate_for_sms_and_email(notify_db, notify_db_session):
+    letter_rate = LetterRate(start_date=datetime(2017, 12, 1),
+                             rate=Decimal(0.33),
+                             crown=True,
+                             sheet_count=1,
+                             post_class='second')
+    dao_create_letter_rate(letter_rate)
+    sms_rate = Rate(valid_from=datetime(2017, 12, 1),
+                    rate=Decimal(0.15),
+                    notification_type=SMS_TYPE)
+    db.session.add(sms_rate)
+    email_rate = Rate(valid_from=datetime(2017, 12, 1),
+                      rate=Decimal(0),
+                      notification_type=EMAIL_TYPE)
+    db.session.add(email_rate)
+
+    non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
+                        Rate.query.order_by(desc(Rate.valid_from)).all()]
+    letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
+                    LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
+
+    rate = get_rate(non_letter_rates, letter_rates, SMS_TYPE, datetime(2018, 1, 1))
+    assert rate == Decimal(0.15)
+
+    rate = get_rate(non_letter_rates, letter_rates, EMAIL_TYPE, datetime(2018, 1, 1))
+    assert rate == Decimal(0)

tests/app/conftest.py

@@ -533,6 +533,7 @@ def sample_notification(
     api_key=None,
     key_type=KEY_TYPE_NORMAL,
     sent_by=None,
+    international=False,
     client_reference=None,
     rate_multiplier=1.0,
     scheduled_for=None,