diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py new file mode 100644 index 000000000..37746742b --- /dev/null +++ b/app/celery/reporting_tasks.py @@ -0,0 +1,109 @@ +from app import notify_celery +from notifications_utils.statsd_decorators import statsd +import random +from datetime import datetime, timedelta +from app.models import (Notification, + Rate, + NOTIFICATION_CREATED, + NOTIFICATION_TECHNICAL_FAILURE, + KEY_TYPE_TEST, + LetterRate, + FactBilling, + Service, + LETTER_TYPE, SMS_TYPE) +from app import db +from sqlalchemy import func, desc, case +from app.dao.dao_utils import transactional + + +def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): + + if notification_type == LETTER_TYPE: + return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2]) + elif notification_type == SMS_TYPE: + return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1]) + else: + return 0 + + +@notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300) +@statsd(namespace="tasks") +@transactional +def create_nightly_billing(self, day_start=None): + if day_start is None: + day_start = datetime.date(datetime.utcnow()) - timedelta(days=3) # Nightly jobs consolidating last 3 days + # Task to be run after mid-night + + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + transit_data = db.session.query( + func.date_trunc('day', Notification.created_at).label('day_created'), + Notification.template_id, + Notification.service_id, + Notification.notification_type, + case( + [ + (Notification.notification_type == 'letter', func.coalesce(Notification.sent_by, 'dvla')), + (Notification.notification_type == 'sms', + func.coalesce(Notification.sent_by, random.choice(['mmg', 'firetext']))) + ], + else_='ses' + ).label('sent_by'), # This could be null - this is a bug to be fixed. + func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'), + func.coalesce(Notification.international, False).label('international'), + func.sum(Notification.billable_units).label('billable_units'), + func.count().label('notifications_sent'), + Service.crown, + ).filter( + Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available + Notification.status != NOTIFICATION_TECHNICAL_FAILURE, + Notification.key_type != KEY_TYPE_TEST, + Notification.created_at >= day_start + ).group_by( + 'day_created', + Notification.template_id, + Notification.service_id, + Notification.notification_type, + 'sent_by', + Notification.rate_multiplier, + Notification.international, + Service.crown + ).join( + Service + ).order_by( + 'day_created' + ).all() + + for data in transit_data: + update_count = FactBilling.query.filter( + FactBilling.bst_date == data.day_created, + FactBilling.template_id == data.template_id, + FactBilling.provider == data.sent_by, # This could be zero - this is a bug that needs to be fixed. + FactBilling.rate_multiplier == data.rate_multiplier, + FactBilling.international == data.international, + ).update( + {"notifications_sent": data.notifications_sent, + "billable_units": data.billable_units}, + synchronize_session=False) + if update_count == 0: + billing_record = FactBilling( + bst_date=data.day_created, + template_id=data.template_id, + service_id=data.service_id, + notification_type=data.notification_type, + provider=data.sent_by, + rate_multiplier=data.rate_multiplier, + international=data.international, + billable_units=data.billable_units, + notifications_sent=data.notifications_sent, + rate=get_rate(non_letter_rates, + letter_rates, + data.notification_type, + data.day_created, + data.crown, + data.rate_multiplier) + ) + db.session.add(billing_record) diff --git a/app/config.py b/app/config.py index feaa8d0f9..a69f61c68 100644 --- a/app/config.py +++ b/app/config.py @@ -213,6 +213,11 @@ class Config(object): 'schedule': crontab(hour=3, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, + 'create-nightly-billing': { + 'task': 'create-nightly-billing', + 'schedule': crontab(hour=3, minute=30), + 'options': {'queue': QueueNames.PERIODIC} + }, 'remove_sms_email_jobs': { 'task': 'remove_csv_files', 'schedule': crontab(hour=4, minute=0), diff --git a/app/models.py b/app/models.py index 2957b7eb7..f4cbc82af 100644 --- a/app/models.py +++ b/app/models.py @@ -1772,15 +1772,12 @@ class FactBilling(db.Model): __tablename__ = "ft_billing" bst_date = db.Column(db.Date, nullable=False, primary_key=True, index=True) - template_id = db.Column(UUID(as_uuid=True), nullable=True, primary_key=True, index=True) - service_id = db.Column(UUID(as_uuid=True), nullable=True, index=True) - organisation_id = db.Column(UUID(as_uuid=True), nullable=True) - annual_billing_id = db.Column(UUID(as_uuid=True), nullable=True) + template_id = db.Column(UUID(as_uuid=True), nullable=False, primary_key=True, index=True) + service_id = db.Column(UUID(as_uuid=True), nullable=False, index=True) notification_type = db.Column(db.Text, nullable=True) - provider = db.Column(db.Text, nullable=True) - crown = db.Column(db.Text, nullable=True) - rate_multiplier = db.Column(db.Numeric(), nullable=True) - international = db.Column(db.Boolean, nullable=True) + provider = db.Column(db.Text, nullable=True, primary_key=True) + rate_multiplier = db.Column(db.Numeric(), nullable=True, primary_key=True) + international = db.Column(db.Boolean, nullable=True, primary_key=True) rate = db.Column(db.Numeric(), nullable=True) billable_units = db.Column(db.Numeric(), nullable=True) notifications_sent = db.Column(db.Integer(), nullable=True) diff --git a/migrations/versions/0179_billing_primary_const.py b/migrations/versions/0179_billing_primary_const.py new file mode 100644 index 000000000..ab5b49739 --- /dev/null +++ b/migrations/versions/0179_billing_primary_const.py @@ -0,0 +1,36 @@ +""" + +Revision ID: 0179_billing_primary_const +Revises: 0178_add_filename +Create Date: 2018-03-13 14:52:40.413474 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0179_billing_primary_const' +down_revision = '0178_add_filename' + + +def upgrade(): + op.drop_column('ft_billing', 'crown') + op.drop_column('ft_billing', 'annual_billing_id') + op.drop_column('ft_billing', 'organisation_id') + op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary') + # These are the orthogonal dimensions that define a row (except international). + # These entries define a unique record. + op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date', + 'template_id', + 'rate_multiplier', + 'provider', + 'international']) + + +def downgrade(): + op.add_column('ft_billing', sa.Column('organisation_id', postgresql.UUID(), autoincrement=False, nullable=True)) + op.add_column('ft_billing', sa.Column('annual_billing_id', postgresql.UUID(), autoincrement=False, nullable=True)) + op.add_column('ft_billing', sa.Column('crown', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary') + op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date', + 'template_id']) diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py new file mode 100644 index 000000000..3b3fcdacb --- /dev/null +++ b/tests/app/celery/test_reporting_tasks.py @@ -0,0 +1,356 @@ +from datetime import datetime, timedelta, date +from tests.app.conftest import sample_notification +from app.celery.reporting_tasks import create_nightly_billing, get_rate +from app.models import (FactBilling, + Notification, + LETTER_TYPE, + EMAIL_TYPE, + SMS_TYPE) +from decimal import Decimal +import pytest +from app.dao.letter_rate_dao import dao_create_letter_rate +from app.models import LetterRate, Rate +from app import db +from freezegun import freeze_time +from sqlalchemy import desc + + +def test_reporting_should_have_decorated_tasks_functions(): + assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing' + + +def mocker_get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): + if notification_type == LETTER_TYPE: + return Decimal(2.1) + elif notification_type == SMS_TYPE: + return Decimal(1.33) + elif notification_type == EMAIL_TYPE: + return Decimal(0) + + +@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier', + [(1.0, 1, 2, [1]), + (2.0, 2, 1, [1, 2])]) +def test_create_nightly_billing_sms_rate_multiplier( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker, + second_rate, + records_num, + billable_units, + multiplier): + + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=second_rate, + billable_units=1, + ) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing(yesterday) + records = FactBilling.query.all() + assert len(records) == records_num + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == billable_units + assert record.rate_multiplier == multiplier[i] + + +def test_create_nightly_billing_different_templates( + notify_db, + notify_db_session, + sample_service, + sample_template, + sample_email_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_email_template, + status='delivered', + sent_by='ses', + international=False, + rate_multiplier=0, + billable_units=0, + ) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + + assert len(records) == 2 + multiplier = [0, 1] + billable_units = [0, 1] + rate = [0, Decimal(1.33)] + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == rate[i] + assert record.billable_units == billable_units[i] + assert record.rate_multiplier == multiplier[i] + + +def test_create_nightly_billing_different_sent_by( + notify_db, + notify_db_session, + sample_service, + sample_template, + sample_email_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='firetext', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + + assert len(records) == 2 + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == 1 + assert record.rate_multiplier == 1.0 + + +def test_create_nightly_billing_letter( + notify_db, + notify_db_session, + sample_service, + sample_letter_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_letter_template, + status='delivered', + sent_by='dvla', + international=False, + rate_multiplier=2.0, + billable_units=2, + ) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + assert len(records) == 1 + record = records[0] + assert record.notification_type == LETTER_TYPE + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(2.1) + assert record.billable_units == 2 + assert record.rate_multiplier == 2.0 + + +def test_create_nightly_billing_null_sent_by_sms( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by=None, + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing(yesterday) + records = FactBilling.query.all() + + assert len(records) == 1 + record = records[0] + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == 1 + assert record.rate_multiplier == 1 + assert record.provider in ['mmg', 'firetext'] + + +@freeze_time('2018-01-15T03:30:00') +def test_create_nightly_billing_consolidate_from_3_days_delta( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker): + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + # create records from 11th to 15th + for i in range(0, 5): + sample_notification( + notify_db, + notify_db_session, + created_at=datetime.now() - timedelta(days=i), + service=sample_service, + template=sample_template, + status='delivered', + sent_by=None, + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + + notification = Notification.query.order_by(Notification.created_at).all() + assert datetime.date(notification[0].created_at) == date(2018, 1, 11) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing() + records = FactBilling.query.order_by(FactBilling.bst_date).all() + + assert len(records) == 4 + assert records[0].bst_date == date(2018, 1, 12) + assert records[-1].bst_date == date(2018, 1, 15) + + +def test_get_rate_for_letter_latest(notify_db, notify_db_session): + letter_rate = LetterRate(start_date=datetime(2017, 12, 1), + rate=Decimal(0.33), + crown=True, + sheet_count=1, + post_class='second') + + dao_create_letter_rate(letter_rate) + letter_rate = LetterRate(start_date=datetime(2016, 12, 1), + end_date=datetime(2017, 12, 1), + rate=Decimal(0.30), + crown=True, + sheet_count=1, + post_class='second') + dao_create_letter_rate(letter_rate) + + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + rate = get_rate(non_letter_rates, letter_rates, LETTER_TYPE, datetime(2018, 1, 1), True, 1) + assert rate == Decimal(0.33) + + +def test_get_rate_for_sms_and_email(notify_db, notify_db_session): + letter_rate = LetterRate(start_date=datetime(2017, 12, 1), + rate=Decimal(0.33), + crown=True, + sheet_count=1, + post_class='second') + dao_create_letter_rate(letter_rate) + sms_rate = Rate(valid_from=datetime(2017, 12, 1), + rate=Decimal(0.15), + notification_type=SMS_TYPE) + db.session.add(sms_rate) + email_rate = Rate(valid_from=datetime(2017, 12, 1), + rate=Decimal(0), + notification_type=EMAIL_TYPE) + db.session.add(email_rate) + + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + rate = get_rate(non_letter_rates, letter_rates, SMS_TYPE, datetime(2018, 1, 1)) + assert rate == Decimal(0.15) + + rate = get_rate(non_letter_rates, letter_rates, EMAIL_TYPE, datetime(2018, 1, 1)) + assert rate == Decimal(0) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 1d52c0db6..9dad50c18 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -533,6 +533,7 @@ def sample_notification( api_key=None, key_type=KEY_TYPE_NORMAL, sent_by=None, + international=False, client_reference=None, rate_multiplier=1.0, scheduled_for=None,