From ea70c6454a90a8784de4c6790fd226acfd94eac3 Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 14 Mar 2018 14:47:30 +0000 Subject: [PATCH 1/3] Fine-tuning DB model and create taks for data migration - Removed unused columns in ft_billing - Create tasks for nightly data migration --- app/celery/reporting_tasks.py | 99 +++++++++++++++++++ app/models.py | 13 +-- .../versions/0178_billing_primary_const.py | 36 +++++++ 3 files changed, 140 insertions(+), 8 deletions(-) create mode 100644 app/celery/reporting_tasks.py create mode 100644 migrations/versions/0178_billing_primary_const.py diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py new file mode 100644 index 000000000..6934076ff --- /dev/null +++ b/app/celery/reporting_tasks.py @@ -0,0 +1,99 @@ +from app import notify_celery +from notifications_utils.statsd_decorators import statsd +import random +from app.models import (Notification, + Rate, + NOTIFICATION_CREATED, + NOTIFICATION_TECHNICAL_FAILURE, + KEY_TYPE_TEST, + LetterRate, + FactBilling, + Service, + LETTER_TYPE, SMS_TYPE) +from app import db +from sqlalchemy import func, desc, case +from app.dao.dao_utils import transactional + + +def get_rate(notification_type, date, crown=None, rate_multiplier=None): + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + if notification_type == LETTER_TYPE: + return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2]) + elif notification_type == SMS_TYPE: + return next(r[2] for r in non_letter_rates if notification_type == r[0] and date > r[1]) + else: + return 0 + + +@notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300) +@statsd(namespace="tasks") +@transactional +def create_nightly_billing(self, day_start): + + transit_data = db.session.query( + func.date_trunc('day', Notification.created_at).label('day_created'), + Notification.template_id, + Notification.service_id, + Notification.notification_type, + case( + [ + (Notification.notification_type == 'letter', func.coalesce(Notification.sent_by, 'dvla')), + (Notification.notification_type == 'sms', + func.coalesce(Notification.sent_by, random.choice(['mmg', 'firetext']))) + ], + else_='ses' + ).label('sent_by'), # This could be null - this is a bug to be fixed. + func.coalesce(Notification.rate_multiplier, 1).label('rate_multiplier'), + func.coalesce(Notification.international, False).label('international'), + func.sum(Notification.billable_units).label('billable_units'), + func.count().label('notifications_sent'), + Service.crown, + ).filter( + Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available + Notification.status != NOTIFICATION_TECHNICAL_FAILURE, + Notification.key_type != KEY_TYPE_TEST, + Notification.created_at >= '2018-01-01;' + ).group_by( + 'day_created', + Notification.template_id, + Notification.service_id, + Notification.notification_type, + 'sent_by', + Notification.rate_multiplier, + Notification.international, + Service.crown + ).join( + Service + ).order_by( + 'day_created' + ).all() + + for data in transit_data: + update_count = FactBilling.query.filter( + FactBilling.bst_date == data.day_created, + FactBilling.template_id == data.template_id, + FactBilling.provider == data.sent_by, # This could be zero - this is a bug that needs to be fixed. + FactBilling.rate_multiplier == data.rate_multiplier, + FactBilling.international == data.international, + ).update( + {"notifications_sent": data.notifications_sent, + "billable_units": data.billable_units}, + synchronize_session=False) + if update_count == 0: + billing_record = FactBilling( + bst_date=data.day_created, + template_id=data.template_id, + service_id=data.service_id, + notification_type=data.notification_type, + provider=data.sent_by, + rate_multiplier=data.rate_multiplier, + international=data.international, + billable_units=data.billable_units, + notifications_sent=data.notifications_sent, + rate=get_rate(data.notification_type, data.day_created, data.crown, data.rate_multiplier) + ) + db.session.add(billing_record) diff --git a/app/models.py b/app/models.py index e78fd18ea..ad25775c2 100644 --- a/app/models.py +++ b/app/models.py @@ -1736,15 +1736,12 @@ class FactBilling(db.Model): __tablename__ = "ft_billing" bst_date = db.Column(db.Date, nullable=False, primary_key=True, index=True) - template_id = db.Column(UUID(as_uuid=True), nullable=True, primary_key=True, index=True) - service_id = db.Column(UUID(as_uuid=True), nullable=True, index=True) - organisation_id = db.Column(UUID(as_uuid=True), nullable=True) - annual_billing_id = db.Column(UUID(as_uuid=True), nullable=True) + template_id = db.Column(UUID(as_uuid=True), nullable=False, primary_key=True, index=True) + service_id = db.Column(UUID(as_uuid=True), nullable=False, index=True) notification_type = db.Column(db.Text, nullable=True) - provider = db.Column(db.Text, nullable=True) - crown = db.Column(db.Text, nullable=True) - rate_multiplier = db.Column(db.Numeric(), nullable=True) - international = db.Column(db.Boolean, nullable=True) + provider = db.Column(db.Text, nullable=True, primary_key=True) + rate_multiplier = db.Column(db.Numeric(), nullable=True, primary_key=True) + international = db.Column(db.Boolean, nullable=True, primary_key=True) rate = db.Column(db.Numeric(), nullable=True) billable_units = db.Column(db.Numeric(), nullable=True) notifications_sent = db.Column(db.Integer(), nullable=True) diff --git a/migrations/versions/0178_billing_primary_const.py b/migrations/versions/0178_billing_primary_const.py new file mode 100644 index 000000000..5c189a4cb --- /dev/null +++ b/migrations/versions/0178_billing_primary_const.py @@ -0,0 +1,36 @@ +""" + +Revision ID: 24f47fae3660 +Revises: 0178_billing_primary_const +Create Date: 2018-03-13 14:52:40.413474 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +revision = '0178_billing_primary_const' +down_revision = '0177_add_virus_scan_statuses' + + +def upgrade(): + op.drop_column('ft_billing', 'crown') + op.drop_column('ft_billing', 'annual_billing_id') + op.drop_column('ft_billing', 'organisation_id') + op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary') + # These are the orthogonal dimensions that define a row (except international). + # These entries define a unique record. + op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date', + 'template_id', + 'rate_multiplier', + 'provider', + 'international']) + + +def downgrade(): + op.add_column('ft_billing', sa.Column('organisation_id', postgresql.UUID(), autoincrement=False, nullable=True)) + op.add_column('ft_billing', sa.Column('annual_billing_id', postgresql.UUID(), autoincrement=False, nullable=True)) + op.add_column('ft_billing', sa.Column('crown', sa.TEXT(), autoincrement=False, nullable=True)) + op.drop_constraint('ft_billing_pkey', 'ft_billing', type_='primary') + op.create_primary_key('ft_billing_pkey', 'ft_billing', ['bst_date', + 'template_id']) From 5d7d4a0dd6d1a6ab0759b0ea7fff22a4a721614b Mon Sep 17 00:00:00 2001 From: venusbb Date: Wed, 14 Mar 2018 17:07:52 +0000 Subject: [PATCH 2/3] Added test for reporting tasks --- tests/app/celery/test_reporting_tasks.py | 312 +++++++++++++++++++++++ tests/app/conftest.py | 1 + 2 files changed, 313 insertions(+) create mode 100644 tests/app/celery/test_reporting_tasks.py diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py new file mode 100644 index 000000000..bea58f9b4 --- /dev/null +++ b/tests/app/celery/test_reporting_tasks.py @@ -0,0 +1,312 @@ +from datetime import datetime, timedelta +from tests.app.conftest import sample_notification +from app.celery.reporting_tasks import create_nightly_billing, get_rate +from app.models import FactBilling +from decimal import Decimal +import pytest +from app.dao.letter_rate_dao import dao_create_letter_rate +from app.models import LetterRate, Rate +from app import db + + +def test_reporting_should_have_decorated_tasks_functions(): + assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing' + + +def mocker_get_rate(notification_type, date, crown=None, rate_multiplier=None): + if notification_type == 'letter': + return Decimal(2.1) + elif notification_type == 'sms': + return Decimal(1.33) + elif notification_type == 'email': + return Decimal(0) + + +# Test when notifications with all dimensions are the same, and when the rate is different +@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier', + [(1.0, 1, 2, [1]), + (2.0, 2, 1, [1, 2])]) +def test_create_nightly_billing_sms_rate_multiplier( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker, + second_rate, + records_num, + billable_units, + multiplier): + + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + # two_days_ago = datetime.now() - timedelta(days=6) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=second_rate, + billable_units=1, + ) + # before running migration no records + records = FactBilling.query.all() + assert len(records) == 0 + # before record data after migration + create_nightly_billing(yesterday) + records = FactBilling.query.all() + assert len(records) == records_num + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == billable_units + assert record.rate_multiplier == multiplier[i] + + +def test_create_nightly_billing_different_templates( + notify_db, + notify_db_session, + sample_service, + sample_template, + sample_email_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + # two_days_ago = datetime.now() - timedelta(days=6) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_email_template, + status='delivered', + sent_by='ses', + international=False, + rate_multiplier=0, + billable_units=0, + ) + # before running migration no records + records = FactBilling.query.all() + assert len(records) == 0 + # before record data after migration + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + # First record is ses and second record is sms + assert len(records) == 2 + multiplier = [0, 1] + billable_units = [0, 1] + rate = [0, Decimal(1.33)] + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == rate[i] + assert record.billable_units == billable_units[i] + assert record.rate_multiplier == multiplier[i] + + +def test_create_nightly_billing_different_sent_by( + notify_db, + notify_db_session, + sample_service, + sample_template, + sample_email_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + # two_days_ago = datetime.now() - timedelta(days=6) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='mmg', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by='firetext', + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + # before running migration no records + records = FactBilling.query.all() + assert len(records) == 0 + # before record data after migration + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + # First record is ses and second record is sms + assert len(records) == 2 + for i, record in enumerate(records): + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == 1 + assert record.rate_multiplier == 1.0 + + +def test_create_nightly_billing_letter( + notify_db, + notify_db_session, + sample_service, + sample_letter_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + # two_days_ago = datetime.now() - timedelta(days=6) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_letter_template, + status='delivered', + sent_by='dvla', + international=False, + rate_multiplier=2.0, + billable_units=2, + ) + + # before running migration no records + records = FactBilling.query.all() + assert len(records) == 0 + # before record data after migration + create_nightly_billing(yesterday) + records = FactBilling.query.order_by('rate_multiplier').all() + assert len(records) == 1 + record = records[0] + assert record.notification_type == 'letter' + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(2.1) + assert record.billable_units == 2 + assert record.rate_multiplier == 2.0 + + +def test_create_nightly_billing_null_sent_by_sms( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker): + yesterday = datetime.now() - timedelta(days=1) + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + # two_days_ago = datetime.now() - timedelta(days=6) + + # These are sms notifications + sample_notification( + notify_db, + notify_db_session, + created_at=yesterday, + service=sample_service, + template=sample_template, + status='delivered', + sent_by=None, + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + + # before running migration no records + records = FactBilling.query.all() + assert len(records) == 0 + # before record data after migration + create_nightly_billing(yesterday) + records = FactBilling.query.all() + # First record is ses and second record is sms + assert len(records) == 1 + record = records[0] + assert record.bst_date == datetime.date(yesterday) + assert record.rate == Decimal(1.33) + assert record.billable_units == 1 + assert record.rate_multiplier == 1 + assert record.provider in ['mmg', 'firetext'] + + +def test_get_rate_for_letter_latest(notify_db, notify_db_session): + letter_rate = LetterRate(start_date=datetime(2017, 12, 1), + rate=Decimal(0.33), + crown=True, + sheet_count=1, + post_class='second') + + dao_create_letter_rate(letter_rate) + letter_rate = LetterRate(start_date=datetime(2016, 12, 1), + end_date=datetime(2017, 12, 1), + rate=Decimal(0.30), + crown=True, + sheet_count=1, + post_class='second') + dao_create_letter_rate(letter_rate) + + rate = get_rate('letter', datetime(2018, 1, 1), True, 1) + assert rate == Decimal(0.33) + + +def test_get_rate_for_sms_and_email(notify_db, notify_db_session): + letter_rate = LetterRate(start_date=datetime(2017, 12, 1), + rate=Decimal(0.33), + crown=True, + sheet_count=1, + post_class='second') + dao_create_letter_rate(letter_rate) + sms_rate = Rate(valid_from=datetime(2017, 12, 1), + rate=Decimal(0.15), + notification_type='sms') + db.session.add(sms_rate) + email_rate = Rate(valid_from=datetime(2017, 12, 1), + rate=Decimal(0), + notification_type='email') + db.session.add(email_rate) + + rate = get_rate('sms', datetime(2018, 1, 1)) + assert rate == Decimal(0.15) + + rate = get_rate('email', datetime(2018, 1, 1)) + assert rate == Decimal(0) diff --git a/tests/app/conftest.py b/tests/app/conftest.py index 1d52c0db6..9dad50c18 100644 --- a/tests/app/conftest.py +++ b/tests/app/conftest.py @@ -533,6 +533,7 @@ def sample_notification( api_key=None, key_type=KEY_TYPE_NORMAL, sent_by=None, + international=False, client_reference=None, rate_multiplier=1.0, scheduled_for=None, From bb95a2784f668e032f07157ca4dc159e561e5836 Mon Sep 17 00:00:00 2001 From: venusbb Date: Fri, 16 Mar 2018 09:22:34 +0000 Subject: [PATCH 3/3] Create schedueled job, fixed tests --- app/celery/reporting_tasks.py | 26 ++++-- app/config.py | 5 + tests/app/celery/test_reporting_tasks.py | 112 ++++++++++++++++------- 3 files changed, 101 insertions(+), 42 deletions(-) diff --git a/app/celery/reporting_tasks.py b/app/celery/reporting_tasks.py index 6934076ff..37746742b 100644 --- a/app/celery/reporting_tasks.py +++ b/app/celery/reporting_tasks.py @@ -1,6 +1,7 @@ from app import notify_celery from notifications_utils.statsd_decorators import statsd import random +from datetime import datetime, timedelta from app.models import (Notification, Rate, NOTIFICATION_CREATED, @@ -15,11 +16,7 @@ from sqlalchemy import func, desc, case from app.dao.dao_utils import transactional -def get_rate(notification_type, date, crown=None, rate_multiplier=None): - non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in - Rate.query.order_by(desc(Rate.valid_from)).all()] - letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in - LetterRate.query.order_by(desc(LetterRate.start_date)).all()] +def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): if notification_type == LETTER_TYPE: return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2]) @@ -32,7 +29,15 @@ def get_rate(notification_type, date, crown=None, rate_multiplier=None): @notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300) @statsd(namespace="tasks") @transactional -def create_nightly_billing(self, day_start): +def create_nightly_billing(self, day_start=None): + if day_start is None: + day_start = datetime.date(datetime.utcnow()) - timedelta(days=3) # Nightly jobs consolidating last 3 days + # Task to be run after mid-night + + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] transit_data = db.session.query( func.date_trunc('day', Notification.created_at).label('day_created'), @@ -56,7 +61,7 @@ def create_nightly_billing(self, day_start): Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available Notification.status != NOTIFICATION_TECHNICAL_FAILURE, Notification.key_type != KEY_TYPE_TEST, - Notification.created_at >= '2018-01-01;' + Notification.created_at >= day_start ).group_by( 'day_created', Notification.template_id, @@ -94,6 +99,11 @@ def create_nightly_billing(self, day_start): international=data.international, billable_units=data.billable_units, notifications_sent=data.notifications_sent, - rate=get_rate(data.notification_type, data.day_created, data.crown, data.rate_multiplier) + rate=get_rate(non_letter_rates, + letter_rates, + data.notification_type, + data.day_created, + data.crown, + data.rate_multiplier) ) db.session.add(billing_record) diff --git a/app/config.py b/app/config.py index b1cdd93c2..f57225856 100644 --- a/app/config.py +++ b/app/config.py @@ -213,6 +213,11 @@ class Config(object): 'schedule': crontab(hour=3, minute=0), 'options': {'queue': QueueNames.PERIODIC} }, + 'create-nightly-billing': { + 'task': 'create-nightly-billing', + 'schedule': crontab(hour=3, minute=30), + 'options': {'queue': QueueNames.PERIODIC} + }, 'remove_sms_email_jobs': { 'task': 'remove_csv_files', 'schedule': crontab(hour=4, minute=0), diff --git a/tests/app/celery/test_reporting_tasks.py b/tests/app/celery/test_reporting_tasks.py index bea58f9b4..3b3fcdacb 100644 --- a/tests/app/celery/test_reporting_tasks.py +++ b/tests/app/celery/test_reporting_tasks.py @@ -1,28 +1,33 @@ -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date from tests.app.conftest import sample_notification from app.celery.reporting_tasks import create_nightly_billing, get_rate -from app.models import FactBilling +from app.models import (FactBilling, + Notification, + LETTER_TYPE, + EMAIL_TYPE, + SMS_TYPE) from decimal import Decimal import pytest from app.dao.letter_rate_dao import dao_create_letter_rate from app.models import LetterRate, Rate from app import db +from freezegun import freeze_time +from sqlalchemy import desc def test_reporting_should_have_decorated_tasks_functions(): assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing' -def mocker_get_rate(notification_type, date, crown=None, rate_multiplier=None): - if notification_type == 'letter': +def mocker_get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None): + if notification_type == LETTER_TYPE: return Decimal(2.1) - elif notification_type == 'sms': + elif notification_type == SMS_TYPE: return Decimal(1.33) - elif notification_type == 'email': + elif notification_type == EMAIL_TYPE: return Decimal(0) -# Test when notifications with all dimensions are the same, and when the rate is different @pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier', [(1.0, 1, 2, [1]), (2.0, 2, 1, [1, 2])]) @@ -40,7 +45,6 @@ def test_create_nightly_billing_sms_rate_multiplier( yesterday = datetime.now() - timedelta(days=1) mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) - # two_days_ago = datetime.now() - timedelta(days=6) # These are sms notifications sample_notification( @@ -67,10 +71,10 @@ def test_create_nightly_billing_sms_rate_multiplier( rate_multiplier=second_rate, billable_units=1, ) - # before running migration no records + records = FactBilling.query.all() assert len(records) == 0 - # before record data after migration + create_nightly_billing(yesterday) records = FactBilling.query.all() assert len(records) == records_num @@ -91,9 +95,7 @@ def test_create_nightly_billing_different_templates( yesterday = datetime.now() - timedelta(days=1) mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) - # two_days_ago = datetime.now() - timedelta(days=6) - # These are sms notifications sample_notification( notify_db, notify_db_session, @@ -118,13 +120,13 @@ def test_create_nightly_billing_different_templates( rate_multiplier=0, billable_units=0, ) - # before running migration no records + records = FactBilling.query.all() assert len(records) == 0 - # before record data after migration + create_nightly_billing(yesterday) records = FactBilling.query.order_by('rate_multiplier').all() - # First record is ses and second record is sms + assert len(records) == 2 multiplier = [0, 1] billable_units = [0, 1] @@ -146,7 +148,6 @@ def test_create_nightly_billing_different_sent_by( yesterday = datetime.now() - timedelta(days=1) mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) - # two_days_ago = datetime.now() - timedelta(days=6) # These are sms notifications sample_notification( @@ -173,13 +174,13 @@ def test_create_nightly_billing_different_sent_by( rate_multiplier=1.0, billable_units=1, ) - # before running migration no records + records = FactBilling.query.all() assert len(records) == 0 - # before record data after migration + create_nightly_billing(yesterday) records = FactBilling.query.order_by('rate_multiplier').all() - # First record is ses and second record is sms + assert len(records) == 2 for i, record in enumerate(records): assert record.bst_date == datetime.date(yesterday) @@ -197,9 +198,7 @@ def test_create_nightly_billing_letter( yesterday = datetime.now() - timedelta(days=1) mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) - # two_days_ago = datetime.now() - timedelta(days=6) - # These are sms notifications sample_notification( notify_db, notify_db_session, @@ -213,15 +212,14 @@ def test_create_nightly_billing_letter( billable_units=2, ) - # before running migration no records records = FactBilling.query.all() assert len(records) == 0 - # before record data after migration + create_nightly_billing(yesterday) records = FactBilling.query.order_by('rate_multiplier').all() assert len(records) == 1 record = records[0] - assert record.notification_type == 'letter' + assert record.notification_type == LETTER_TYPE assert record.bst_date == datetime.date(yesterday) assert record.rate == Decimal(2.1) assert record.billable_units == 2 @@ -237,9 +235,7 @@ def test_create_nightly_billing_null_sent_by_sms( yesterday = datetime.now() - timedelta(days=1) mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) - # two_days_ago = datetime.now() - timedelta(days=6) - # These are sms notifications sample_notification( notify_db, notify_db_session, @@ -253,13 +249,12 @@ def test_create_nightly_billing_null_sent_by_sms( billable_units=1, ) - # before running migration no records records = FactBilling.query.all() assert len(records) == 0 - # before record data after migration + create_nightly_billing(yesterday) records = FactBilling.query.all() - # First record is ses and second record is sms + assert len(records) == 1 record = records[0] assert record.bst_date == datetime.date(yesterday) @@ -269,6 +264,45 @@ def test_create_nightly_billing_null_sent_by_sms( assert record.provider in ['mmg', 'firetext'] +@freeze_time('2018-01-15T03:30:00') +def test_create_nightly_billing_consolidate_from_3_days_delta( + notify_db, + notify_db_session, + sample_service, + sample_template, + mocker): + + mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate) + + # create records from 11th to 15th + for i in range(0, 5): + sample_notification( + notify_db, + notify_db_session, + created_at=datetime.now() - timedelta(days=i), + service=sample_service, + template=sample_template, + status='delivered', + sent_by=None, + international=False, + rate_multiplier=1.0, + billable_units=1, + ) + + notification = Notification.query.order_by(Notification.created_at).all() + assert datetime.date(notification[0].created_at) == date(2018, 1, 11) + + records = FactBilling.query.all() + assert len(records) == 0 + + create_nightly_billing() + records = FactBilling.query.order_by(FactBilling.bst_date).all() + + assert len(records) == 4 + assert records[0].bst_date == date(2018, 1, 12) + assert records[-1].bst_date == date(2018, 1, 15) + + def test_get_rate_for_letter_latest(notify_db, notify_db_session): letter_rate = LetterRate(start_date=datetime(2017, 12, 1), rate=Decimal(0.33), @@ -285,7 +319,12 @@ def test_get_rate_for_letter_latest(notify_db, notify_db_session): post_class='second') dao_create_letter_rate(letter_rate) - rate = get_rate('letter', datetime(2018, 1, 1), True, 1) + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + rate = get_rate(non_letter_rates, letter_rates, LETTER_TYPE, datetime(2018, 1, 1), True, 1) assert rate == Decimal(0.33) @@ -298,15 +337,20 @@ def test_get_rate_for_sms_and_email(notify_db, notify_db_session): dao_create_letter_rate(letter_rate) sms_rate = Rate(valid_from=datetime(2017, 12, 1), rate=Decimal(0.15), - notification_type='sms') + notification_type=SMS_TYPE) db.session.add(sms_rate) email_rate = Rate(valid_from=datetime(2017, 12, 1), rate=Decimal(0), - notification_type='email') + notification_type=EMAIL_TYPE) db.session.add(email_rate) - rate = get_rate('sms', datetime(2018, 1, 1)) + non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in + Rate.query.order_by(desc(Rate.valid_from)).all()] + letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in + LetterRate.query.order_by(desc(LetterRate.start_date)).all()] + + rate = get_rate(non_letter_rates, letter_rates, SMS_TYPE, datetime(2018, 1, 1)) assert rate == Decimal(0.15) - rate = get_rate('email', datetime(2018, 1, 1)) + rate = get_rate(non_letter_rates, letter_rates, EMAIL_TYPE, datetime(2018, 1, 1)) assert rate == Decimal(0)