darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-14 09:12:06 -05:00
Code Issues Packages Projects Releases Wiki Activity

Create schedueled job, fixed tests

Browse Source
This commit is contained in:
venusbb
2018-03-16 09:22:34 +00:00
parent 5d7d4a0dd6
commit bb95a2784f
3 changed files with 101 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
app/celery/reporting_tasks.py
View File

@@ -1,6 +1,7 @@
from app import notify_celery
from notifications_utils.statsd_decorators import statsd
import random
from datetime import datetime, timedelta
from app.models import (Notification,
Rate,
NOTIFICATION_CREATED,
@@ -15,11 +16,7 @@ from sqlalchemy import func, desc, case
from app.dao.dao_utils import transactional
def get_rate(notification_type, date, crown=None, rate_multiplier=None):
non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
Rate.query.order_by(desc(Rate.valid_from)).all()]
letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
def get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
if notification_type == LETTER_TYPE:
return next(r[3] for r in letter_rates if date > r[0] and crown == r[1] and rate_multiplier == r[2])
@@ -32,7 +29,15 @@ def get_rate(notification_type, date, crown=None, rate_multiplier=None):
@notify_celery.task(bind=True, name="create-nightly-billing", max_retries=15, default_retry_delay=300)
@statsd(namespace="tasks")
@transactional
def create_nightly_billing(self, day_start):
def create_nightly_billing(self, day_start=None):
if day_start is None:
day_start = datetime.date(datetime.utcnow()) - timedelta(days=3) # Nightly jobs consolidating last 3 days
# Task to be run after mid-night
non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
Rate.query.order_by(desc(Rate.valid_from)).all()]
letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
transit_data = db.session.query(
func.date_trunc('day', Notification.created_at).label('day_created'),
@@ -56,7 +61,7 @@ def create_nightly_billing(self, day_start):
Notification.status != NOTIFICATION_CREATED, # at created status, provider information is not available
Notification.status != NOTIFICATION_TECHNICAL_FAILURE,
Notification.key_type != KEY_TYPE_TEST,
Notification.created_at >= '2018-01-01;'
Notification.created_at >= day_start
).group_by(
'day_created',
Notification.template_id,
@@ -94,6 +99,11 @@ def create_nightly_billing(self, day_start):
international=data.international,
billable_units=data.billable_units,
notifications_sent=data.notifications_sent,
rate=get_rate(data.notification_type, data.day_created, data.crown, data.rate_multiplier)
rate=get_rate(non_letter_rates,
letter_rates,
data.notification_type,
data.day_created,
data.crown,
data.rate_multiplier)
)
db.session.add(billing_record)

5
app/config.py
View File

@@ -213,6 +213,11 @@ class Config(object):
'schedule': crontab(hour=3, minute=0),
'options': {'queue': QueueNames.PERIODIC}
},
'create-nightly-billing': {
'task': 'create-nightly-billing',
'schedule': crontab(hour=3, minute=30),
'options': {'queue': QueueNames.PERIODIC}
},
'remove_sms_email_jobs': {
'task': 'remove_csv_files',
'schedule': crontab(hour=4, minute=0),

112
tests/app/celery/test_reporting_tasks.py
View File

@@ -1,28 +1,33 @@
from datetime import datetime, timedelta
from datetime import datetime, timedelta, date
from tests.app.conftest import sample_notification
from app.celery.reporting_tasks import create_nightly_billing, get_rate
from app.models import FactBilling
from app.models import (FactBilling,
Notification,
LETTER_TYPE,
EMAIL_TYPE,
SMS_TYPE)
from decimal import Decimal
import pytest
from app.dao.letter_rate_dao import dao_create_letter_rate
from app.models import LetterRate, Rate
from app import db
from freezegun import freeze_time
from sqlalchemy import desc
def test_reporting_should_have_decorated_tasks_functions():
assert create_nightly_billing.__wrapped__.__name__ == 'create_nightly_billing'
def mocker_get_rate(notification_type, date, crown=None, rate_multiplier=None):
if notification_type == 'letter':
def mocker_get_rate(non_letter_rates, letter_rates, notification_type, date, crown=None, rate_multiplier=None):
if notification_type == LETTER_TYPE:
return Decimal(2.1)
elif notification_type == 'sms':
elif notification_type == SMS_TYPE:
return Decimal(1.33)
elif notification_type == 'email':
elif notification_type == EMAIL_TYPE:
return Decimal(0)
# Test when notifications with all dimensions are the same, and when the rate is different
@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
[(1.0, 1, 2, [1]),
(2.0, 2, 1, [1, 2])])
@@ -40,7 +45,6 @@ def test_create_nightly_billing_sms_rate_multiplier(
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# two_days_ago = datetime.now() - timedelta(days=6)
# These are sms notifications
sample_notification(
@@ -67,10 +71,10 @@ def test_create_nightly_billing_sms_rate_multiplier(
rate_multiplier=second_rate,
billable_units=1,
)
# before running migration no records
records = FactBilling.query.all()
assert len(records) == 0
# before record data after migration
create_nightly_billing(yesterday)
records = FactBilling.query.all()
assert len(records) == records_num
@@ -91,9 +95,7 @@ def test_create_nightly_billing_different_templates(
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# two_days_ago = datetime.now() - timedelta(days=6)
# These are sms notifications
sample_notification(
notify_db,
notify_db_session,
@@ -118,13 +120,13 @@ def test_create_nightly_billing_different_templates(
rate_multiplier=0,
billable_units=0,
)
# before running migration no records
records = FactBilling.query.all()
assert len(records) == 0
# before record data after migration
create_nightly_billing(yesterday)
records = FactBilling.query.order_by('rate_multiplier').all()
# First record is ses and second record is sms
assert len(records) == 2
multiplier = [0, 1]
billable_units = [0, 1]
@@ -146,7 +148,6 @@ def test_create_nightly_billing_different_sent_by(
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# two_days_ago = datetime.now() - timedelta(days=6)
# These are sms notifications
sample_notification(
@@ -173,13 +174,13 @@ def test_create_nightly_billing_different_sent_by(
rate_multiplier=1.0,
billable_units=1,
)
# before running migration no records
records = FactBilling.query.all()
assert len(records) == 0
# before record data after migration
create_nightly_billing(yesterday)
records = FactBilling.query.order_by('rate_multiplier').all()
# First record is ses and second record is sms
assert len(records) == 2
for i, record in enumerate(records):
assert record.bst_date == datetime.date(yesterday)
@@ -197,9 +198,7 @@ def test_create_nightly_billing_letter(
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# two_days_ago = datetime.now() - timedelta(days=6)
# These are sms notifications
sample_notification(
notify_db,
notify_db_session,
@@ -213,15 +212,14 @@ def test_create_nightly_billing_letter(
billable_units=2,
)
# before running migration no records
records = FactBilling.query.all()
assert len(records) == 0
# before record data after migration
create_nightly_billing(yesterday)
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == 1
record = records[0]
assert record.notification_type == 'letter'
assert record.notification_type == LETTER_TYPE
assert record.bst_date == datetime.date(yesterday)
assert record.rate == Decimal(2.1)
assert record.billable_units == 2
@@ -237,9 +235,7 @@ def test_create_nightly_billing_null_sent_by_sms(
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# two_days_ago = datetime.now() - timedelta(days=6)
# These are sms notifications
sample_notification(
notify_db,
notify_db_session,
@@ -253,13 +249,12 @@ def test_create_nightly_billing_null_sent_by_sms(
billable_units=1,
)
# before running migration no records
records = FactBilling.query.all()
assert len(records) == 0
# before record data after migration
create_nightly_billing(yesterday)
records = FactBilling.query.all()
# First record is ses and second record is sms
assert len(records) == 1
record = records[0]
assert record.bst_date == datetime.date(yesterday)
@@ -269,6 +264,45 @@ def test_create_nightly_billing_null_sent_by_sms(
assert record.provider in ['mmg', 'firetext']
@freeze_time('2018-01-15T03:30:00')
def test_create_nightly_billing_consolidate_from_3_days_delta(
notify_db,
notify_db_session,
sample_service,
sample_template,
mocker):
mocker.patch('app.celery.reporting_tasks.get_rate', side_effect=mocker_get_rate)
# create records from 11th to 15th
for i in range(0, 5):
sample_notification(
notify_db,
notify_db_session,
created_at=datetime.now() - timedelta(days=i),
service=sample_service,
template=sample_template,
status='delivered',
sent_by=None,
international=False,
rate_multiplier=1.0,
billable_units=1,
)
notification = Notification.query.order_by(Notification.created_at).all()
assert datetime.date(notification[0].created_at) == date(2018, 1, 11)
records = FactBilling.query.all()
assert len(records) == 0
create_nightly_billing()
records = FactBilling.query.order_by(FactBilling.bst_date).all()
assert len(records) == 4
assert records[0].bst_date == date(2018, 1, 12)
assert records[-1].bst_date == date(2018, 1, 15)
def test_get_rate_for_letter_latest(notify_db, notify_db_session):
letter_rate = LetterRate(start_date=datetime(2017, 12, 1),
rate=Decimal(0.33),
@@ -285,7 +319,12 @@ def test_get_rate_for_letter_latest(notify_db, notify_db_session):
post_class='second')
dao_create_letter_rate(letter_rate)
rate = get_rate('letter', datetime(2018, 1, 1), True, 1)
non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
Rate.query.order_by(desc(Rate.valid_from)).all()]
letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
rate = get_rate(non_letter_rates, letter_rates, LETTER_TYPE, datetime(2018, 1, 1), True, 1)
assert rate == Decimal(0.33)
@@ -298,15 +337,20 @@ def test_get_rate_for_sms_and_email(notify_db, notify_db_session):
dao_create_letter_rate(letter_rate)
sms_rate = Rate(valid_from=datetime(2017, 12, 1),
rate=Decimal(0.15),
notification_type='sms')
notification_type=SMS_TYPE)
db.session.add(sms_rate)
email_rate = Rate(valid_from=datetime(2017, 12, 1),
rate=Decimal(0),
notification_type='email')
notification_type=EMAIL_TYPE)
db.session.add(email_rate)
rate = get_rate('sms', datetime(2018, 1, 1))
non_letter_rates = [(r.notification_type, r.valid_from, r.rate) for r in
Rate.query.order_by(desc(Rate.valid_from)).all()]
letter_rates = [(r.start_date, r.crown, r.sheet_count, r.rate) for r in
LetterRate.query.order_by(desc(LetterRate.start_date)).all()]
rate = get_rate(non_letter_rates, letter_rates, SMS_TYPE, datetime(2018, 1, 1))
assert rate == Decimal(0.15)
rate = get_rate('email', datetime(2018, 1, 1))
rate = get_rate(non_letter_rates, letter_rates, EMAIL_TYPE, datetime(2018, 1, 1))
assert rate == Decimal(0)