darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-23 08:51:30 -05:00
Code Issues Packages Projects Releases Wiki Activity
Files
a69d1635a1fa72bb6dff3f13283f32aaf584ceaf
notifications-api/tests/app/celery/test_reporting_tasks.py
Ben Thorner 1213463b8e Only aggregate status when necessary for a service 
This takes a similar approach to the nightly deletion task so that
we only create sub-tasks when there are actually notifications to
aggregate for a given type and day [1].

We're making this change to stop the duplication errors we're getting
at the moment and ensure the task can scale to more messages and more
services. There are two parts to this:

- Each subtask should now run within the 5 minute visibility timeout.
However, they may still be duplicated if the parent task overruns [2].

- The parent task creates a mininal number of subtasks, and the query
to determine this is very fast for a normal process day (milliseconds).

Since all tasks will run quickly, there should be no more duplication.

In order to test this more nuanced task, I rewrote the tests:

- One test checks the subtask is called correctly.
- One test checks we create all the right subtasks.

[1]: https://github.com/alphagov/notifications-api/pull/3381
[2]: https://docs.google.com/document/d/1MaP6Nyy3nJKkuh_4lP1wuDm19X8LZITOLRd9n3Ax-xg/edit#heading=h.q3intzwqhfzl
2022-02-09 17:39:07 +00:00

644 lines
22 KiB
Python
Raw Blame History

from datetime import date, datetime, time, timedelta
from decimal import Decimal
from uuid import UUID
import pytest
from freezegun import freeze_time
from app.celery.reporting_tasks import (
create_nightly_billing,
create_nightly_billing_for_day,
create_nightly_notification_status,
create_nightly_notification_status_for_service_and_day,
)
from app.config import QueueNames
from app.dao.fact_billing_dao import get_rate
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_TYPES,
SMS_TYPE,
FactBilling,
FactNotificationStatus,
Notification,
)
from tests.app.db import (
create_letter_rate,
create_notification,
create_notification_history,
create_rate,
create_service,
create_service_data_retention,
create_template,
)
def mocker_get_rate(
non_letter_rates, letter_rates, notification_type, bst_date, crown=None, rate_multiplier=None, post_class="second"
):
if notification_type == LETTER_TYPE:
return Decimal(2.1)
elif notification_type == SMS_TYPE:
return Decimal(1.33)
elif notification_type == EMAIL_TYPE:
return Decimal(0)
@freeze_time('2019-08-01')
@pytest.mark.parametrize('day_start, expected_kwargs', [
(None, ['2019-07-31', '2019-07-30', '2019-07-29', '2019-07-28']),
('2019-07-21', ['2019-07-21', '2019-07-20', '2019-07-19', '2019-07-18']),
])
def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_start, expected_kwargs):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_billing_for_day')
create_nightly_billing(day_start)
assert mock_celery.apply_async.call_count == 4
for i in range(4):
assert mock_celery.apply_async.call_args_list[i][1]['kwargs'] == {'process_day': expected_kwargs[i]}
@freeze_time('2019-08-01T00:30')
def test_create_nightly_notification_status_triggers_tasks(
notify_api,
sample_service,
sample_template,
mocker,
):
mock_celery = mocker.patch(
'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
).apply_async
create_notification(template=sample_template, created_at='2019-07-31')
create_nightly_notification_status()
mock_celery.assert_called_with(
kwargs={
'service_id': sample_service.id,
'process_day': '2019-07-31',
'notification_type': SMS_TYPE
},
queue=QueueNames.REPORTING
)
@freeze_time('2019-08-01T00:30')
@pytest.mark.parametrize('notification_date, expected_types_aggregated', [
('2019-08-01', set()),
('2019-07-31', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
('2019-07-28', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
('2019-07-27', {LETTER_TYPE}),
('2019-07-22', {LETTER_TYPE}),
('2019-07-21', set()),
])
def test_create_nightly_notification_status_triggers_relevant_tasks(
notify_api,
sample_service,
mocker,
notification_date,
expected_types_aggregated,
):
mock_celery = mocker.patch(
'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
).apply_async
for notification_type in NOTIFICATION_TYPES:
template = create_template(sample_service, template_type=notification_type)
create_notification(template=template, created_at=notification_date)
create_nightly_notification_status()
types = {call.kwargs['kwargs']['notification_type'] for call in mock_celery.mock_calls}
assert types == expected_types_aggregated
@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
[(1.0, 1, 2, [1]),
(2.0, 2, 1, [1, 2])])
def test_create_nightly_billing_for_day_sms_rate_multiplier(
sample_service,
sample_template,
mocker,
second_rate,
records_num,
billable_units,
multiplier):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
# These are sms notifications
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='mmg',
international=False,
rate_multiplier=1.0,
billable_units=1,
)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='mmg',
international=False,
rate_multiplier=second_rate,
billable_units=1,
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == records_num
for i, record in enumerate(records):
assert record.bst_date == datetime.date(yesterday)
assert record.rate == Decimal(1.33)
assert record.billable_units == billable_units
assert record.rate_multiplier == multiplier[i]
def test_create_nightly_billing_for_day_different_templates(
sample_service,
sample_template,
sample_email_template,
mocker):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='mmg',
international=False,
rate_multiplier=1.0,
billable_units=1,
)
create_notification(
created_at=yesterday,
template=sample_email_template,
status='delivered',
sent_by='ses',
international=False,
rate_multiplier=0,
billable_units=0,
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == 2
multiplier = [0, 1]
billable_units = [0, 1]
rate = [0, Decimal(1.33)]
for i, record in enumerate(records):
assert record.bst_date == datetime.date(yesterday)
assert record.rate == rate[i]
assert record.billable_units == billable_units[i]
assert record.rate_multiplier == multiplier[i]
def test_create_nightly_billing_for_day_different_sent_by(
sample_service,
sample_template,
sample_email_template,
mocker):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
# These are sms notifications
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='mmg',
international=False,
rate_multiplier=1.0,
billable_units=1,
)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by='firetext',
international=False,
rate_multiplier=1.0,
billable_units=1,
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == 2
for _, record in enumerate(records):
assert record.bst_date == datetime.date(yesterday)
assert record.rate == Decimal(1.33)
assert record.billable_units == 1
assert record.rate_multiplier == 1.0
def test_create_nightly_billing_for_day_different_letter_postage(
notify_db_session,
sample_letter_template,
mocker):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
for _ in range(2):
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=2,
postage='first'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=2,
postage='second'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=1,
postage='europe'
)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
billable_units=3,
postage='rest-of-world'
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.order_by('postage').all()
assert len(records) == 4
assert records[0].notification_type == LETTER_TYPE
assert records[0].bst_date == datetime.date(yesterday)
assert records[0].postage == 'europe'
assert records[0].notifications_sent == 1
assert records[0].billable_units == 1
assert records[1].notification_type == LETTER_TYPE
assert records[1].bst_date == datetime.date(yesterday)
assert records[1].postage == 'first'
assert records[1].notifications_sent == 2
assert records[1].billable_units == 4
assert records[2].notification_type == LETTER_TYPE
assert records[2].bst_date == datetime.date(yesterday)
assert records[2].postage == 'rest-of-world'
assert records[2].notifications_sent == 1
assert records[2].billable_units == 3
assert records[3].notification_type == LETTER_TYPE
assert records[3].bst_date == datetime.date(yesterday)
assert records[3].postage == 'second'
assert records[3].notifications_sent == 1
assert records[3].billable_units == 2
def test_create_nightly_billing_for_day_letter(
sample_service,
sample_letter_template,
mocker):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_letter_template,
status='delivered',
sent_by='dvla',
international=False,
rate_multiplier=2.0,
billable_units=2,
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.order_by('rate_multiplier').all()
assert len(records) == 1
record = records[0]
assert record.notification_type == LETTER_TYPE
assert record.bst_date == datetime.date(yesterday)
assert record.rate == Decimal(2.1)
assert record.billable_units == 2
assert record.rate_multiplier == 2.0
def test_create_nightly_billing_for_day_null_sent_by_sms(
sample_service,
sample_template,
mocker):
yesterday = datetime.now() - timedelta(days=1)
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
create_notification(
created_at=yesterday,
template=sample_template,
status='delivered',
sent_by=None,
international=False,
rate_multiplier=1.0,
billable_units=1,
)
records = FactBilling.query.all()
assert len(records) == 0
# Celery expects the arguments to be a string or primitive type.
yesterday_str = datetime.strftime(yesterday, "%Y-%m-%d")
create_nightly_billing_for_day(yesterday_str)
records = FactBilling.query.all()
assert len(records) == 1
record = records[0]
assert record.bst_date == datetime.date(yesterday)
assert record.rate == Decimal(1.33)
assert record.billable_units == 1
assert record.rate_multiplier == 1
assert record.provider == 'unknown'
def test_get_rate_for_letter_latest(notify_db_session):
# letter rates should be passed into the get_rate function as a tuple of start_date, crown, sheet_count,
# rate and post_class
new = create_letter_rate(datetime(2017, 12, 1), crown=True, sheet_count=1, rate=0.33, post_class='second')
old = create_letter_rate(datetime(2016, 12, 1), crown=True, sheet_count=1, rate=0.30, post_class='second')
letter_rates = [new, old]
rate = get_rate([], letter_rates, LETTER_TYPE, date(2018, 1, 1), True, 1)
assert rate == Decimal('0.33')
def test_get_rate_for_letter_latest_if_crown_is_none(notify_db_session):
# letter rates should be passed into the get_rate function as a tuple of start_date, crown, sheet_count,
# rate and post_class
crown = create_letter_rate(datetime(2017, 12, 1), crown=True, sheet_count=1, rate=0.33, post_class='second')
non_crown = create_letter_rate(datetime(2017, 12, 1), crown=False, sheet_count=1, rate=0.35, post_class='second')
letter_rates = [crown, non_crown]
rate = get_rate([], letter_rates, LETTER_TYPE, date(2018, 1, 1), crown=None, letter_page_count=1)
assert rate == Decimal('0.33')
def test_get_rate_for_sms_and_email(notify_db_session):
non_letter_rates = [
create_rate(datetime(2017, 12, 1), 0.15, SMS_TYPE),
create_rate(datetime(2017, 12, 1), 0, EMAIL_TYPE)
]
rate = get_rate(non_letter_rates, [], SMS_TYPE, date(2018, 1, 1))
assert rate == Decimal(0.15)
rate = get_rate(non_letter_rates, [], EMAIL_TYPE, date(2018, 1, 1))
assert rate == Decimal(0)
@freeze_time('2018-03-30T01:00:00')
# summer time starts on 2018-03-25
def test_create_nightly_billing_for_day_use_BST(
sample_service,
sample_template,
mocker):
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
# too late
create_notification(
created_at=datetime(2018, 3, 25, 23, 1),
template=sample_template,
status='delivered',
rate_multiplier=1.0,
billable_units=1,
)
create_notification(
created_at=datetime(2018, 3, 25, 22, 59),
template=sample_template,
status='delivered',
rate_multiplier=1.0,
billable_units=2,
)
# too early
create_notification(
created_at=datetime(2018, 3, 24, 23, 59),
template=sample_template,
status='delivered',
rate_multiplier=1.0,
billable_units=4,
)
assert Notification.query.count() == 3
assert FactBilling.query.count() == 0
create_nightly_billing_for_day('2018-03-25')
records = FactBilling.query.order_by(FactBilling.bst_date).all()
assert len(records) == 1
assert records[0].bst_date == date(2018, 3, 25)
assert records[0].billable_units == 2
@freeze_time('2018-01-15T03:30:00')
def test_create_nightly_billing_for_day_update_when_record_exists(
sample_service,
sample_template,
mocker):
mocker.patch('app.dao.fact_billing_dao.get_rate', side_effect=mocker_get_rate)
create_notification(
created_at=datetime.now() - timedelta(days=1),
template=sample_template,
status='delivered',
sent_by=None,
international=False,
rate_multiplier=1.0,
billable_units=1,
)
records = FactBilling.query.all()
assert len(records) == 0
create_nightly_billing_for_day('2018-01-14')
records = FactBilling.query.order_by(FactBilling.bst_date).all()
assert len(records) == 1
assert records[0].bst_date == date(2018, 1, 14)
assert records[0].billable_units == 1
assert not records[0].updated_at
create_notification(
created_at=datetime.now() - timedelta(days=1),
template=sample_template,
status='delivered',
sent_by=None,
international=False,
rate_multiplier=1.0,
billable_units=1,
)
# run again, make sure create_nightly_billing() updates with no error
create_nightly_billing_for_day('2018-01-14')
assert len(records) == 1
assert records[0].billable_units == 2
assert records[0].updated_at
def test_create_nightly_notification_status_for_service_and_day(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_service_data_retention(second_service, 'email', days_of_retention=3)
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
assert len(FactNotificationStatus.query.all()) == 0
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
create_nightly_notification_status_for_service_and_day(str(process_day), third_service.id, 'letter')
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].key_type == KEY_TYPE_NORMAL
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[1].key_type == KEY_TYPE_TEAM
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
assert new_fact_data[2].key_type == KEY_TYPE_NORMAL
def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
create_notification(template=first_template, status='delivered')
process_day = date.today()
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
create_notification(template=first_template, status='delivered')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
updated_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(updated_fact_data) == 1
assert updated_fact_data[0].notification_count == 2
# the job runs at 12:30am London time. 04/01 is in BST.
@freeze_time('2019-04-01T23:30')
def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template):
create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 1, 23, 0)) # too new
create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 22, 59))
create_notification(sample_template, status='created', created_at=datetime(2019, 3, 31, 23, 0))
create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 31, 22, 59)) # too old
create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms')
noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date).all()
assert len(noti_status) == 1
assert noti_status[0].bst_date == date(2019, 4, 1)
assert noti_status[0].notification_status == 'created'
Reference in New Issue View Git Blame Copy Permalink