darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-24 01:11:38 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3425 from alphagov/parallelise-ft-status-180693991

Browse Source 
Parallelise status aggregation by service and day
This commit is contained in:
Ben Thorner
2022-01-19 13:49:28 +00:00
committed by GitHub
parent 5cd6fcbb4f 9686595fa8
commit 6be489daa7
5 changed files with 164 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
tests/app/celery/test_reporting_tasks.py
View File

@@ -1,6 +1,7 @@
import itertools
from datetime import date, datetime, timedelta
from datetime import date, datetime, time, timedelta
from decimal import Decimal
from uuid import UUID
import pytest
from freezegun import freeze_time
@@ -9,12 +10,15 @@ from app.celery.reporting_tasks import (
create_nightly_billing,
create_nightly_billing_for_day,
create_nightly_notification_status,
create_nightly_notification_status_for_day,
create_nightly_notification_status_for_service_and_day,
)
from app.config import QueueNames
from app.dao.fact_billing_dao import get_rate
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
SMS_TYPE,
FactBilling,
@@ -24,8 +28,10 @@ from app.models import (
from tests.app.db import (
create_letter_rate,
create_notification,
create_notification_history,
create_rate,
create_service,
create_service_data_retention,
create_template,
)
@@ -56,8 +62,8 @@ def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_
@freeze_time('2019-08-01')
def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, mocker):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_day')
def test_create_nightly_notification_status_triggers_tasks(notify_api, sample_service, mocker):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day')
create_nightly_notification_status()
assert mock_celery.apply_async.call_count == (
@@ -71,13 +77,21 @@ def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api,
[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
):
mock_celery.apply_async.assert_any_call(
kwargs={'process_day': process_date, 'notification_type': notification_type},
kwargs={
'process_day': process_date,
'notification_type': notification_type,
'service_id': sample_service.id,
},
queue=QueueNames.REPORTING
)
for process_date in ['2019-07-27', '2019-07-26', '2019-07-25', '2019-07-24', '2019-07-23', '2019-07-22']:
mock_celery.apply_async.assert_any_call(
kwargs={'process_day': process_date, 'notification_type': LETTER_TYPE},
kwargs={
'process_day': process_date,
'notification_type': LETTER_TYPE,
'service_id': sample_service.id,
},
queue=QueueNames.REPORTING
)
@@ -495,8 +509,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
assert records[0].updated_at
@freeze_time('2019-01-05')
def test_create_nightly_notification_status_for_day(notify_db_session):
def test_create_nightly_notification_status_for_service_and_day(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
@@ -504,40 +517,97 @@ def test_create_nightly_notification_status_for_day(notify_db_session):
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_notification(template=first_template, status='delivered')
create_notification(template=first_template, status='delivered', created_at=datetime(2019, 1, 1, 12, 0))
create_service_data_retention(second_service, 'email', days_of_retention=3)
create_notification(template=second_template, status='temporary-failure')
create_notification(template=second_template, status='temporary-failure', created_at=datetime(2019, 1, 1, 12, 0))
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
create_notification(template=third_template, status='created')
create_notification(template=third_template, status='created', created_at=datetime(2019, 1, 1, 12, 0))
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
assert len(FactNotificationStatus.query.all()) == 0
create_nightly_notification_status_for_day('2019-01-01', 'sms')
create_nightly_notification_status_for_day('2019-01-01', 'email')
create_nightly_notification_status_for_day('2019-01-01', 'letter')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
create_nightly_notification_status_for_service_and_day(str(process_day), third_service.id, 'letter')
new_data = FactNotificationStatus.query.order_by(FactNotificationStatus.created_at).all()
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_type
).all()
assert len(new_data) == 3
assert new_data[0].bst_date == date(2019, 1, 1)
assert new_data[1].bst_date == date(2019, 1, 1)
assert new_data[2].bst_date == date(2019, 1, 1)
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].key_type == KEY_TYPE_NORMAL
assert new_data[0].notification_type == 'sms'
assert new_data[1].notification_type == 'email'
assert new_data[2].notification_type == 'letter'
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[1].key_type == KEY_TYPE_TEAM
assert new_data[0].notification_status == 'delivered'
assert new_data[1].notification_status == 'temporary-failure'
assert new_data[2].notification_status == 'created'
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
assert new_fact_data[2].key_type == KEY_TYPE_NORMAL
def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
create_notification(template=first_template, status='delivered')
process_day = date.today()
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
create_notification(template=first_template, status='delivered')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
updated_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(updated_fact_data) == 1
assert updated_fact_data[0].notification_count == 2
# the job runs at 12:30am London time. 04/01 is in BST.
@freeze_time('2019-04-01T23:30')
def test_create_nightly_notification_status_for_day_respects_bst(sample_template):
def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template):
create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 1, 23, 0)) # too new
create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 22, 59))
@@ -545,7 +615,7 @@ def test_create_nightly_notification_status_for_day_respects_bst(sample_template
create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 31, 22, 59)) # too old
create_nightly_notification_status_for_day('2019-04-01', 'sms')
create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms')
noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date).all()
assert len(noti_status) == 1

101
tests/app/dao/test_fact_notification_status_dao.py
View File

@@ -1,4 +1,4 @@
from datetime import date, datetime, time, timedelta
from datetime import date, datetime, timedelta
from unittest import mock
from uuid import UUID
@@ -8,7 +8,6 @@ from freezegun import freeze_time
from app.dao.fact_notification_status_dao import (
fetch_monthly_notification_statuses_per_service,
fetch_monthly_template_usage_for_service,
fetch_notification_status_for_day,
fetch_notification_status_for_service_by_month,
fetch_notification_status_for_service_for_day,
fetch_notification_status_for_service_for_today_and_7_previous_days,
@@ -16,11 +15,9 @@ from app.dao.fact_notification_status_dao import (
fetch_notification_statuses_for_job,
fetch_stats_for_all_services_by_date_range,
get_total_notifications_for_date_range,
update_fact_notification_status,
)
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
@@ -34,112 +31,16 @@ from app.models import (
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
)
from tests.app.db import (
create_ft_notification_status,
create_job,
create_notification,
create_notification_history,
create_service,
create_service_data_retention,
create_template,
)
def test_update_fact_notification_status(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_service_data_retention(second_service, 'email', days_of_retention=3)
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
for notification_type in ('letter', 'sms', 'email'):
data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
update_fact_notification_status(data=data, process_day=process_day, notification_type=notification_type)
new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].key_type == KEY_TYPE_NORMAL
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[1].key_type == KEY_TYPE_TEAM
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
assert new_fact_data[2].key_type == KEY_TYPE_NORMAL
def test__update_fact_notification_status_updates_row(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
create_notification(template=first_template, status='delivered')
process_day = date.today()
data = fetch_notification_status_for_day(process_day=process_day, notification_type='sms')
update_fact_notification_status(data=data, process_day=process_day, notification_type='sms')
new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
create_notification(template=first_template, status='delivered')
data = fetch_notification_status_for_day(process_day=process_day, notification_type='sms')
update_fact_notification_status(data=data, process_day=process_day, notification_type='sms')
updated_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(updated_fact_data) == 1
assert updated_fact_data[0].notification_count == 2
def test_fetch_notification_status_for_service_by_month(notify_db_session):
service_1 = create_service(service_name='service_1')
service_2 = create_service(service_name='service_2')