darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2026-02-04 02:11:11 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3425 from alphagov/parallelise-ft-status-180693991

Browse Source 
Parallelise status aggregation by service and day
This commit is contained in:
Ben Thorner
2022-01-19 13:49:28 +00:00
committed by GitHub
parent 5cd6fcbb4f 9686595fa8
commit 6be489daa7
5 changed files with 164 additions and 314 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
app/celery/reporting_tasks.py
View File

@@ -3,7 +3,7 @@ from datetime import datetime, timedelta
from flask import current_app
from notifications_utils.timezones import convert_utc_to_bst
from app import notify_celery
from app import db, notify_celery
from app.config import QueueNames
from app.cronitor import cronitor
from app.dao.fact_billing_dao import (
@@ -11,10 +11,10 @@ from app.dao.fact_billing_dao import (
update_fact_billing,
)
from app.dao.fact_notification_status_dao import (
fetch_notification_status_for_day,
fetch_status_data_for_service_and_day,
update_fact_notification_status,
)
from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE
from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, Service
@notify_celery.task(name="create-nightly-billing")
@@ -91,50 +91,58 @@ def create_nightly_notification_status():
yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
# email and sms
for i in range(4):
process_day = yesterday - timedelta(days=i)
for notification_type in [SMS_TYPE, EMAIL_TYPE]:
create_nightly_notification_status_for_day.apply_async(
kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
queue=QueueNames.REPORTING
)
current_app.logger.info(
f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
f"for type {notification_type} for {process_day}"
)
for (service_id,) in db.session.query(Service.id):
for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
days = 10 if notification_type == LETTER_TYPE else 4
# letters
for i in range(10):
process_day = yesterday - timedelta(days=i)
create_nightly_notification_status_for_day.apply_async(
kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
queue=QueueNames.REPORTING
)
current_app.logger.info(
f"create-nightly-notification-status task: create-nightly-notification-status-for-day task created "
f"for type letter for {process_day}"
)
for i in range(days):
process_day = yesterday - timedelta(days=i)
create_nightly_notification_status_for_service_and_day.apply_async(
kwargs={
'process_day': process_day.isoformat(),
'notification_type': notification_type,
'service_id': service_id,
},
queue=QueueNames.REPORTING
)
current_app.logger.info(
f"create-nightly-notification-status-for-day task created "
f"for {service_id}, {notification_type} and {process_day}"
)
@notify_celery.task(name="create-nightly-notification-status-for-day")
def create_nightly_notification_status_for_day(process_day, notification_type):
@notify_celery.task(name="create-nightly-notification-status-for-service-and-day")
def create_nightly_notification_status_for_service_and_day(process_day, service_id, notification_type):
process_day = datetime.strptime(process_day, "%Y-%m-%d").date()
current_app.logger.info(
f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: started'
f'create-nightly-notification-status-for-day task started '
f'for {service_id}, {notification_type} for {process_day}'
)
start = datetime.utcnow()
transit_data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
new_status_rows = fetch_status_data_for_service_and_day(
process_day=process_day,
notification_type=notification_type,
service_id=service_id,
)
end = datetime.utcnow()
current_app.logger.info(
f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
f'create-nightly-notification-status-for-day task fetch '
f'for {service_id}, {notification_type} for {process_day}: '
f'data fetched in {(end - start).seconds} seconds'
)
update_fact_notification_status(transit_data, process_day, notification_type)
update_fact_notification_status(
new_status_rows=new_status_rows,
process_day=process_day,
notification_type=notification_type,
service_id=service_id
)
current_app.logger.info(
f'create-nightly-notification-status-for-day task for {process_day} type {notification_type}: '
f'task complete - {len(transit_data)} rows updated'
f'create-nightly-notification-status-for-day task finished '
f'for {service_id}, {notification_type} for {process_day}: '
f'{len(new_status_rows)} rows updated'
)

109
app/commands.py
View File

@@ -23,9 +23,6 @@ from app.celery.letters_pdf_tasks import (
get_pdf_for_templated_letter,
resanitise_pdf,
)
from app.celery.reporting_tasks import (
create_nightly_notification_status_for_day,
)
from app.celery.tasks import process_row, record_daily_sorted_counts
from app.config import QueueNames
from app.dao.annual_billing_dao import (
@@ -62,9 +59,7 @@ from app.dao.users_dao import (
get_user_by_email,
)
from app.models import (
EMAIL_TYPE,
KEY_TYPE_TEST,
LETTER_TYPE,
NOTIFICATION_CREATED,
PROVIDERS,
SMS_TYPE,
@@ -281,87 +276,6 @@ def setup_commands(application):
application.cli.add_command(command_group)
@notify_command(name='migrate-data-to-ft-billing')
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@statsd(namespace="tasks")
def migrate_data_to_ft_billing(start_date, end_date):
current_app.logger.info('Billing migration from date {} to {}'.format(start_date, end_date))
process_date = start_date
total_updated = 0
while process_date < end_date:
start_time = datetime.utcnow()
# migrate data into ft_billing, upserting the data if it the record already exists
sql = \
"""
insert into ft_billing (bst_date, template_id, service_id, notification_type, provider, rate_multiplier,
international, billable_units, notifications_sent, rate, postage, created_at)
select bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
sum(billable_units) as billable_units, sum(notifications_sent) as notification_sent,
case when notification_type = 'sms' then sms_rate else letter_rate end as rate, postage, created_at
from (
select
n.id,
(n.created_at at time zone 'UTC' at time zone 'Europe/London')::timestamp::date as bst_date,
coalesce(n.template_id, '00000000-0000-0000-0000-000000000000') as template_id,
coalesce(n.service_id, '00000000-0000-0000-0000-000000000000') as service_id,
n.notification_type,
coalesce(n.sent_by, (
case
when notification_type = 'sms' then
coalesce(sent_by, 'unknown')
when notification_type = 'letter' then
coalesce(sent_by, 'dvla')
else
coalesce(sent_by, 'ses')
end )) as provider,
coalesce(n.rate_multiplier,1) as rate_multiplier,
s.crown,
coalesce((select rates.rate from rates
where n.notification_type = rates.notification_type and n.created_at > rates.valid_from
order by rates.valid_from desc limit 1), 0) as sms_rate,
coalesce((select l.rate from letter_rates l where n.billable_units = l.sheet_count
and s.crown = l.crown and n.postage = l.post_class and n.created_at >= l.start_date
and n.created_at < coalesce(l.end_date, now()) and n.notification_type='letter'), 0)
as letter_rate,
coalesce(n.international, false) as international,
n.billable_units,
1 as notifications_sent,
coalesce(n.postage, 'none') as postage,
now() as created_at
from public.notification_history n
left join services s on s.id = n.service_id
where n.key_type!='test'
and n.notification_status in
('sending', 'sent', 'delivered', 'temporary-failure', 'permanent-failure', 'failed')
and n.created_at >= (date :start + time '00:00:00') at time zone 'Europe/London'
at time zone 'UTC'
and n.created_at < (date :end + time '00:00:00') at time zone 'Europe/London' at time zone 'UTC'
) as individual_record
group by bst_date, template_id, service_id, notification_type, provider, rate_multiplier, international,
sms_rate, letter_rate, postage, created_at
order by bst_date
on conflict on constraint ft_billing_pkey do update set
billable_units = excluded.billable_units,
notifications_sent = excluded.notifications_sent,
rate = excluded.rate,
updated_at = now()
"""
result = db.session.execute(sql, {"start": process_date, "end": process_date + timedelta(days=1)})
db.session.commit()
current_app.logger.info('ft_billing: --- Completed took {}ms. Migrated {} rows for {}'.format(
datetime.now() - start_time, result.rowcount, process_date))
process_date += timedelta(days=1)
total_updated += result.rowcount
current_app.logger.info('Total inserted/updated records = {}'.format(total_updated))
@notify_command(name='rebuild-ft-billing-for-day')
@click.option('-s', '--service_id', required=False, type=click.UUID)
@click.option('-d', '--day', help="The date to recalculate, as YYYY-MM-DD", required=True,
@@ -401,29 +315,6 @@ def rebuild_ft_billing_for_day(service_id, day):
rebuild_ft_data(day, row.service_id)
@notify_command(name='migrate-data-to-ft-notification-status')
@click.option('-s', '--start_date', required=True, help="start date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-e', '--end_date', required=True, help="end date inclusive", type=click_dt(format='%Y-%m-%d'))
@click.option('-t', '--notification-type', required=False, help="notification type (or leave blank for all types)")
@statsd(namespace="tasks")
def migrate_data_to_ft_notification_status(start_date, end_date, notification_type=None):
notification_types = [SMS_TYPE, LETTER_TYPE, EMAIL_TYPE] if notification_type is None else [notification_type]
start_date = start_date.date()
end_date = end_date.date()
for day_diff in range((end_date - start_date).days + 1):
process_day = start_date + timedelta(days=day_diff)
for notification_type in notification_types:
print('create_nightly_notification_status_for_day triggered for {} and {}'.format(
process_day,
notification_type
))
create_nightly_notification_status_for_day.apply_async(
kwargs={'process_day': process_day.strftime('%Y-%m-%d'), 'notification_type': notification_type},
queue=QueueNames.REPORTING
)
@notify_command(name='bulk-invite-user-to-service')
@click.option('-f', '--file_name', required=True,
help="Full path of the file containing a list of email address for people to invite to a service")

62
app/dao/fact_notification_status_dao.py
View File

@@ -1,6 +1,5 @@
from datetime import datetime, time, timedelta
from flask import current_app
from notifications_utils.timezones import convert_bst_to_utc
from sqlalchemy import Date, case, func
from sqlalchemy.dialects.postgresql import insert
@@ -36,35 +35,16 @@ from app.utils import (
)
def fetch_notification_status_for_day(process_day, notification_type):
def fetch_status_data_for_service_and_day(process_day, service_id, notification_type):
start_date = convert_bst_to_utc(datetime.combine(process_day, time.min))
end_date = convert_bst_to_utc(datetime.combine(process_day + timedelta(days=1), time.min))
current_app.logger.info("Fetch ft_notification_status for {} to {}".format(start_date, end_date))
# query notifications or notification_history for the day, depending on their data retention
service = Service.query.get(service_id)
table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
all_data_for_process_day = []
services = Service.query.all()
# for each service query notifications or notification_history for the day, depending on their data retention
for service in services:
table = get_notification_table_to_use(service, notification_type, process_day, has_delete_task_run=False)
data_for_service_and_type = query_for_fact_status_data(
table=table,
start_date=start_date,
end_date=end_date,
notification_type=notification_type,
service_id=service.id
)
all_data_for_process_day += data_for_service_and_type
return all_data_for_process_day
def query_for_fact_status_data(table, start_date, end_date, notification_type, service_id):
query = db.session.query(
return db.session.query(
table.template_id,
table.service_id,
func.coalesce(table.job_id, '00000000-0000-0000-0000-000000000000').label('job_id'),
table.key_type,
table.status,
@@ -77,34 +57,34 @@ def query_for_fact_status_data(table, start_date, end_date, notification_type, s
table.key_type.in_((KEY_TYPE_NORMAL, KEY_TYPE_TEAM)),
).group_by(
table.template_id,
table.service_id,
'job_id',
table.key_type,
table.status
)
return query.all()
).all()
@autocommit
def update_fact_notification_status(data, process_day, notification_type):
def update_fact_notification_status(new_status_rows, process_day, notification_type, service_id):
table = FactNotificationStatus.__table__
FactNotificationStatus.query.filter(
FactNotificationStatus.bst_date == process_day,
FactNotificationStatus.notification_type == notification_type
FactNotificationStatus.notification_type == notification_type,
FactNotificationStatus.service_id == service_id,
).delete()
for row in data:
stmt = insert(table).values(
bst_date=process_day,
template_id=row.template_id,
service_id=row.service_id,
job_id=row.job_id,
notification_type=notification_type,
key_type=row.key_type,
notification_status=row.status,
notification_count=row.notification_count,
for row in new_status_rows:
db.session.connection().execute(
insert(table).values(
bst_date=process_day,
template_id=row.template_id,
service_id=service_id,
job_id=row.job_id,
notification_type=notification_type,
key_type=row.key_type,
notification_status=row.status,
notification_count=row.notification_count,
)
)
db.session.connection().execute(stmt)
def fetch_notification_status_for_service_by_month(start_date, end_date, service_id):

130
tests/app/celery/test_reporting_tasks.py
View File

@@ -1,6 +1,7 @@
import itertools
from datetime import date, datetime, timedelta
from datetime import date, datetime, time, timedelta
from decimal import Decimal
from uuid import UUID
import pytest
from freezegun import freeze_time
@@ -9,12 +10,15 @@ from app.celery.reporting_tasks import (
create_nightly_billing,
create_nightly_billing_for_day,
create_nightly_notification_status,
create_nightly_notification_status_for_day,
create_nightly_notification_status_for_service_and_day,
)
from app.config import QueueNames
from app.dao.fact_billing_dao import get_rate
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
SMS_TYPE,
FactBilling,
@@ -24,8 +28,10 @@ from app.models import (
from tests.app.db import (
create_letter_rate,
create_notification,
create_notification_history,
create_rate,
create_service,
create_service_data_retention,
create_template,
)
@@ -56,8 +62,8 @@ def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_
@freeze_time('2019-08-01')
def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, mocker):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_day')
def test_create_nightly_notification_status_triggers_tasks(notify_api, sample_service, mocker):
mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day')
create_nightly_notification_status()
assert mock_celery.apply_async.call_count == (
@@ -71,13 +77,21 @@ def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api,
[SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
):
mock_celery.apply_async.assert_any_call(
kwargs={'process_day': process_date, 'notification_type': notification_type},
kwargs={
'process_day': process_date,
'notification_type': notification_type,
'service_id': sample_service.id,
},
queue=QueueNames.REPORTING
)
for process_date in ['2019-07-27', '2019-07-26', '2019-07-25', '2019-07-24', '2019-07-23', '2019-07-22']:
mock_celery.apply_async.assert_any_call(
kwargs={'process_day': process_date, 'notification_type': LETTER_TYPE},
kwargs={
'process_day': process_date,
'notification_type': LETTER_TYPE,
'service_id': sample_service.id,
},
queue=QueueNames.REPORTING
)
@@ -495,8 +509,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
assert records[0].updated_at
@freeze_time('2019-01-05')
def test_create_nightly_notification_status_for_day(notify_db_session):
def test_create_nightly_notification_status_for_service_and_day(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
@@ -504,40 +517,97 @@ def test_create_nightly_notification_status_for_day(notify_db_session):
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_notification(template=first_template, status='delivered')
create_notification(template=first_template, status='delivered', created_at=datetime(2019, 1, 1, 12, 0))
create_service_data_retention(second_service, 'email', days_of_retention=3)
create_notification(template=second_template, status='temporary-failure')
create_notification(template=second_template, status='temporary-failure', created_at=datetime(2019, 1, 1, 12, 0))
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
create_notification(template=third_template, status='created')
create_notification(template=third_template, status='created', created_at=datetime(2019, 1, 1, 12, 0))
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
assert len(FactNotificationStatus.query.all()) == 0
create_nightly_notification_status_for_day('2019-01-01', 'sms')
create_nightly_notification_status_for_day('2019-01-01', 'email')
create_nightly_notification_status_for_day('2019-01-01', 'letter')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
create_nightly_notification_status_for_service_and_day(str(process_day), third_service.id, 'letter')
new_data = FactNotificationStatus.query.order_by(FactNotificationStatus.created_at).all()
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.notification_type
).all()
assert len(new_data) == 3
assert new_data[0].bst_date == date(2019, 1, 1)
assert new_data[1].bst_date == date(2019, 1, 1)
assert new_data[2].bst_date == date(2019, 1, 1)
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].key_type == KEY_TYPE_NORMAL
assert new_data[0].notification_type == 'sms'
assert new_data[1].notification_type == 'email'
assert new_data[2].notification_type == 'letter'
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[1].key_type == KEY_TYPE_TEAM
assert new_data[0].notification_status == 'delivered'
assert new_data[1].notification_status == 'temporary-failure'
assert new_data[2].notification_status == 'created'
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
assert new_fact_data[2].key_type == KEY_TYPE_NORMAL
def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
create_notification(template=first_template, status='delivered')
process_day = date.today()
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
new_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
create_notification(template=first_template, status='delivered')
create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
updated_fact_data = FactNotificationStatus.query.order_by(
FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(updated_fact_data) == 1
assert updated_fact_data[0].notification_count == 2
# the job runs at 12:30am London time. 04/01 is in BST.
@freeze_time('2019-04-01T23:30')
def test_create_nightly_notification_status_for_day_respects_bst(sample_template):
def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template):
create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 1, 23, 0)) # too new
create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 22, 59))
@@ -545,7 +615,7 @@ def test_create_nightly_notification_status_for_day_respects_bst(sample_template
create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 31, 22, 59)) # too old
create_nightly_notification_status_for_day('2019-04-01', 'sms')
create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms')
noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date).all()
assert len(noti_status) == 1

101
tests/app/dao/test_fact_notification_status_dao.py
View File

@@ -1,4 +1,4 @@
from datetime import date, datetime, time, timedelta
from datetime import date, datetime, timedelta
from unittest import mock
from uuid import UUID
@@ -8,7 +8,6 @@ from freezegun import freeze_time
from app.dao.fact_notification_status_dao import (
fetch_monthly_notification_statuses_per_service,
fetch_monthly_template_usage_for_service,
fetch_notification_status_for_day,
fetch_notification_status_for_service_by_month,
fetch_notification_status_for_service_for_day,
fetch_notification_status_for_service_for_today_and_7_previous_days,
@@ -16,11 +15,9 @@ from app.dao.fact_notification_status_dao import (
fetch_notification_statuses_for_job,
fetch_stats_for_all_services_by_date_range,
get_total_notifications_for_date_range,
update_fact_notification_status,
)
from app.models import (
EMAIL_TYPE,
KEY_TYPE_NORMAL,
KEY_TYPE_TEAM,
KEY_TYPE_TEST,
LETTER_TYPE,
@@ -34,112 +31,16 @@ from app.models import (
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
)
from tests.app.db import (
create_ft_notification_status,
create_job,
create_notification,
create_notification_history,
create_service,
create_service_data_retention,
create_template,
)
def test_update_fact_notification_status(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
second_service = create_service(service_name='second Service')
second_template = create_template(service=second_service, template_type='email')
third_service = create_service(service_name='third Service')
third_template = create_template(service=third_service, template_type='letter')
create_service_data_retention(second_service, 'email', days_of_retention=3)
process_day = date.today() - timedelta(days=5)
with freeze_time(datetime.combine(process_day, time.min)):
create_notification(template=first_template, status='delivered')
# 2nd service email has 3 day data retention - data has been moved to history and doesn't exist in notifications
create_notification_history(template=second_template, status='temporary-failure')
# team API key notifications are included
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEAM)
# test notifications are ignored
create_notification(template=third_template, status='sending', key_type=KEY_TYPE_TEST)
# these created notifications from a different day get ignored
with freeze_time(datetime.combine(date.today() - timedelta(days=4), time.min)):
create_notification(template=first_template)
create_notification_history(template=second_template)
create_notification(template=third_template)
for notification_type in ('letter', 'sms', 'email'):
data = fetch_notification_status_for_day(process_day=process_day, notification_type=notification_type)
update_fact_notification_status(data=data, process_day=process_day, notification_type=notification_type)
new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 3
assert new_fact_data[0].bst_date == process_day
assert new_fact_data[0].template_id == second_template.id
assert new_fact_data[0].service_id == second_service.id
assert new_fact_data[0].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[0].notification_type == 'email'
assert new_fact_data[0].notification_status == 'temporary-failure'
assert new_fact_data[0].notification_count == 1
assert new_fact_data[0].key_type == KEY_TYPE_NORMAL
assert new_fact_data[1].bst_date == process_day
assert new_fact_data[1].template_id == third_template.id
assert new_fact_data[1].service_id == third_service.id
assert new_fact_data[1].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[1].notification_type == 'letter'
assert new_fact_data[1].notification_status == 'sending'
assert new_fact_data[1].notification_count == 1
assert new_fact_data[1].key_type == KEY_TYPE_TEAM
assert new_fact_data[2].bst_date == process_day
assert new_fact_data[2].template_id == first_template.id
assert new_fact_data[2].service_id == first_service.id
assert new_fact_data[2].job_id == UUID('00000000-0000-0000-0000-000000000000')
assert new_fact_data[2].notification_type == 'sms'
assert new_fact_data[2].notification_status == 'delivered'
assert new_fact_data[2].notification_count == 1
assert new_fact_data[2].key_type == KEY_TYPE_NORMAL
def test__update_fact_notification_status_updates_row(notify_db_session):
first_service = create_service(service_name='First Service')
first_template = create_template(service=first_service)
create_notification(template=first_template, status='delivered')
process_day = date.today()
data = fetch_notification_status_for_day(process_day=process_day, notification_type='sms')
update_fact_notification_status(data=data, process_day=process_day, notification_type='sms')
new_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(new_fact_data) == 1
assert new_fact_data[0].notification_count == 1
create_notification(template=first_template, status='delivered')
data = fetch_notification_status_for_day(process_day=process_day, notification_type='sms')
update_fact_notification_status(data=data, process_day=process_day, notification_type='sms')
updated_fact_data = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date,
FactNotificationStatus.notification_type
).all()
assert len(updated_fact_data) == 1
assert updated_fact_data[0].notification_count == 2
def test_fetch_notification_status_for_service_by_month(notify_db_session):
service_1 = create_service(service_name='service_1')
service_2 = create_service(service_name='service_2')