Update FactStatus table in bulk for each service

Previously we were looping over data from the Notifications/History
table and then shovelling it into the status table, one row at a time
- plus an extra delete to clean up any existing data.

This replaces that with a batch insertion, similar to how we archive
notifications [1], but using a simple subquery (via "from_select" [2])
instead of a temporary table.

To make the select compatible with the insert, I've used "literal"
to inject the constant pieces of data, so each row has everything it
needs to go into the status table.

[1]: 9ce6d2fe92/app/dao/notifications_dao.py (L295)
[2]: https://docs.sqlalchemy.org/en/14/core/dml.html#sqlalchemy.sql.expression.Insert.from_select
This commit is contained in:
Ben Thorner
2022-02-10 15:00:57 +00:00
parent efde271c5a
commit a69d1635a1
3 changed files with 58 additions and 64 deletions

View File

@@ -14,8 +14,8 @@ from app.dao.fact_notification_status_dao import (
fetch_notification_status_totals_for_all_services,
fetch_notification_statuses_for_job,
fetch_stats_for_all_services_by_date_range,
fetch_status_data_for_service_and_day,
get_total_notifications_for_date_range,
update_fact_notification_status,
)
from app.models import (
EMAIL_TYPE,
@@ -32,6 +32,7 @@ from app.models import (
NOTIFICATION_TECHNICAL_FAILURE,
NOTIFICATION_TEMPORARY_FAILURE,
SMS_TYPE,
FactNotificationStatus,
)
from tests.app.db import (
create_ft_notification_status,
@@ -618,7 +619,7 @@ def test_get_total_notifications_for_date_range(sample_service):
('2022-03-27T23:30', date(2022, 3, 27), 0), # 28/03 00:30 BST
('2022-03-26T23:30', date(2022, 3, 26), 1), # 26/03 23:30 GMT
])
def test_fetch_status_data_for_service_and_day_respects_gmt_bst(
def test_update_fact_notification_status_respects_gmt_bst(
sample_template,
sample_service,
created_at_utc,
@@ -626,5 +627,9 @@ def test_fetch_status_data_for_service_and_day_respects_gmt_bst(
expected_count,
):
create_notification(template=sample_template, created_at=created_at_utc)
rows = fetch_status_data_for_service_and_day(process_day, sample_service.id, SMS_TYPE)
assert len(rows) == expected_count
update_fact_notification_status(process_day, SMS_TYPE, sample_service.id)
assert FactNotificationStatus.query.filter_by(
service_id=sample_service.id,
bst_date=process_day
).count() == expected_count