Merge pull request #3437 from alphagov/retry-parallel-status-180693991

Attempt 2 of parallelising status aggregation more
2026-08-04 05:30:53 -04:00 · 2022-02-14 10:50:26 +00:00
parent e5b1a65f1e 966c4db8c6
commit 5a87d8c7d7
9 changed files with 166 additions and 87 deletions
--- a/tests/app/celery/test_reporting_tasks.py
+++ b/tests/app/celery/test_reporting_tasks.py
@@ -1,4 +1,3 @@
-import itertools
 from datetime import date, datetime, time, timedelta
 from decimal import Decimal
 from uuid import UUID
@@ -10,7 +9,7 @@ from app.celery.reporting_tasks import (
    create_nightly_billing,
    create_nightly_billing_for_day,
    create_nightly_notification_status,
-    create_nightly_notification_status_for_day,
+    create_nightly_notification_status_for_service_and_day,
 )
 from app.config import QueueNames
 from app.dao.fact_billing_dao import get_rate
@@ -20,6 +19,7 @@ from app.models import (
    KEY_TYPE_TEAM,
    KEY_TYPE_TEST,
    LETTER_TYPE,
+    NOTIFICATION_TYPES,
    SMS_TYPE,
    FactBilling,
    FactNotificationStatus,
@@ -61,37 +61,58 @@ def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_
        assert mock_celery.apply_async.call_args_list[i][1]['kwargs'] == {'process_day': expected_kwargs[i]}


-@freeze_time('2019-08-01')
-def test_create_nightly_notification_status_triggers_tasks(notify_api, sample_service, mocker):
-    mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_day')
+@freeze_time('2019-08-01T00:30')
+def test_create_nightly_notification_status_triggers_tasks(
+    notify_api,
+    sample_service,
+    sample_template,
+    mocker,
+):
+    mock_celery = mocker.patch(
+        'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
+    ).apply_async
+
+    create_notification(template=sample_template, created_at='2019-07-31')
    create_nightly_notification_status()

-    assert mock_celery.apply_async.call_count == (
-        (4 * 3)  # four days, three notification types
-        +
-        6  # six more days of just letters
+    mock_celery.assert_called_with(
+        kwargs={
+            'service_id': sample_service.id,
+            'process_day': '2019-07-31',
+            'notification_type': SMS_TYPE
+        },
+        queue=QueueNames.REPORTING
    )

-    for process_date, notification_type in itertools.product(
-        ['2019-07-31', '2019-07-30', '2019-07-29', '2019-07-28'],
-        [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
-    ):
-        mock_celery.apply_async.assert_any_call(
-            kwargs={
-                'process_day': process_date,
-                'notification_type': notification_type,
-            },
-            queue=QueueNames.REPORTING
-        )

-    for process_date in ['2019-07-27', '2019-07-26', '2019-07-25', '2019-07-24', '2019-07-23', '2019-07-22']:
-        mock_celery.apply_async.assert_any_call(
-            kwargs={
-                'process_day': process_date,
-                'notification_type': LETTER_TYPE,
-            },
-            queue=QueueNames.REPORTING
-        )
+@freeze_time('2019-08-01T00:30')
+@pytest.mark.parametrize('notification_date, expected_types_aggregated', [
+    ('2019-08-01', set()),
+    ('2019-07-31', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
+    ('2019-07-28', {EMAIL_TYPE, SMS_TYPE, LETTER_TYPE}),
+    ('2019-07-27', {LETTER_TYPE}),
+    ('2019-07-22', {LETTER_TYPE}),
+    ('2019-07-21', set()),
+])
+def test_create_nightly_notification_status_triggers_relevant_tasks(
+    notify_api,
+    sample_service,
+    mocker,
+    notification_date,
+    expected_types_aggregated,
+):
+    mock_celery = mocker.patch(
+        'app.celery.reporting_tasks.create_nightly_notification_status_for_service_and_day'
+    ).apply_async
+
+    for notification_type in NOTIFICATION_TYPES:
+        template = create_template(sample_service, template_type=notification_type)
+        create_notification(template=template, created_at=notification_date)
+
+    create_nightly_notification_status()
+
+    types = {call.kwargs['kwargs']['notification_type'] for call in mock_celery.mock_calls}
+    assert types == expected_types_aggregated


@pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
@@ -507,7 +528,7 @@ def test_create_nightly_billing_for_day_update_when_record_exists(
    assert records[0].updated_at


-def test_create_nightly_notification_status_for_day(notify_db_session):
+def test_create_nightly_notification_status_for_service_and_day(notify_db_session):
    first_service = create_service(service_name='First Service')
    first_template = create_template(service=first_service)
    second_service = create_service(service_name='second Service')
@@ -538,9 +559,9 @@ def test_create_nightly_notification_status_for_day(notify_db_session):

    assert len(FactNotificationStatus.query.all()) == 0

-    create_nightly_notification_status_for_day(str(process_day), 'sms')
-    create_nightly_notification_status_for_day(str(process_day), 'email')
-    create_nightly_notification_status_for_day(str(process_day), 'letter')
+    create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')
+    create_nightly_notification_status_for_service_and_day(str(process_day), second_service.id, 'email')
+    create_nightly_notification_status_for_service_and_day(str(process_day), third_service.id, 'letter')

    new_fact_data = FactNotificationStatus.query.order_by(
        FactNotificationStatus.notification_type
@@ -575,13 +596,13 @@ def test_create_nightly_notification_status_for_day(notify_db_session):
    assert new_fact_data[2].key_type == KEY_TYPE_NORMAL


-def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_db_session):
+def test_create_nightly_notification_status_for_service_and_day_overwrites_old_data(notify_db_session):
    first_service = create_service(service_name='First Service')
    first_template = create_template(service=first_service)
    create_notification(template=first_template, status='delivered')

    process_day = date.today()
-    create_nightly_notification_status_for_day(str(process_day), 'sms')
+    create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')

    new_fact_data = FactNotificationStatus.query.order_by(
        FactNotificationStatus.bst_date,
@@ -592,7 +613,7 @@ def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_d
    assert new_fact_data[0].notification_count == 1

    create_notification(template=first_template, status='delivered')
-    create_nightly_notification_status_for_day(str(process_day), 'sms')
+    create_nightly_notification_status_for_service_and_day(str(process_day), first_service.id, 'sms')

    updated_fact_data = FactNotificationStatus.query.order_by(
        FactNotificationStatus.bst_date,
@@ -605,7 +626,7 @@ def test_create_nightly_notification_status_for_day_overwrites_old_data(notify_d

 # the job runs at 12:30am London time. 04/01 is in BST.
@freeze_time('2019-04-01T23:30')
-def test_create_nightly_notification_status_for_day_respects_bst(sample_template):
+def test_create_nightly_notification_status_for_service_and_day_respects_bst(sample_template):
    create_notification(sample_template, status='delivered', created_at=datetime(2019, 4, 1, 23, 0))  # too new

    create_notification(sample_template, status='created', created_at=datetime(2019, 4, 1, 22, 59))
@@ -613,7 +634,7 @@ def test_create_nightly_notification_status_for_day_respects_bst(sample_template

    create_notification(sample_template, status='delivered', created_at=datetime(2019, 3, 31, 22, 59))  # too old

-    create_nightly_notification_status_for_day('2019-04-01', 'sms')
+    create_nightly_notification_status_for_service_and_day('2019-04-01', sample_template.service_id, 'sms')

    noti_status = FactNotificationStatus.query.order_by(FactNotificationStatus.bst_date).all()
    assert len(noti_status) == 1
--- a/tests/app/dao/notification_dao/test_notification_dao.py
+++ b/tests/app/dao/notification_dao/test_notification_dao.py
@@ -1,5 +1,5 @@
 import uuid
-from datetime import datetime, timedelta
+from datetime import date, datetime, timedelta
 from functools import partial

 import pytest
@@ -24,6 +24,7 @@ from app.dao.notifications_dao import (
    get_notification_with_personalisation,
    get_notifications_for_job,
    get_notifications_for_service,
+    get_service_ids_with_notifications_on_date,
    is_delivery_slow_for_providers,
    notifications_not_yet_sent,
    update_notification_status_by_id,
@@ -41,6 +42,7 @@ from app.models import (
    NOTIFICATION_STATUS_TYPES,
    NOTIFICATION_STATUS_TYPES_FAILED,
    NOTIFICATION_TEMPORARY_FAILURE,
+    SMS_TYPE,
    Job,
    Notification,
    NotificationHistory,
@@ -1721,3 +1723,22 @@ def test_dao_get_letters_and_sheets_volume_by_postage(notify_db_session):

    for result in results:
        assert result._asdict() in expected_results
+
+
+@pytest.mark.parametrize('created_at_utc,date_to_check,expected_count', [
+    # Clocks change on the 27th of March 2022, so the query needs to look at the
+    # time range 00:00 - 23:00 (UTC) thereafter.
+    ('2022-03-27T00:30', date(2022, 3, 27), 1),  # 27/03 00:30 GMT
+    ('2022-03-27T22:30', date(2022, 3, 27), 1),  # 27/03 23:30 BST
+    ('2022-03-27T23:30', date(2022, 3, 27), 0),  # 28/03 00:30 BST
+    ('2022-03-26T23:30', date(2022, 3, 26), 1),  # 26/03 23:30 GMT
+])
+def test_get_service_ids_with_notifications_on_date_respects_gmt_bst(
+    sample_template,
+    created_at_utc,
+    date_to_check,
+    expected_count
+):
+    create_notification(template=sample_template, created_at=created_at_utc)
+    service_ids = get_service_ids_with_notifications_on_date(SMS_TYPE, date_to_check)
+    assert len(service_ids) == expected_count
--- a/tests/app/dao/test_fact_notification_status_dao.py
+++ b/tests/app/dao/test_fact_notification_status_dao.py
@@ -14,6 +14,7 @@ from app.dao.fact_notification_status_dao import (
    fetch_notification_status_totals_for_all_services,
    fetch_notification_statuses_for_job,
    fetch_stats_for_all_services_by_date_range,
+    fetch_status_data_for_service_and_day,
    get_total_notifications_for_date_range,
 )
 from app.models import (
@@ -607,3 +608,23 @@ def test_get_total_notifications_for_date_range(sample_service):

    assert len(results) == 1
    assert results[0] == ("2021-03-01", 15, 20, 3)
+
+
+@pytest.mark.parametrize('created_at_utc,process_day,expected_count', [
+    # Clocks change on the 27th of March 2022, so the query needs to look at the
+    # time range 00:00 - 23:00 (UTC) thereafter.
+    ('2022-03-27T00:30', date(2022, 3, 27), 1),  # 27/03 00:30 GMT
+    ('2022-03-27T22:30', date(2022, 3, 27), 1),  # 27/03 23:30 BST
+    ('2022-03-27T23:30', date(2022, 3, 27), 0),  # 28/03 00:30 BST
+    ('2022-03-26T23:30', date(2022, 3, 26), 1),  # 26/03 23:30 GMT
+])
+def test_fetch_status_data_for_service_and_day_respects_gmt_bst(
+    sample_template,
+    sample_service,
+    created_at_utc,
+    process_day,
+    expected_count,
+):
+    create_notification(template=sample_template, created_at=created_at_utc)
+    rows = fetch_status_data_for_service_and_day(process_day, sample_service.id, SMS_TYPE)
+    assert len(rows) == expected_count