Move saving of processing time into separate task

We current do this as part of send-daily-performance-platform-stats but
now this moves it into its own separate task. This is for two reasons
- we will shortly get rid of the send-daily-performance-platform-stats
  task as we no longer will need to send anything to performance
  platform
- even if we did decide to keep the task
  send-daily-performance-platform-stats and remove the specific bits
  that relate to the performance platform, it's probably nicer to
  rewrite the new task from scratch to make sure it's all clear and easy
  to understand

app/celery/nightly_tasks.py

@@ -14,12 +14,14 @@ from app.celery.service_callback_tasks import (
 )
 from app.config import QueueNames
 from app.cronitor import cronitor
+from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.inbound_sms_dao import delete_inbound_sms_older_than_retention
 from app.dao.jobs_dao import (
     dao_archive_job,
     dao_get_jobs_older_than_data_retention,
 )
 from app.dao.notifications_dao import (
+    dao_get_total_notifications_sent_per_day_for_performance_platform,
     dao_timeout_notifications,
     delete_notifications_older_than_retention_by_type,
 )
@@ -33,6 +35,7 @@ from app.models import (
     LETTER_TYPE,
     NOTIFICATION_SENDING,
     SMS_TYPE,
+    FactProcessingTime,
     Notification,
 )
 from app.performance_platform import processing_time, total_sent_notifications
@@ -318,3 +321,26 @@ def letter_raise_alert_if_no_ack_file_for_zip():
         current_app.logger.info(
             "letter ack contains zip that is not for today: {}".format(ack_file_set - zip_file_set)
         )
+
+
+@notify_celery.task(name='save-daily-notification-processing-time')
+@cronitor("save-daily-notification-processing-time")
+@statsd(namespace="tasks")
+def save_daily_notification_processing_time(bst_date=None):
+    # bst_date is a string in the format of "YYYY-MM-DD"
+    if bst_date is None:
+        # if a date is not provided, we run against yesterdays data
+        bst_date = (datetime.utcnow() - timedelta(days=1)).date()
+    else:
+        bst_date = datetime.strptime(bst_date, "%Y-%m-%d").date()
+
+    start_time = get_london_midnight_in_utc(bst_date)
+    end_time = get_london_midnight_in_utc(bst_date + timedelta(days=1))
+    result = dao_get_total_notifications_sent_per_day_for_performance_platform(start_time, end_time)
+    insert_update_processing_time(
+        FactProcessingTime(
+            bst_date=bst_date,
+            messages_total=result.messages_total,
+            messages_within_10_secs=result.messages_within_10_secs
+        )
+    )

app/config.py

@@ -278,6 +278,11 @@ class Config(object):
             'schedule': crontab(hour=2, minute=0),
             'options': {'queue': QueueNames.PERIODIC}
         },
+        'save-daily-notification-processing-time': {
+            'task': 'save-daily-notification-processing-time',
+            'schedule': crontab(hour=2, minute=0),
+            'options': {'queue': QueueNames.PERIODIC}
+        },
         'remove_sms_email_jobs': {
             'task': 'remove_sms_email_jobs',
             'schedule': crontab(hour=4, minute=0),

app/performance_platform/processing_time.py

@@ -3,11 +3,9 @@ from datetime import timedelta
 from flask import current_app
 
 from app import performance_platform_client
-from app.dao.fact_processing_time_dao import insert_update_processing_time
 from app.dao.notifications_dao import (
     dao_get_total_notifications_sent_per_day_for_performance_platform,
 )
-from app.models import FactProcessingTime
 from app.utils import get_london_midnight_in_utc
 
 
@@ -29,11 +27,6 @@ def send_processing_time_for_start_and_end(start_time, end_time, bst_date):
 
     send_processing_time_data(start_time, 'messages-total', result.messages_total)
     send_processing_time_data(start_time, 'messages-within-10-secs', result.messages_within_10_secs)
-    insert_update_processing_time(FactProcessingTime(
-        bst_date=bst_date,
-        messages_total=result.messages_total,
-        messages_within_10_secs=result.messages_within_10_secs)
-    )
 
 
 def send_processing_time_data(start_time, status, count):

tests/app/celery/test_nightly_tasks.py

@@ -19,6 +19,7 @@ from app.celery.nightly_tasks import (
     remove_letter_csv_files,
     remove_sms_email_csv_files,
     s3,
+    save_daily_notification_processing_time,
     send_daily_performance_platform_stats,
     send_total_sent_notifications_to_performance_platform,
     timeout_notifications,
@@ -31,7 +32,7 @@ from app.clients.performance_platform.performance_platform_client import (
 )
 from app.config import QueueNames
 from app.exceptions import NotificationTechnicalFailureException
-from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE
+from app.models import EMAIL_TYPE, LETTER_TYPE, SMS_TYPE, FactProcessingTime
 from tests.app.db import (
     create_ft_notification_status,
     create_job,
@@ -439,3 +440,77 @@ def test_letter_not_raise_alert_if_no_files_do_not_cause_error(mocker, notify_db
     letter_raise_alert_if_no_ack_file_for_zip()
 
     assert mock_file_list.call_count == 2
+
+
+@freeze_time('2021-01-18T02:00')
+@pytest.mark.parametrize('date_provided', [None, '2021-1-17'])
+def test_save_daily_notification_processing_time(mocker, sample_template, date_provided):
+    # notification created too early to be counted
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 1, 16, 23, 59),
+        sent_at=datetime(2021, 1, 16, 23, 59) + timedelta(seconds=5)
+    )
+    # notification counted and sent within 10 seconds
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 1, 17, 00, 00),
+        sent_at=datetime(2021, 1, 17, 00, 00) + timedelta(seconds=5)
+    )
+    # notification counted but not sent within 10 seconds
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 1, 17, 23, 59),
+        sent_at=datetime(2021, 1, 17, 23, 59) + timedelta(seconds=15)
+    )
+    # notification created too late to be counted
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 1, 18, 00, 00),
+        sent_at=datetime(2021, 1, 18, 00, 00) + timedelta(seconds=5)
+    )
+
+    save_daily_notification_processing_time(date_provided)
+
+    persisted_to_db = FactProcessingTime.query.all()
+    assert len(persisted_to_db) == 1
+    assert persisted_to_db[0].bst_date == date(2021, 1, 17)
+    assert persisted_to_db[0].messages_total == 2
+    assert persisted_to_db[0].messages_within_10_secs == 1
+
+
+@freeze_time('2021-04-18T02:00')
+@pytest.mark.parametrize('date_provided', [None, '2021-4-17'])
+def test_save_daily_notification_processing_time_when_in_bst(mocker, sample_template, date_provided):
+    # notification created too early to be counted
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 4, 16, 22, 59),
+        sent_at=datetime(2021, 4, 16, 22, 59) + timedelta(seconds=15)
+    )
+    # notification counted and sent within 10 seconds
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 4, 16, 23, 00),
+        sent_at=datetime(2021, 4, 16, 23, 00) + timedelta(seconds=5)
+    )
+    # notification counted and sent within 10 seconds
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 4, 17, 22, 59),
+        sent_at=datetime(2021, 4, 17, 22, 59) + timedelta(seconds=5)
+    )
+    # notification created too late to be counted
+    create_notification(
+        sample_template,
+        created_at=datetime(2021, 4, 17, 23, 00),
+        sent_at=datetime(2021, 4, 17, 23, 00) + timedelta(seconds=15)
+    )
+
+    save_daily_notification_processing_time(date_provided)
+
+    persisted_to_db = FactProcessingTime.query.all()
+    assert len(persisted_to_db) == 1
+    assert persisted_to_db[0].bst_date == date(2021, 4, 17)
+    assert persisted_to_db[0].messages_total == 2
+    assert persisted_to_db[0].messages_within_10_secs == 2

tests/app/performance_platform/test_processing_time.py

@@ -2,7 +2,6 @@ from datetime import date, datetime, timedelta
 
 from freezegun import freeze_time
 
-from app.models import FactProcessingTime
 from app.performance_platform.processing_time import (
     send_processing_time_data,
     send_processing_time_to_performance_platform,
@@ -24,11 +23,6 @@ def test_send_processing_time_to_performance_platform_generates_correct_calls(mo
 
     send_mock.assert_any_call(datetime(2016, 10, 16, 23, 0), 'messages-total', 2)
     send_mock.assert_any_call(datetime(2016, 10, 16, 23, 0), 'messages-within-10-secs', 1)
-    persisted_to_db = FactProcessingTime.query.all()
-    assert len(persisted_to_db) == 1
-    assert persisted_to_db[0].bst_date == date(2016, 10, 17)
-    assert persisted_to_db[0].messages_total == 2
-    assert persisted_to_db[0].messages_within_10_secs == 1
 
 
 def test_send_processing_time_to_performance_platform_creates_correct_call_to_perf_platform(mocker):