Merge pull request #2671 from alphagov/letters-for-longer

process letters for 10 days when updating ft_notification_status

app/celery/reporting_tasks.py

@@ -67,21 +67,24 @@ def create_nightly_billing_for_day(process_day):
 @notify_celery.task(name="create-nightly-notification-status")
 @cronitor("create-nightly-notification-status")
 @statsd(namespace="tasks")
-def create_nightly_notification_status(day_start=None):
-    # day_start is a datetime.date() object. e.g.
-    # 4 days of data counting back from day_start is consolidated
-    if day_start is None:
-        day_start = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
-    else:
-        # When calling the task its a string in the format of "YYYY-MM-DD"
-        day_start = datetime.strptime(day_start, "%Y-%m-%d").date()
-    for i in range(0, 4):
-        process_day = day_start - timedelta(days=i)
-        for notification_type in [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]:
+def create_nightly_notification_status():
+    yesterday = convert_utc_to_bst(datetime.utcnow()).date() - timedelta(days=1)
+
+    # email and sms
+    for i in range(4):
+        process_day = yesterday - timedelta(days=i)
+        for notification_type in [SMS_TYPE, EMAIL_TYPE]:
             create_nightly_notification_status_for_day.apply_async(
                 kwargs={'process_day': process_day.isoformat(), 'notification_type': notification_type},
                 queue=QueueNames.REPORTING
             )
+    # letters get modified for a longer time period than sms and email, so we need to reprocess for more days
+    for i in range(10):
+        process_day = yesterday - timedelta(days=i)
+        create_nightly_notification_status_for_day.apply_async(
+            kwargs={'process_day': process_day.isoformat(), 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )
 
 
 @notify_celery.task(name="create-nightly-notification-status-for-day")

app/commands.py

@@ -505,7 +505,7 @@ def migrate_data_to_ft_notification_status(start_date, end_date, notification_ty
 
     start_date = start_date.date()
     end_date = end_date.date()
-    for day_diff in range((end_date - start_date).days):
+    for day_diff in range((end_date - start_date).days + 1):
         process_day = start_date + timedelta(days=day_diff)
         for notification_type in notification_types:
             print('create_nightly_notification_status_for_day triggered for {} and {}'.format(

tests/app/celery/test_reporting_tasks.py

@@ -50,21 +50,31 @@ def test_create_nightly_billing_triggers_tasks_for_days(notify_api, mocker, day_
 
 
 @freeze_time('2019-08-01')
-@pytest.mark.parametrize('day_start, expected_kwargs', [
-    (None, ['2019-07-31', '2019-07-30', '2019-07-29', '2019-07-28']),
-    ('2019-07-21', ['2019-07-21', '2019-07-20', '2019-07-19', '2019-07-18']),
-])
-def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, mocker, day_start, expected_kwargs):
+def test_create_nightly_notification_status_triggers_tasks_for_days(notify_api, mocker):
     mock_celery = mocker.patch('app.celery.reporting_tasks.create_nightly_notification_status_for_day')
-    create_nightly_notification_status(day_start)
+    create_nightly_notification_status()
 
-    assert mock_celery.apply_async.call_count == 4 * 3  # four days, three notification types
-    for process_date, notification_type in itertools.product(expected_kwargs, ['sms', 'email', 'letter']):
+    assert mock_celery.apply_async.call_count == (
+        (4 * 3)  # four days, three notification types
+        +
+        6  # six more days of just letters
+    )
+
+    for process_date, notification_type in itertools.product(
+        ['2019-07-31', '2019-07-30', '2019-07-29', '2019-07-28'],
+        [SMS_TYPE, EMAIL_TYPE, LETTER_TYPE]
+    ):
         mock_celery.apply_async.assert_any_call(
             kwargs={'process_day': process_date, 'notification_type': notification_type},
             queue=QueueNames.REPORTING
         )
 
+    for process_date in ['2019-07-27', '2019-07-26', '2019-07-25', '2019-07-24', '2019-07-23', '2019-07-22']:
+        mock_celery.apply_async.assert_any_call(
+            kwargs={'process_day': process_date, 'notification_type': LETTER_TYPE},
+            queue=QueueNames.REPORTING
+        )
+
 
 @pytest.mark.parametrize('second_rate, records_num, billable_units, multiplier',
                          [(1.0, 1, 2, [1]),