In this PR we remove trigger-letter-pdfs-for-day scheduled task and just call collate_letter_pdfs_for_day instead.

There was a datetime bug in the query which resulted in files not being sent to the postal provider.
The trigger-letter-pdfs-for-day task is no longer needed, so rather than fix the query just call collate_letter_pdfs_for_day directly.
Less code is always better.

Deployment considerations: I realized this is strictly not backwards compatible if the scheduled job is in progress and a task is on the queue that no longer exists. This is ok since we will deploy this well before 17:50.

app/celery/letters_pdf_tasks.py

@@ -105,7 +105,12 @@ def get_letters_pdf(template, contact_block, org_id, values):
 
 
 @notify_celery.task(name='collate-letter-pdfs-for-day')
-def collate_letter_pdfs_for_day(date):
+def collate_letter_pdfs_for_day(date=None):
+    if not date:
+        # Using the truncated date is ok because UTC to BST does not make a difference to the date,
+        # since it is triggered mid afternoon.
+        date = datetime.utcnow().strftime("%Y-%m-%d")
+
     letter_pdfs = s3.get_s3_bucket_objects(
         current_app.config['LETTERS_PDF_BUCKET_NAME'],
         subfolder=date

app/celery/scheduled_tasks.py

@@ -1,5 +1,4 @@
 from datetime import (
-    date,
     datetime,
     timedelta
 )
@@ -32,7 +31,6 @@ from app.dao.notifications_dao import (
     dao_timeout_notifications,
     is_delivery_slow_for_provider,
     delete_notifications_created_more_than_a_week_ago_by_type,
-    dao_get_count_of_letters_to_process_for_date,
     dao_get_scheduled_notifications,
     set_scheduled_notification_to_processed,
     notifications_not_yet_sent
@@ -385,20 +383,6 @@ def run_letter_jobs():
         current_app.logger.info("Queued {} ready letter job ids onto {}".format(len(job_ids), QueueNames.PROCESS_FTP))
 
 
-@notify_celery.task(name="trigger-letter-pdfs-for-day")
-@statsd(namespace="tasks")
-def trigger_letter_pdfs_for_day():
-    letter_pdfs_count = dao_get_count_of_letters_to_process_for_date()
-    if letter_pdfs_count:
-        notify_celery.send_task(
-            name='collate-letter-pdfs-for-day',
-            args=(date.today().strftime("%Y-%m-%d"),),
-            queue=QueueNames.LETTERS
-        )
-    current_app.logger.info("{} letter pdfs to be process by {} task".format(
-        letter_pdfs_count, 'collate-letter-pdfs-for-day'))
-
-
 @notify_celery.task(name='check-job-status')
 @statsd(namespace="tasks")
 def check_job_status():

app/config.py

@@ -242,8 +242,10 @@ class Config(object):
             'schedule': crontab(hour=16, minute=30),
             'options': {'queue': QueueNames.PERIODIC}
         },
-        'trigger-letter-pdfs-for-day': {
-            'task': 'trigger-letter-pdfs-for-day',
+        # The collate-letter-pdf does assume it is called in an hour that BST does not make a
+        # difference to the truncate date which translates to the filename to process
+        'collate-letter-pdfs-for-day': {
+            'task': 'collate-letter-pdfs-for-day',
             'schedule': crontab(hour=17, minute=50),
             'options': {'queue': QueueNames.PERIODIC}
         },

app/dao/notifications_dao.py

@@ -3,7 +3,6 @@ import string
 from datetime import (
     datetime,
     timedelta,
-    date
 )
 
 from boto.exception import BotoClientError
@@ -31,7 +30,6 @@ from app.models import (
     Notification,
     NotificationHistory,
     ScheduledNotification,
-    Service,
     Template,
     TemplateHistory,
     KEY_TYPE_TEST,
@@ -607,37 +605,6 @@ def dao_get_last_notification_added_for_job_id(job_id):
     return last_notification_added
 
 
-def dao_get_count_of_letters_to_process_for_date(date_to_process=None):
-    """
-    Returns a count of letter notifications to be processed today if no
-    argument passed in otherwise will return the count for the date passed in.
-    Records processed today = yesterday 17:30 to today 17:29:59
-
-    Note - services in research mode are ignored
-    """
-    if date_to_process is None:
-        date_to_process = date.today()
-
-    day_before = date_to_process - timedelta(days=1)
-    letter_deadline_time = current_app.config.get('LETTER_PROCESSING_DEADLINE')
-
-    start_datetime = datetime.combine(day_before, letter_deadline_time)
-    end_datetime = start_datetime + timedelta(days=1)
-
-    count_of_letters_to_process_for_date = Notification.query.join(
-        Service
-    ).filter(
-        Notification.created_at >= start_datetime,
-        Notification.created_at < end_datetime,
-        Notification.notification_type == LETTER_TYPE,
-        Notification.status == NOTIFICATION_CREATED,
-        Notification.key_type != KEY_TYPE_TEST,
-        Service.research_mode.is_(False)
-    ).count()
-
-    return count_of_letters_to_process_for_date
-
-
 def notifications_not_yet_sent(should_be_sending_after_seconds, notification_type):
     older_than_date = datetime.utcnow() - timedelta(seconds=should_be_sending_after_seconds)