From f6d210f1e6e6ea6d3ffe35c338821954bbebd18d Mon Sep 17 00:00:00 2001 From: Leo Hemsted Date: Fri, 3 Dec 2021 13:28:16 +0000 Subject: [PATCH] put delete tasks on the reporting worker they share a lot with the reporting tasks (creating ft_billing and ft_notification_status), in that they're run nightly, take a long time, and we see error messages if they get run multiple times (due to visibility timeout). The periodic app has two concurrent processes - previously there was just one delete task, which would use one of those processes, while the other process would pick up anything else on the queue (at that time of night, the regular provider switch checks and scheduled job checks). However, when we switched to running the three delete notification types separately, we saw visibility timeout issues - three tasks would be created, all three would be picked up by one celery instance, the two worker processes would start on two of them, and the third would sit on the box, wait longer than the visibility timeout to be picked up (and acknowledged), and so SQS would assume the task was lost and replay it. it's queues all the way down! By putting them on the reporting worker we can take advantage of tuning that app (for example setting the prefetch multiplier to one) which is designed to run large tasks. We've also got more concurrent workers on this box, so we can run all three tasks at once. --- app/celery/nightly_tasks.py | 6 +++--- app/config.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/celery/nightly_tasks.py b/app/celery/nightly_tasks.py index ae8c68992..65b9a94d8 100644 --- a/app/celery/nightly_tasks.py +++ b/app/celery/nightly_tasks.py @@ -65,9 +65,9 @@ def _remove_csv_files(job_types): @notify_celery.task(name="delete-notifications-older-than-retention") def delete_notifications_older_than_retention(): - delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.PERIODIC) - delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.PERIODIC) - delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.PERIODIC) + delete_email_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING) + delete_sms_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING) + delete_letter_notifications_older_than_retention.apply_async(queue=QueueNames.REPORTING) @notify_celery.task(name="delete-sms-notifications") diff --git a/app/config.py b/app/config.py index 177e78113..ab5a08b27 100644 --- a/app/config.py +++ b/app/config.py @@ -267,7 +267,7 @@ class Config(object): 'delete-notifications-older-than-retention': { 'task': 'delete-notifications-older-than-retention', 'schedule': crontab(hour=3, minute=0), # after 'create-nightly-notification-status' - 'options': {'queue': QueueNames.PERIODIC} + 'options': {'queue': QueueNames.REPORTING} }, 'delete-inbound-sms': { 'task': 'delete-inbound-sms',