From 9ad33045dd3b663c1c5c299433dda8ebdd5c3396 Mon Sep 17 00:00:00 2001 From: Chris Hill-Scott Date: Sat, 26 Sep 2020 10:57:21 +0100 Subject: [PATCH] Improve efficiency of process missing rows task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For every missing row this was: - downloading the CSV file from S3 - looping through every row in it until it found the one matching the index of the missing row `RecipientCSV` implements `__getitem__`[1] (which maybe it didn’t before) so we can create it once, then index the relevant row directly. *** 1. https://github.com/alphagov/notifications-utils/blob/5ae0572d4188483fa140bccc128ecf80f3473000/notifications_utils/recipients.py#L78-L79 --- app/celery/scheduled_tasks.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/app/celery/scheduled_tasks.py b/app/celery/scheduled_tasks.py index 4fcac0bbe..c6d8aea5d 100644 --- a/app/celery/scheduled_tasks.py +++ b/app/celery/scheduled_tasks.py @@ -237,14 +237,13 @@ def check_for_missing_rows_in_completed_jobs(): jobs_and_job_size = find_jobs_with_missing_rows() for x in jobs_and_job_size: job = x[1] + recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job) missing_rows = find_missing_row_for_job(job.id, job.notification_count) for row_to_process in missing_rows: - recipient_csv, template, sender_id = get_recipient_csv_and_template_and_sender_id(job) - for row in recipient_csv.get_rows(): - if row.index == row_to_process.missing_row: - current_app.logger.info( - "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id)) - process_row(row, template, job, job.service, sender_id=sender_id) + row = recipient_csv[row_to_process.missing_row] + current_app.logger.info( + "Processing missing row: {} for job: {}".format(row_to_process.missing_row, job.id)) + process_row(row, template, job, job.service, sender_id=sender_id) @notify_celery.task(name='check-for-services-with-high-failure-rates-or-sending-to-tv-numbers')