fix personalization_bug for one-offs

app/aws/s3.py

@@ -366,7 +366,6 @@ def extract_phones(job):
 
 
 def extract_personalisation(job):
-
     job = job[0].split("\r\n")
     first_row = job[0]
     job.pop(0)
@@ -398,11 +397,8 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
         )
         return "Unavailable"
 
-    # If we look in the job_cache for the quick lookup dictionary of phones for a given job
-    # and that dictionary is not there, create it
-    if job_cache.get(f"{job_id}_phones") is None:
-        phones = extract_phones(job)
-        set_job_cache(job_cache, f"{job_id}_phones", phones)
+    phones = extract_phones(job)
+    set_job_cache(job_cache, f"{job_id}_phones", phones)
 
     # If we can find the quick dictionary, use it
     phone_to_return = phones[job_row_number]
@@ -420,9 +416,6 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
     # At the same time we don't want to store it in redis or the db
     # So this is a little recycling mechanism to reduce the number of downloads.
     job = job_cache.get(job_id)
-    if job is None:
-        job = get_job_from_s3(service_id, job_id)
-        set_job_cache(job_cache, job_id, job)
 
     # If the job is None after our attempt to retrieve it from s3, it
     # probably means the job is old and has been deleted from s3, in
@@ -435,12 +428,7 @@ def get_personalisation_from_s3(service_id, job_id, job_row_number):
         )
         return {}
 
-    # If we look in the job_cache for the quick lookup dictionary of personalisations for a given job
-    # and that dictionary is not there, create it
-    if job_cache.get(f"{job_id}_personalisation") is None:
-        set_job_cache(
-            job_cache, f"{job_id}_personalisation", extract_personalisation(job)
-        )
+    set_job_cache(job_cache, f"{job_id}_personalisation", extract_personalisation(job))
 
     # If we can find the quick dictionary, use it
     if job_cache.get(f"{job_id}_personalisation") is not None:

app/delivery/send_to_providers.py

@@ -13,7 +13,11 @@ from app import (
     notification_provider_clients,
     redis_store,
 )
-from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
+from app.aws.s3 import (
+    get_job_from_s3,
+    get_personalisation_from_s3,
+    get_phone_number_from_s3,
+)
 from app.celery.test_key_tasks import send_email_response, send_sms_response
 from app.dao.email_branding_dao import dao_get_email_branding_by_id
 from app.dao.notifications_dao import dao_update_notification
@@ -36,17 +40,32 @@ def send_sms_to_provider(notification):
     Get data for recipient, template,
     notification and send it to sns.
     """
-    # we no longer store the personalisation in the db,
-    # need to retrieve from s3 before generating content
-    # However, we are still sending the initial verify code through personalisation
-    # so if there is some value there, don't overwrite it
-    if not notification.personalisation:
-        personalisation = get_personalisation_from_s3(
-            notification.service_id,
-            notification.job_id,
-            notification.job_row_number,
-        )
-        notification.personalisation = personalisation
+    # Take this path for report generation, where we know
+    # everything is in the cache.
+    personalisation = get_personalisation_from_s3(
+        notification.service_id,
+        notification.job_id,
+        notification.job_row_number,
+    )
+
+    # For one-off sends, they may not get into the cache
+    # by the time we get here, so do this slow direct-from-s3
+    # approach.  It is okay to be slow, since one-offs have
+    # to be typed in by hand.
+    if personalisation == {}:
+        job = get_job_from_s3(notification.service_id, notification.job_id)
+        job = job.split("\r\n")
+        first_row = job[0]
+        job.pop(0)
+        first_row = first_row.split(",")
+        personalisation = {}
+        job_row = 0
+        for row in job:
+            row = row.split(",")
+            temp = dict(zip(first_row, row))
+            personalisation[job_row] = temp
+            job_row = job_row + 1
+    notification.personalisation = personalisation[notification.job_row_number]
 
     service = SerialisedService.from_id(notification.service_id)
     message_id = None