final

app/aws/s3.py

@@ -1,5 +1,6 @@
 import botocore
 from boto3 import Session
+from expiringdict import ExpiringDict
 from flask import current_app
 
 from app.clients import AWS_CLIENT_CONFIG
@@ -7,6 +8,9 @@ from app.clients import AWS_CLIENT_CONFIG
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
 
 
+JOBS = ExpiringDict(max_len=100, max_age_seconds=3600)
+
+
 def get_s3_file(bucket_name, file_location, access_key, secret_key, region):
     s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
     return s3_file.get()["Body"].read().decode("utf-8")
@@ -62,10 +66,51 @@ def get_job_and_metadata_from_s3(service_id, job_id):
 
 
 def get_job_from_s3(service_id, job_id):
+    print(f"ENTERE get_job_from_s3 with {service_id} {job_id}")
     obj = get_s3_object(*get_job_location(service_id, job_id))
+    print(f"obj is {obj}")
     return obj.get()["Body"].read().decode("utf-8")
 
 
+def get_phone_number_from_s3(service_id, job_id, job_row_number):
+    # We don't want to constantly pull down a job from s3 every time we need a phone number.
+    # At the same time we don't want to store it in redis or the db
+    # So this is a little recycling mechanism to reduce the number of downloads.
+    print(
+        f"ENTER GET_PHONE_NUMBER_FROM_S3 WITH JOB_ID {job_id} SERVICE_ID {service_id} JOB_ROW_NUMBER {job_row_number}"
+    )
+    job = JOBS.get(job_id)
+    print(f"JOB FROM EXPIRINGDICT {job} JOB ID {job_id}")
+    if job is None:
+        print("JOB IS NONE")
+        job = get_job_from_s3(service_id, job_id)
+        print(f"FRESH JOB {job}")
+        JOBS[job_id] = job
+        print("ADDED {job} to EXPRING DICT")
+    else:
+        print("JOB IS NOT NONE")
+    job = job.split("\r\n")
+    print(f"JOB AFTER SPLIT {job}")
+    first_row = job[0]
+    job.pop(0)
+    first_row = first_row.split(",")
+    phone_index = 0
+    for item in first_row:
+        if item == "phone number":
+            break
+        phone_index = phone_index + 1
+    correct_row = job[job_row_number]
+    correct_row = correct_row.split(",")
+
+    my_phone = correct_row[phone_index]
+    my_phone = my_phone.replace("(", "")
+    my_phone = my_phone.replace(")", "")
+    my_phone = my_phone.replace("-", "")
+    my_phone = my_phone.replace(" ", "")
+    my_phone = my_phone.replace("+", "")
+    return my_phone
+
+
 def get_job_metadata_from_s3(service_id, job_id):
     obj = get_s3_object(*get_job_location(service_id, job_id))
     return obj.get()["Metadata"]

app/celery/provider_tasks.py

@@ -31,7 +31,7 @@ DELIVERY_RECEIPT_DELAY_IN_SECONDS = 120
 @notify_celery.task(
     bind=True,
     name="check_sms_delivery_receipt",
-    max_retries=48,
+    max_retries=12,
     default_retry_delay=300,
 )
 def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
@@ -92,7 +92,7 @@ def check_sms_delivery_receipt(self, message_id, notification_id, sent_at):
 
 
 @notify_celery.task(
-    bind=True, name="deliver_sms", max_retries=48, default_retry_delay=300
+    bind=True, name="deliver_sms", max_retries=12, default_retry_delay=300
 )
 def deliver_sms(self, notification_id):
     try:

app/delivery/send_to_providers.py

@@ -9,7 +9,8 @@ from notifications_utils.template import (
     SMSMessageTemplate,
 )
 
-from app import create_uuid, db, notification_provider_clients
+from app import create_uuid, db, notification_provider_clients, redis_store
+from app.aws.s3 import get_phone_number_from_s3
 from app.celery.test_key_tasks import send_email_response, send_sms_response
 from app.dao.email_branding_dao import dao_get_email_branding_by_id
 from app.dao.notifications_dao import dao_update_notification
@@ -65,8 +66,26 @@ def send_sms_to_provider(notification):
                 # providers as a slow down of our providers can cause us to run out of DB connections
                 # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
                 # closing the session (as otherwise it would be reopened immediately)
+
+                # We start by trying to get the phone number from a job in s3.  If we fail, we assume
+                # the phone number is for the verification code on login, which is not a job.
+                my_phone = None
+                try:
+                    my_phone = get_phone_number_from_s3(
+                        notification.service_id,
+                        notification.job_id,
+                        notification.job_row_number,
+                    )
+                    print(f"MY PHONE FROM JOB {my_phone}")
+                except BaseException:
+                    my_phone = redis_store.get(f"2facode_{notification.id}")
+                    if my_phone:
+                        my_phone = my_phone.decode("utf-8")
+                        print(f"MY PHONE FROM VERIFY CODE {my_phone}")
+                if my_phone is None:
+                    raise Exception("what happened to the phone number")
                 send_sms_kwargs = {
-                    "to": notification.normalised_to,
+                    "to": my_phone,
                     "content": str(template),
                     "reference": str(notification.id),
                     "sender": notification.reply_to_text,
@@ -101,7 +120,7 @@ def send_email_to_provider(notification):
         html_email = HTMLEmailTemplate(
             template_dict,
             values=notification.personalisation,
-            **get_html_email_options(service)
+            **get_html_email_options(service),
         )
 
         plain_text_email = PlainTextEmailTemplate(

app/user/rest.py

@@ -7,6 +7,7 @@ from flask import Blueprint, abort, current_app, jsonify, request
 from notifications_utils.recipients import is_us_phone_number, use_numeric_sender
 from sqlalchemy.exc import IntegrityError
 
+from app import redis_store
 from app.config import QueueNames
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_user, dao_update_service_user
@@ -337,6 +338,7 @@ def create_2fa_code(
         reply_to = get_sms_reply_to_for_notify_service(recipient, template)
     elif template.template_type == EMAIL_TYPE:
         reply_to = template.service.get_default_reply_to_email_address()
+
     saved_notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
@@ -348,6 +350,8 @@ def create_2fa_code(
         key_type=KEY_TYPE_NORMAL,
         reply_to_text=reply_to,
     )
+
+    redis_store.set(f"2facode_{saved_notification.id}", recipient, ex=1800)
     # Assume that we never want to observe the Notify service's research mode
     # setting for this notification - we still need to be able to log into the
     # admin even if we're doing user research using this service: