Merge pull request #707 from GSA/notify-api-685f

get phone numbers from the jobs in s3

app/aws/s3.py

@@ -1,5 +1,8 @@
+import re
+
 import botocore
 from boto3 import Session
+from expiringdict import ExpiringDict
 from flask import current_app
 
 from app.clients import AWS_CLIENT_CONFIG
@@ -7,6 +10,9 @@ from app.clients import AWS_CLIENT_CONFIG
 FILE_LOCATION_STRUCTURE = "service-{}-notify/{}.csv"
 
 
+JOBS = ExpiringDict(max_len=100, max_age_seconds=3600 * 4)
+
+
 def get_s3_file(bucket_name, file_location, access_key, secret_key, region):
     s3_file = get_s3_object(bucket_name, file_location, access_key, secret_key, region)
     return s3_file.get()["Body"].read().decode("utf-8")
@@ -66,6 +72,31 @@ def get_job_from_s3(service_id, job_id):
     return obj.get()["Body"].read().decode("utf-8")
 
 
+def get_phone_number_from_s3(service_id, job_id, job_row_number):
+    # We don't want to constantly pull down a job from s3 every time we need a phone number.
+    # At the same time we don't want to store it in redis or the db
+    # So this is a little recycling mechanism to reduce the number of downloads.
+    job = JOBS.get(job_id)
+    if job is None:
+        job = get_job_from_s3(service_id, job_id)
+        JOBS[job_id] = job
+    job = job.split("\r\n")
+    first_row = job[0]
+    job.pop(0)
+    first_row = first_row.split(",")
+    phone_index = 0
+    for item in first_row:
+        if item == "phone number":
+            break
+        phone_index = phone_index + 1
+    correct_row = job[job_row_number]
+    correct_row = correct_row.split(",")
+
+    my_phone = correct_row[phone_index]
+    my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
+    return my_phone
+
+
 def get_job_metadata_from_s3(service_id, job_id):
     obj = get_s3_object(*get_job_location(service_id, job_id))
     return obj.get()["Metadata"]

app/delivery/send_to_providers.py

@@ -9,7 +9,8 @@ from notifications_utils.template import (
     SMSMessageTemplate,
 )
 
-from app import create_uuid, db, notification_provider_clients
+from app import create_uuid, db, notification_provider_clients, redis_store
+from app.aws.s3 import get_phone_number_from_s3
 from app.celery.test_key_tasks import send_email_response, send_sms_response
 from app.dao.email_branding_dao import dao_get_email_branding_by_id
 from app.dao.notifications_dao import dao_update_notification
@@ -65,8 +66,29 @@ def send_sms_to_provider(notification):
                 # providers as a slow down of our providers can cause us to run out of DB connections
                 # Therefore we pull all the data from our DB models into `send_sms_kwargs`now before
                 # closing the session (as otherwise it would be reopened immediately)
+
+                # We start by trying to get the phone number from a job in s3.  If we fail, we assume
+                # the phone number is for the verification code on login, which is not a job.
+                my_phone = None
+                try:
+                    my_phone = get_phone_number_from_s3(
+                        notification.service_id,
+                        notification.job_id,
+                        notification.job_row_number,
+                    )
+                except BaseException:
+                    my_phone = redis_store.get(f"2facode_{notification.id}")
+                    if my_phone:
+                        my_phone = my_phone.decode("utf-8")
+                if my_phone is None:
+                    si = notification.service_id
+                    ji = notification.job_id
+                    jrn = notification.job_row_number
+                    raise Exception(
+                        f"The phone number for (Service ID: {si}; Job ID: {ji}; Job Row Number {jrn} was not found."
+                    )
                 send_sms_kwargs = {
-                    "to": notification.normalised_to,
+                    "to": my_phone,
                     "content": str(template),
                     "reference": str(notification.id),
                     "sender": notification.reply_to_text,
@@ -101,7 +123,7 @@ def send_email_to_provider(notification):
         html_email = HTMLEmailTemplate(
             template_dict,
             values=notification.personalisation,
-            **get_html_email_options(service)
+            **get_html_email_options(service),
         )
 
         plain_text_email = PlainTextEmailTemplate(

app/user/rest.py

@@ -7,6 +7,7 @@ from flask import Blueprint, abort, current_app, jsonify, request
 from notifications_utils.recipients import is_us_phone_number, use_numeric_sender
 from sqlalchemy.exc import IntegrityError
 
+from app import redis_store
 from app.config import QueueNames
 from app.dao.permissions_dao import permission_dao
 from app.dao.service_user_dao import dao_get_service_user, dao_update_service_user
@@ -337,6 +338,7 @@ def create_2fa_code(
         reply_to = get_sms_reply_to_for_notify_service(recipient, template)
     elif template.template_type == EMAIL_TYPE:
         reply_to = template.service.get_default_reply_to_email_address()
+
     saved_notification = persist_notification(
         template_id=template.id,
         template_version=template.version,
@@ -348,6 +350,8 @@ def create_2fa_code(
         key_type=KEY_TYPE_NORMAL,
         reply_to_text=reply_to,
     )
+
+    redis_store.set(f"2facode_{saved_notification.id}", recipient, ex=1800)
     # Assume that we never want to observe the Notify service's research mode
     # setting for this notification - we still need to be able to log into the
     # admin even if we're doing user research using this service: