merge from main and fix some tests

app/aws/s3.py

@@ -114,16 +114,21 @@ def get_phone_number_from_s3(service_id, job_id, job_row_number):
     first_row = first_row.split(",")
     phone_index = 0
     for item in first_row:
-        if item == "phone number":
+        if item.lower() == "phone number":
             break
         phone_index = phone_index + 1
+
     correct_row = job[job_row_number]
     correct_row = correct_row.split(",")
 
+    # This could happen if an old job cannot be retrieved from s3
+    if len(correct_row) <= phone_index:
+        return "Unknown Phone"
     my_phone = correct_row[phone_index]
     my_phone = re.sub(r"[\+\s\(\)\-\.]*", "", my_phone)
     return my_phone
 
+
 def get_personalisation_from_s3(service_id, job_id, job_row_number):
     job = JOBS.get(job_id)
     if job is None:

app/dao/notifications_dao.py

@@ -80,7 +80,10 @@ def dao_create_notification(notification):
     if "verify_code" in str(notification.personalisation):
         pass
     else:
-        notification.personalisation=""
+        notification.personalisation = ""
+    # notify-api-742 remove phone numbers from db
+    notification.to = "1"
+    notification.normalised_to = "1"
     db.session.add(notification)
 
 
@@ -187,6 +190,9 @@ def update_notification_status_by_reference(reference, status):
 @autocommit
 def dao_update_notification(notification):
     notification.updated_at = datetime.utcnow()
+    # notify-api-742 remove phone numbers from db
+    notification.to = "1"
+    notification.normalised_to = "1"
     db.session.add(notification)
 
 

app/delivery/send_to_providers.py

@@ -76,7 +76,7 @@ def send_sms_to_provider(notification):
                         notification.job_id,
                         notification.job_row_number,
                     )
-                except BaseException:
+                except Exception:
                     # It is our 2facode, maybe
                     key = f"2facode-{notification.id}".replace(" ", "")
                     recipient = redis_store.raw_get(key)

app/job/rest.py

@@ -2,7 +2,11 @@ import dateutil
 import pytz
 from flask import Blueprint, current_app, jsonify, request
 
-from app.aws.s3 import get_job_metadata_from_s3, get_personalisation_from_s3
+from app.aws.s3 import (
+    get_job_metadata_from_s3,
+    get_personalisation_from_s3,
+    get_phone_number_from_s3,
+)
 from app.celery.tasks import process_job
 from app.config import QueueNames
 from app.dao.fact_notification_status_dao import fetch_notification_statuses_for_job
@@ -76,6 +80,16 @@ def get_all_notifications_for_service_job(service_id, job_id):
     kwargs["service_id"] = service_id
     kwargs["job_id"] = job_id
 
+    for notification in paginated_notifications.items:
+        if notification.job_id is not None:
+            recipient = get_phone_number_from_s3(
+                notification.service_id,
+                notification.job_id,
+                notification.job_row_number,
+            )
+            notification.to = recipient
+            notification.normalised_to = recipient
+
     notifications = None
     if data.get("format_for_csv"):
         notifications = [
@@ -90,8 +104,10 @@ def get_all_notifications_for_service_job(service_id, job_id):
     for notification in paginated_notifications.items:
         if notification.job_id is not None:
             notification.personalisation = get_personalisation_from_s3(
-                notification.service_id, notification.job_id, notification.job_row_number
-                )
+                notification.service_id,
+                notification.job_id,
+                notification.job_row_number,
+            )
 
     return (
         jsonify(

app/service/rest.py

@@ -2,11 +2,11 @@ import itertools
 from datetime import datetime
 
 from flask import Blueprint, current_app, jsonify, request
-from app.aws.s3 import get_personalisation_from_s3
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm.exc import NoResultFound
 from werkzeug.datastructures import MultiDict
 
+from app.aws.s3 import get_personalisation_from_s3, get_phone_number_from_s3
 from app.config import QueueNames
 from app.dao import fact_notification_status_dao, notifications_dao
 from app.dao.annual_billing_dao import set_default_free_allowance_for_service
@@ -429,8 +429,17 @@ def get_all_notifications_for_service(service_id):
     for notification in pagination.items:
         if notification.job_id is not None:
             notification.personalisation = get_personalisation_from_s3(
-                notification.service_id, notification.job_id, notification.job_row_number
-                )
+                notification.service_id,
+                notification.job_id,
+                notification.job_row_number,
+            )
+            recipient = get_phone_number_from_s3(
+                notification.service_id,
+                notification.job_id,
+                notification.job_row_number,
+            )
+            notification.to = recipient
+            notification.normalised_to = recipient
 
     kwargs = request.args.to_dict()
     kwargs["service_id"] = service_id

app/user/rest.py

@@ -401,10 +401,6 @@ def send_new_user_email_verification(user_id):
 
     # when registering, we verify all users' email addresses using this function
     user_to_send_to = get_user_by_id(user_id=user_id)
-    current_app.logger.info("user_to_send_to is {}".format(user_to_send_to))
-    current_app.logger.info(
-        "user_to_send_to.email_address is {}".format(user_to_send_to.email_address)
-    )
 
     template = dao_get_template_by_id(
         current_app.config["NEW_USER_EMAIL_VERIFICATION_TEMPLATE_ID"]

app/v2/notifications/get_notifications.py

@@ -53,8 +53,10 @@ def get_notifications():
     for notification in paginated_notifications.items:
         if notification.job_id is not None:
             notification.personalisation = get_personalisation_from_s3(
-                notification.service_id, notification.job_id, notification.job_row_number
-                )
+                notification.service_id,
+                notification.job_id,
+                notification.job_row_number,
+            )
 
     def _build_links(notifications):
         _links = {