Need magic PII-free debugging method for API

app/celery/tasks.py

@@ -23,7 +23,7 @@ from app.notifications.process_notifications import persist_notification
 from app.notifications.validators import check_service_over_total_message_limit
 from app.serialised_models import SerialisedService, SerialisedTemplate
 from app.service.utils import service_allowed_to_send_to
-from app.utils import DATETIME_FORMAT, hilite, scrub, utc_now
+from app.utils import DATETIME_FORMAT, hilite, utc_now
 from app.v2.errors import TotalRequestsError
 from notifications_utils.recipients import RecipientCSV
 
@@ -189,7 +189,9 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i
     # Return False when trial mode services try sending notifications
     # to non-team and non-simulated recipients.
     if not service_allowed_to_send_to(notification["to"], service, KeyType.NORMAL):
-        current_app.logger.info(hilite(scrub(f"service not allowed to send to {notification['to']}, aborting")))
+        current_app.logger.info(
+            hilite(f"service not allowed to send to {notification['to']}, aborting")
+        )
         current_app.logger.debug(
             "SMS {} failed as restricted service".format(notification_id)
         )
@@ -220,7 +222,9 @@ def save_sms(self, service_id, notification_id, encrypted_notification, sender_i
         )
 
         # Kick off sns process in provider_tasks.py
-        current_app.logger.info(hilite(scrub(f"Going to deliver sms for recipient: {notification['to']}")))
+        current_app.logger.info(
+            hilite(f"Going to deliver sms for recipient: {notification['to']}")
+        )
         provider_tasks.deliver_sms.apply_async(
             [str(saved_notification.id)], queue=QueueNames.SEND_SMS
         )

app/clients/cloudwatch/aws_cloudwatch.py

@@ -9,7 +9,7 @@ from flask import current_app
 from app.clients import AWS_CLIENT_CONFIG, Client
 from app.cloudfoundry_config import cloud_config
 from app.exceptions import NotificationTechnicalFailureException
-from app.utils import hilite, scrub, utc_now
+from app.utils import hilite, utc_now
 
 
 class AwsCloudwatchClient(Client):
@@ -124,7 +124,7 @@ class AwsCloudwatchClient(Client):
             self.warn_if_dev_is_opted_out(
                 message["delivery"]["providerResponse"], notification_id
             )
-            current_app.logger.info(hilite(scrub(f"DELIVERED: {message}")))
+            current_app.logger.info(hilite(f"DELIVERED: {message}"))
             return (
                 "success",
                 message["delivery"]["providerResponse"],
@@ -142,7 +142,7 @@ class AwsCloudwatchClient(Client):
                 message["delivery"]["providerResponse"], notification_id
             )
 
-            current_app.logger.info(hilite(scrub(f"FAILED: {message}")))
+            current_app.logger.info(hilite(f"FAILED: {message}"))
             return (
                 "failure",
                 message["delivery"]["providerResponse"],

app/delivery/send_to_providers.py

@@ -13,7 +13,7 @@ from app.dao.provider_details_dao import get_provider_details_by_notification_ty
 from app.enums import BrandType, KeyType, NotificationStatus, NotificationType
 from app.exceptions import NotificationTechnicalFailureException
 from app.serialised_models import SerialisedService, SerialisedTemplate
-from app.utils import hilite, scrub, utc_now
+from app.utils import hilite, utc_now
 from notifications_utils.template import (
     HTMLEmailTemplate,
     PlainTextEmailTemplate,
@@ -114,14 +114,14 @@ def send_sms_to_provider(notification):
                 current_app.logger.info(f"got message_id {message_id}")
             except Exception as e:
                 msg = f"FAILED sending message for this recipient: {recipient} to sms"
-                current_app.logger.error(hilite(scrub(f"{msg} {e}")))
+                current_app.logger.error(hilite(f"{msg} {e}"))
 
                 notification.billable_units = template.fragment_count
                 dao_update_notification(notification)
                 raise e
             else:
                 msg = f"Sending message for this recipient: {recipient} to sms"
-                current_app.logger.info(hilite(scrub(msg)))
+                current_app.logger.info(hilite(msg))
                 notification.billable_units = template.fragment_count
                 update_notification_to_sending(notification, provider)
     return message_id

app/notifications/process_notifications.py

@@ -11,7 +11,7 @@ from app.dao.notifications_dao import (
 )
 from app.enums import KeyType, NotificationStatus, NotificationType
 from app.models import Notification
-from app.utils import hilite, scrub, utc_now
+from app.utils import hilite, utc_now
 from app.v2.errors import BadRequestError
 from notifications_utils.recipients import (
     format_email_address,
@@ -110,7 +110,9 @@ def persist_notification(
         formatted_recipient = validate_and_format_phone_number(
             recipient, international=True
         )
-        current_app.logger.info(hilite(scrub(f"Persisting notification with recipient {formatted_recipient}")))
+        current_app.logger.info(
+            hilite(f"Persisting notification with recipient {formatted_recipient}")
+        )
         recipient_info = get_international_phone_info(formatted_recipient)
         notification.normalised_to = formatted_recipient
         notification.international = recipient_info.international

app/utils.py

@@ -1,4 +1,3 @@
-import re
 from datetime import datetime, timedelta, timezone
 
 from flask import url_for
@@ -145,15 +144,3 @@ def naive_utcnow():
 
 def utc_now():
     return naive_utcnow()
-
-
-def scrub(msg):
-    # Eventually we want to scrub all messages in all logs for phone numbers
-    # and email addresses, masking them.  Ultimately this will probably get
-    # refactored into a 'SafeLogger' subclass or something, but let's start here
-    # with phones.
-    phones = re.findall("(?:\\+ *)?\\d[\\d\\- ]{7,}\\d", msg)
-    phones = [phone.replace("-", "").replace(" ", "") for phone in phones]
-    for phone in phones:
-        msg = msg.replace(phone, f"1XXXXX{phone[-5:]}")
-    return msg