merge from main

app/__init__.py

@@ -1,3 +1,4 @@
+import logging as real_logging
 import os
 import secrets
 import string
@@ -36,6 +37,9 @@ class NotifyCelery(Celery):
 
         # Configure Celery app with options from the main app config.
         self.config_from_object(app.config["CELERY"])
+        self.conf.worker_hijack_root_logger = False
+        logger = real_logging.getLogger("celery")
+        logger.propagate = False
 
     def send_task(self, name, args=None, kwargs=None, **other_kwargs):
         other_kwargs["headers"] = other_kwargs.get("headers") or {}

app/aws/s3.py

@@ -402,7 +402,12 @@ def extract_phones(job):
     phone_index = 0
     for item in first_row:
         # Note: may contain a BOM and look like \ufeffphone number
-        if item.lower() in ["phone number", "\\ufeffphone number"]:
+        if item.lower() in [
+            "phone number",
+            "\\ufeffphone number",
+            "\\ufeffphone number\n",
+            "phone number\n",
+        ]:
             break
         phone_index = phone_index + 1
 

app/celery/tasks.py

@@ -1,5 +1,6 @@
 import json
 
+from celery.signals import task_postrun
 from flask import current_app
 from requests import HTTPError, RequestException, request
 from sqlalchemy.exc import IntegrityError, SQLAlchemyError
@@ -170,6 +171,13 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id):
         return True
 
 
+@task_postrun.connect
+def log_task_ejection(sender=None, task_id=None, **kwargs):
+    current_app.logger.info(
+        f"Task {task_id} ({sender.name if sender else 'unknown_task'}) has been completed and removed"
+    )
+
+
 @notify_celery.task(bind=True, name="save-sms", max_retries=2, default_retry_delay=600)
 def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None):
     """Persist notification to db and place notification in queue to send to sns."""

app/config.py

@@ -168,6 +168,8 @@ class Config(object):
 
     CELERY = {
         "worker_max_tasks_per_child": 500,
+        "task_ignore_result": True,
+        "result_persistent": False,
         "broker_url": REDIS_URL,
         "broker_transport_options": {
             "visibility_timeout": 310,

notifications_utils/logging.py

@@ -70,10 +70,12 @@ def init_app(app):
     for logger_instance, handler in product(loggers, handlers):
         logger_instance.addHandler(handler)
         logger_instance.setLevel(loglevel)
+        logger_instance.propagate = False
     warning_loggers = [logging.getLogger("boto3"), logging.getLogger("s3transfer")]
     for logger_instance, handler in product(warning_loggers, handlers):
         logger_instance.addHandler(handler)
         logger_instance.setLevel(logging.WARNING)
+        logger_instance.propagate = False
 
     # Suppress specific loggers to prevent leaking sensitive info
     logging.getLogger("boto3").setLevel(logging.ERROR)

tests/app/aws/test_s3.py

@@ -219,6 +219,20 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker):
             2,
             "5555555552",
         ),
+        (
+            # simulate file saved with utf8withbom
+            "\\ufeffPHONE NUMBER\n",
+            "eee",
+            2,
+            "5555555552",
+        ),
+        (
+            # simulate file saved without utf8withbom
+            "\\PHONE NUMBER\n",
+            "eee",
+            2,
+            "5555555552",
+        ),
     ],
 )
 def test_get_phone_number_from_s3(