diff --git a/app/__init__.py b/app/__init__.py index 16ffbd5a9..23c2399e1 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,3 +1,4 @@ +import logging as real_logging import os import secrets import string @@ -36,6 +37,9 @@ class NotifyCelery(Celery): # Configure Celery app with options from the main app config. self.config_from_object(app.config["CELERY"]) + self.conf.worker_hijack_root_logger = False + logger = real_logging.getLogger("celery") + logger.propagate = False def send_task(self, name, args=None, kwargs=None, **other_kwargs): other_kwargs["headers"] = other_kwargs.get("headers") or {} diff --git a/app/aws/s3.py b/app/aws/s3.py index e0022f20b..d97c421f2 100644 --- a/app/aws/s3.py +++ b/app/aws/s3.py @@ -402,7 +402,12 @@ def extract_phones(job): phone_index = 0 for item in first_row: # Note: may contain a BOM and look like \ufeffphone number - if item.lower() in ["phone number", "\\ufeffphone number"]: + if item.lower() in [ + "phone number", + "\\ufeffphone number", + "\\ufeffphone number\n", + "phone number\n", + ]: break phone_index = phone_index + 1 diff --git a/app/celery/tasks.py b/app/celery/tasks.py index b612933ef..c21f4e65c 100644 --- a/app/celery/tasks.py +++ b/app/celery/tasks.py @@ -1,5 +1,6 @@ import json +from celery.signals import task_postrun from flask import current_app from requests import HTTPError, RequestException, request from sqlalchemy.exc import IntegrityError, SQLAlchemyError @@ -170,6 +171,13 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id): return True +@task_postrun.connect +def log_task_ejection(sender=None, task_id=None, **kwargs): + current_app.logger.info( + f"Task {task_id} ({sender.name if sender else 'unknown_task'}) has been completed and removed" + ) + + @notify_celery.task(bind=True, name="save-sms", max_retries=2, default_retry_delay=600) def save_sms(self, service_id, notification_id, encrypted_notification, sender_id=None): """Persist notification to db and place notification in queue to send to sns.""" diff --git a/app/config.py b/app/config.py index e7f4af14d..53a2f9a0d 100644 --- a/app/config.py +++ b/app/config.py @@ -168,6 +168,8 @@ class Config(object): CELERY = { "worker_max_tasks_per_child": 500, + "task_ignore_result": True, + "result_persistent": False, "broker_url": REDIS_URL, "broker_transport_options": { "visibility_timeout": 310, diff --git a/notifications_utils/logging.py b/notifications_utils/logging.py index 0a13555d4..a15d00169 100644 --- a/notifications_utils/logging.py +++ b/notifications_utils/logging.py @@ -70,10 +70,12 @@ def init_app(app): for logger_instance, handler in product(loggers, handlers): logger_instance.addHandler(handler) logger_instance.setLevel(loglevel) + logger_instance.propagate = False warning_loggers = [logging.getLogger("boto3"), logging.getLogger("s3transfer")] for logger_instance, handler in product(warning_loggers, handlers): logger_instance.addHandler(handler) logger_instance.setLevel(logging.WARNING) + logger_instance.propagate = False # Suppress specific loggers to prevent leaking sensitive info logging.getLogger("boto3").setLevel(logging.ERROR) diff --git a/tests/app/aws/test_s3.py b/tests/app/aws/test_s3.py index e4a9c1c07..843ce3ba0 100644 --- a/tests/app/aws/test_s3.py +++ b/tests/app/aws/test_s3.py @@ -219,6 +219,20 @@ def test_get_s3_file_makes_correct_call(notify_api, mocker): 2, "5555555552", ), + ( + # simulate file saved with utf8withbom + "\\ufeffPHONE NUMBER\n", + "eee", + 2, + "5555555552", + ), + ( + # simulate file saved without utf8withbom + "\\PHONE NUMBER\n", + "eee", + 2, + "5555555552", + ), ], ) def test_get_phone_number_from_s3(