Merge pull request #2642 from alphagov/celery-task-id

Add request_ids to some Celery tasks

app/celery/celery.py

@@ -2,7 +2,8 @@ import time
 
 from celery import Celery, Task
 from celery.signals import worker_process_shutdown
-from flask import current_app
+from flask import current_app, g, request
+from flask.ctx import has_request_context
 
 
 @worker_process_shutdown.connect
@@ -32,8 +33,20 @@ def make_task(app):
             # ensure task has flask context to access config, logger, etc
             with app.app_context():
                 self.start = time.time()
+                # Remove 'request_id' from the kwargs (so the task doesn't get an unexpected kwarg), then add it to g
+                # so that it gets logged
+                g.request_id = kwargs.pop('request_id', None)
                 return super().__call__(*args, **kwargs)
 
+        def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
+                        link=None, link_error=None, **options):
+            kwargs = kwargs or {}
+
+            if has_request_context() and hasattr(request, 'request_id'):
+                kwargs['request_id'] = request.request_id
+
+            return super().apply_async(args, kwargs, task_id, producer, link, link_error, **options)
+
     return NotifyTask
 
 

app/v2/notifications/post_notifications.py

@@ -1,9 +1,9 @@
 import base64
 import functools
 
-import werkzeug
 from flask import request, jsonify, current_app, abort
 from notifications_utils.recipients import try_validate_and_format_phone_number
+from werkzeug.exceptions import BadRequest
 
 from app import api_user, authenticated_service, notify_celery, document_download_client
 from app.celery.letters_pdf_tasks import create_letters_pdf, process_virus_scan_passed
@@ -101,7 +101,7 @@ def post_precompiled_letter_notification():
 def post_notification(notification_type):
     try:
         request_json = request.get_json()
-    except werkzeug.exceptions.BadRequest as e:
+    except BadRequest as e:
         raise BadRequestError(message="Error decoding arguments: {}".format(e.description),
                               status_code=400)
 

requirements-app.txt

@@ -29,6 +29,6 @@ awscli-cwlogs>=1.4,<1.5
 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
 itsdangerous==0.24  # pyup: <1.0.0
 
-git+https://github.com/alphagov/notifications-utils.git@34.0.1#egg=notifications-utils==34.0.1
+git+https://github.com/alphagov/notifications-utils.git@36.1.0#egg=notifications-utils==36.1.0
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3

requirements.txt

@@ -31,32 +31,32 @@ awscli-cwlogs>=1.4,<1.5
 # Putting upgrade on hold due to v1.0.0 using sha512 instead of sha1 by default
 itsdangerous==0.24  # pyup: <1.0.0
 
-git+https://github.com/alphagov/notifications-utils.git@34.0.1#egg=notifications-utils==34.0.1
+git+https://github.com/alphagov/notifications-utils.git@36.1.0#egg=notifications-utils==36.1.0
 
 git+https://github.com/alphagov/boto.git@2.43.0-patch3#egg=boto==2.43.0-patch3
 
 ## The following requirements were added by pip freeze:
-alembic==1.1.0
+alembic==1.2.1
 amqp==1.4.9
 anyjson==0.3.3
-attrs==19.1.0
-awscli==1.16.231
+attrs==19.3.0
+awscli==1.16.263
 bcrypt==3.1.7
 billiard==3.3.0.23
 bleach==3.1.0
-boto3==1.6.16
-botocore==1.12.221
-certifi==2019.6.16
+boto3==1.9.221
+botocore==1.12.253
+certifi==2019.9.11
 chardet==3.0.4
 Click==7.0
-colorama==0.3.9
+colorama==0.4.1
 dnspython==1.16.0
 docutils==0.15.2
 flask-redis==0.4.0
-future==0.17.1
+future==0.18.1
 greenlet==0.4.15
 idna==2.8
-Jinja2==2.10.1
+Jinja2==2.10.3
 jmespath==0.9.4
 kombu==3.0.37
 Mako==1.1.0
@@ -64,7 +64,7 @@ MarkupSafe==1.1.1
 mistune==0.8.4
 monotonic==1.5
 orderedset==2.0.1
-phonenumbers==8.10.13
+phonenumbers==8.10.17
 pyasn1==0.4.7
 pycparser==2.19
 PyPDF2==1.26.0
@@ -72,15 +72,15 @@ pyrsistent==0.15.4
 python-dateutil==2.8.0
 python-editor==1.0.4
 python-json-logger==0.1.11
-pytz==2019.2
-PyYAML==4.2b1
-redis==3.3.8
+pytz==2019.3
+PyYAML==5.1.2
+redis==3.3.11
 requests==2.22.0
 rsa==3.4.2
 s3transfer==0.2.1
 six==1.12.0
 smartypants==2.0.1
 statsd==3.3.0
-urllib3==1.25.3
+urllib3==1.25.6
 webencodings==0.5.1
-Werkzeug==0.15.5
+Werkzeug==0.16.0

requirements_for_test.txt

@@ -13,4 +13,4 @@ requests-mock==1.5.2
 strict-rfc3339==0.7
 rfc3987==1.3.8
 # used for creating manifest file locally
-jinja2-cli[yaml]==0.6.0
+jinja2-cli[yaml]==0.7.0