Replace celery[sqs] with celery[redis]

app/celery/tasks.py

@@ -449,6 +449,7 @@ def handle_exception(task, notification, notification_id, exc):
         # Sometimes, SQS plays the same message twice. We should be able to catch an IntegrityError, but it seems
         # SQLAlchemy is throwing a FlushError. So we check if the notification id already exists then do not
         # send to the retry queue.
+        # This probably (hopefully) is not an issue with Redis as the celery backing store
         current_app.logger.exception('Retry' + retry_msg)
         try:
             task.retry(queue=QueueNames.RETRY, exc=exc)

app/config.py

@@ -114,9 +114,6 @@ class Config(object):
     FIRETEXT_API_KEY = os.environ.get("FIRETEXT_API_KEY", "placeholder")
     FIRETEXT_INTERNATIONAL_API_KEY = os.environ.get("FIRETEXT_INTERNATIONAL_API_KEY", "placeholder")
 
-    # Prefix to identify queues in SQS
-    NOTIFICATION_QUEUE_PREFIX = os.environ.get('NOTIFICATION_QUEUE_PREFIX')
-
     # Use notify.sandbox.10x sending domain unless overwritten by environment
     NOTIFY_EMAIL_DOMAIN = 'notify.sandbox.10x.gsa.gov'
 
@@ -200,11 +197,9 @@ class Config(object):
     DVLA_EMAIL_ADDRESSES = json.loads(os.environ.get('DVLA_EMAIL_ADDRESSES', '[]'))
 
     CELERY = {
-        'broker_url': 'sqs://',
+        'broker_url': REDIS_URL,
         'broker_transport_options': {
-            'region': AWS_REGION,
             'visibility_timeout': 310,
-            'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX,
         },
         'timezone': 'Europe/London',
         'imports': [
@@ -447,7 +442,6 @@ class Development(Config):
     NOTIFY_EMAIL_DOMAIN = os.getenv('NOTIFY_EMAIL_DOMAIN', 'notify.sandbox.10x.gsa.gov')
 
     SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://postgres:chummy@db:5432/notification_api')
-    REDIS_URL = os.environ.get('REDIS_URL')
 
     ANTIVIRUS_ENABLED = os.environ.get('ANTIVIRUS_ENABLED') == '1'
 
@@ -572,7 +566,6 @@ class Live(Config):
     REDIS_ENABLED = os.environ.get('REDIS_ENABLED')
 
     NOTIFY_LOG_PATH = os.environ.get('NOTIFY_LOG_PATH', 'application.log')
-    REDIS_URL = os.environ.get('REDIS_URL')
 
 
 class CloudFoundryConfig(Config):

app/letters/rest.py

@@ -10,6 +10,7 @@ letter_job = Blueprint("letter-job", __name__)
 register_errors(letter_job)
 
 # too many references will make SQS error (as the task can only be 256kb)
+# Maybe doesn't matter anymore with Redis as the celery backing store
 MAX_REFERENCES_PER_TASK = 5000
 
 

app/v2/notifications/post_notifications.py

@@ -236,6 +236,7 @@ def process_sms_or_email_notification(
             # If SQS cannot put the task on the queue, it's probably because the notification body was too long and it
             # went over SQS's 256kb message limit. If the body is very large, it may exceed the HTTP max content length;
             # the exception we get here isn't handled correctly by botocore - we get a ResponseParserError instead.
+            # Hopefully this is no longer an issue with Redis as celery's backing store
             current_app.logger.info(
                 f'Notification {notification_id} failed to save to high volume queue. Using normal flow instead'
             )