mirror of
https://github.com/GSA/notifications-api.git
synced 2026-07-31 03:38:57 -04:00
Revert celery4
Revert the following three pull requests: https://github.com/alphagov/notifications-api/pull/1085 https://github.com/alphagov/notifications-api/pull/1086 https://github.com/alphagov/notifications-api/pull/1088 celery 4.0.2 looked promising, however, on staging under mild load (5/sec api calls) the performance was actually worse than 3.1.25
This commit is contained in:
Leo Hemsted
144
app/config.py
144
app/config.py
@@ -1,6 +1,10 @@
|
||||
from datetime import timedelta
|
||||
import os
|
||||
import json
|
||||
|
||||
from celery.schedules import crontab
|
||||
from kombu import Exchange, Queue
|
||||
|
||||
from app.models import (
|
||||
EMAIL_TYPE, SMS_TYPE, LETTER_TYPE,
|
||||
KEY_TYPE_NORMAL, KEY_TYPE_TEAM, KEY_TYPE_TEST
|
||||
@@ -14,6 +18,34 @@ if os.environ.get('VCAP_SERVICES'):
|
||||
extract_cloudfoundry_config()
|
||||
|
||||
|
||||
class QueueNames(object):
|
||||
PERIODIC = 'periodic-tasks'
|
||||
PRIORITY = 'priority-tasks'
|
||||
DATABASE = 'database-tasks'
|
||||
SEND = 'send-tasks'
|
||||
RESEARCH_MODE = 'research-mode-tasks'
|
||||
STATISTICS = 'statistics-tasks'
|
||||
JOBS = 'job-tasks'
|
||||
RETRY = 'retry-tasks'
|
||||
NOTIFY = 'notify-internal-tasks'
|
||||
PROCESS_FTP = 'process-ftp-tasks'
|
||||
|
||||
@staticmethod
|
||||
def all_queues():
|
||||
return [
|
||||
QueueNames.PRIORITY,
|
||||
QueueNames.PERIODIC,
|
||||
QueueNames.DATABASE,
|
||||
QueueNames.SEND,
|
||||
QueueNames.RESEARCH_MODE,
|
||||
QueueNames.STATISTICS,
|
||||
QueueNames.JOBS,
|
||||
QueueNames.RETRY,
|
||||
QueueNames.NOTIFY,
|
||||
QueueNames.PROCESS_FTP
|
||||
]
|
||||
|
||||
|
||||
class Config(object):
|
||||
# URL of admin app
|
||||
ADMIN_BASE_URL = os.environ['ADMIN_BASE_URL']
|
||||
@@ -94,6 +126,104 @@ class Config(object):
|
||||
CHANGE_EMAIL_CONFIRMATION_TEMPLATE_ID = 'eb4d9930-87ab-4aef-9bce-786762687884'
|
||||
SERVICE_NOW_LIVE_TEMPLATE_ID = '618185c6-3636-49cd-b7d2-6f6f5eb3bdde'
|
||||
|
||||
BROKER_URL = 'sqs://'
|
||||
BROKER_TRANSPORT_OPTIONS = {
|
||||
'region': AWS_REGION,
|
||||
'polling_interval': 1, # 1 second
|
||||
'visibility_timeout': 310,
|
||||
'queue_name_prefix': NOTIFICATION_QUEUE_PREFIX
|
||||
}
|
||||
CELERY_ENABLE_UTC = True,
|
||||
CELERY_TIMEZONE = 'Europe/London'
|
||||
CELERY_ACCEPT_CONTENT = ['json']
|
||||
CELERY_TASK_SERIALIZER = 'json'
|
||||
CELERY_IMPORTS = ('app.celery.tasks', 'app.celery.scheduled_tasks')
|
||||
CELERYBEAT_SCHEDULE = {
|
||||
'run-scheduled-jobs': {
|
||||
'task': 'run-scheduled-jobs',
|
||||
'schedule': crontab(minute=1),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
# 'send-scheduled-notifications': {
|
||||
# 'task': 'send-scheduled-notifications',
|
||||
# 'schedule': crontab(minute='*/15'),
|
||||
# 'options': {'queue': 'periodic'}
|
||||
# },
|
||||
'delete-verify-codes': {
|
||||
'task': 'delete-verify-codes',
|
||||
'schedule': timedelta(minutes=63),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-invitations': {
|
||||
'task': 'delete-invitations',
|
||||
'schedule': timedelta(minutes=66),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-sms-notifications': {
|
||||
'task': 'delete-sms-notifications',
|
||||
'schedule': crontab(minute=0, hour=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-email-notifications': {
|
||||
'task': 'delete-email-notifications',
|
||||
'schedule': crontab(minute=20, hour=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-letter-notifications': {
|
||||
'task': 'delete-letter-notifications',
|
||||
'schedule': crontab(minute=40, hour=0),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete-inbound-sms': {
|
||||
'task': 'delete-inbound-sms',
|
||||
'schedule': crontab(minute=0, hour=1),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'send-daily-performance-platform-stats': {
|
||||
'task': 'send-daily-performance-platform-stats',
|
||||
'schedule': crontab(minute=0, hour=2),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'switch-current-sms-provider-on-slow-delivery': {
|
||||
'task': 'switch-current-sms-provider-on-slow-delivery',
|
||||
'schedule': crontab(), # Every minute
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'timeout-sending-notifications': {
|
||||
'task': 'timeout-sending-notifications',
|
||||
'schedule': crontab(minute=0, hour=3),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'remove_sms_email_jobs': {
|
||||
'task': 'remove_csv_files',
|
||||
'schedule': crontab(minute=0, hour=4),
|
||||
'options': {'queue': QueueNames.PERIODIC},
|
||||
'kwargs': {'job_types': [EMAIL_TYPE, SMS_TYPE]}
|
||||
},
|
||||
'remove_letter_jobs': {
|
||||
'task': 'remove_csv_files',
|
||||
'schedule': crontab(minute=20, hour=4),
|
||||
'options': {'queue': QueueNames.PERIODIC},
|
||||
'kwargs': {'job_types': [LETTER_TYPE]}
|
||||
},
|
||||
'remove_transformed_dvla_files': {
|
||||
'task': 'remove_transformed_dvla_files',
|
||||
'schedule': crontab(minute=40, hour=4),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'delete_dvla_response_files': {
|
||||
'task': 'delete_dvla_response_files',
|
||||
'schedule': crontab(minute=10, hour=5),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
},
|
||||
'timeout-job-statistics': {
|
||||
'task': 'timeout-job-statistics',
|
||||
'schedule': crontab(minute=0, hour=5),
|
||||
'options': {'queue': QueueNames.PERIODIC}
|
||||
}
|
||||
}
|
||||
CELERY_QUEUES = []
|
||||
|
||||
NOTIFICATIONS_ALERT = 5 # five mins
|
||||
FROM_NUMBER = 'development'
|
||||
|
||||
@@ -132,7 +262,6 @@ class Config(object):
|
||||
}
|
||||
|
||||
FREE_SMS_TIER_FRAGMENT_COUNT = 250000
|
||||
INITIALISE_QUEUES = False
|
||||
|
||||
SMS_INBOUND_WHITELIST = json.loads(os.environ.get('SMS_INBOUND_WHITELIST', '[]'))
|
||||
|
||||
@@ -142,20 +271,24 @@ class Config(object):
|
||||
######################
|
||||
|
||||
class Development(Config):
|
||||
INITIALISE_QUEUES = True
|
||||
SQLALCHEMY_ECHO = False
|
||||
NOTIFY_EMAIL_DOMAIN = 'notify.tools'
|
||||
CSV_UPLOAD_BUCKET_NAME = 'development-notifications-csv-upload'
|
||||
DVLA_RESPONSE_BUCKET_NAME = 'notify.tools-ftp'
|
||||
NOTIFY_ENVIRONMENT = 'development'
|
||||
NOTIFICATION_QUEUE_PREFIX = 'development'
|
||||
DEBUG = True
|
||||
|
||||
for queue in QueueNames.all_queues():
|
||||
Config.CELERY_QUEUES.append(
|
||||
Queue(queue, Exchange('default'), routing_key=queue)
|
||||
)
|
||||
|
||||
API_HOST_NAME = "http://localhost:6011"
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
|
||||
|
||||
class Test(Config):
|
||||
INITIALISE_QUEUES = True
|
||||
NOTIFY_EMAIL_DOMAIN = 'test.notify.com'
|
||||
FROM_NUMBER = 'testing'
|
||||
NOTIFY_ENVIRONMENT = 'test'
|
||||
@@ -169,6 +302,11 @@ class Test(Config):
|
||||
|
||||
BROKER_URL = 'you-forgot-to-mock-celery-in-your-tests://'
|
||||
|
||||
for queue in QueueNames.all_queues():
|
||||
Config.CELERY_QUEUES.append(
|
||||
Queue(queue, Exchange('default'), routing_key=queue)
|
||||
)
|
||||
|
||||
API_RATE_LIMIT_ENABLED = True
|
||||
API_HOST_NAME = "http://localhost:6011"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user