mirror of
https://github.com/GSA/notifications-api.git
synced 2025-12-13 16:52:23 -05:00
e7efeec309
TL;DR After a chat with some team members we've decided to double the concurrency of the delivery-worker-reporting app to 4 from 2. Looking at the memory usage during the reporting task runs we don't believe this to be a risk. There are some other things to look at, but this could be a quick win in the short term. Longer read: Every night we have 2 "reporting" tasks that run. - create-nightly-billing starts at 00:15 - populates data for ft_billing for the previous days. - 4 days for email - 4 days for sms - 10 days for letters - create-nightly-notification-status starts at 00:30 - populates data for ft_notification - 4 days for email - 4 days for sms - 10 days for letters These tasks are picked up by the `notify-delivery-worker-reporting` app, we run 3 instances with a concurrency = 2. This means that we have 6 worker threads that pick up the 18 tasks created at 00:15 and 00:30. Each celery main thread picks up 10 tasks of the queue, the 2 worker threads start working on a task and acknowledge the task to SQS. Meanwhile the other 8 tasks wait in the internal celery queue and are no acknowledgement is sent to SQS. As each task is complete a worker picks up a new thread, acknowledges the task. If a task is kept in the Celery internal queue for longer than 5 minutes the visibility timeout in SQS will assume the task has not completed and put the task back on the availability queue, therefore creating a duplicate task. At some point all the tasks are completed, some are completed twice.
68 lines
2.9 KiB
Bash
Executable File
68 lines
2.9 KiB
Bash
Executable File
#!/bin/bash
|
|
case $NOTIFY_APP_NAME in
|
|
api)
|
|
unset GUNICORN_CMD_ARGS
|
|
exec scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application
|
|
;;
|
|
delivery-worker-retry-tasks)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q retry-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-letters)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q create-letters-pdf-tasks,letter-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-jobs)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q database-tasks,job-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-research)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q research-mode-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-sender)
|
|
exec scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 4 -A run_celery.notify_celery --loglevel=INFO \
|
|
--logfile=/dev/null --pidfile=/tmp/celery%N.pid -Q send-sms-tasks,send-email-tasks
|
|
;;
|
|
delivery-worker-periodic)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
|
|
-Q periodic-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-reporting)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q reporting-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-priority)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q priority-tasks 2> /dev/null
|
|
;;
|
|
# Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority
|
|
delivery-worker-internal)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q notify-internal-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-broadcasts)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \
|
|
-Q broadcast-tasks 2> /dev/null
|
|
;;
|
|
delivery-worker-receipts)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q ses-callbacks,sms-callbacks 2> /dev/null
|
|
;;
|
|
delivery-worker-service-callbacks)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q service-callbacks,service-callbacks-retry 2> /dev/null
|
|
;;
|
|
delivery-worker-save-api-notifications)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \
|
|
-Q save-api-email-tasks,save-api-sms-tasks 2> /dev/null
|
|
;;
|
|
delivery-celery-beat)
|
|
exec scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO
|
|
;;
|
|
*)
|
|
echo "Unknown notify_app_name $NOTIFY_APP_NAME"
|
|
exit 1
|
|
;;
|
|
esac
|