From b7e2dfa7e37b432069e3ffe29ff03061de07928d Mon Sep 17 00:00:00 2001 From: Ryan Ahearn Date: Tue, 18 Oct 2022 11:54:54 -0400 Subject: [PATCH] Remove unused scripts files --- scripts/paas_app_wrapper.sh | 67 ----------- scripts/run_multi_worker_app_paas.sh | 167 --------------------------- tests/test_all_queues_used.py | 19 --- 3 files changed, 253 deletions(-) delete mode 100755 scripts/paas_app_wrapper.sh delete mode 100755 scripts/run_multi_worker_app_paas.sh delete mode 100644 tests/test_all_queues_used.py diff --git a/scripts/paas_app_wrapper.sh b/scripts/paas_app_wrapper.sh deleted file mode 100755 index 7aeb46868..000000000 --- a/scripts/paas_app_wrapper.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -case $NOTIFY_APP_NAME in - api) - unset GUNICORN_CMD_ARGS - exec scripts/run_app_paas.sh gunicorn -c /home/vcap/app/gunicorn_config.py application - ;; - delivery-worker-retry-tasks) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q retry-tasks 2> /dev/null - ;; - delivery-worker-letters) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q create-letters-pdf-tasks,letter-tasks 2> /dev/null - ;; - delivery-worker-jobs) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q database-tasks,job-tasks 2> /dev/null - ;; - delivery-worker-research) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q research-mode-tasks 2> /dev/null - ;; - delivery-worker-sender) - exec scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 4 -A run_celery.notify_celery --loglevel=INFO \ - --logfile=/dev/null --pidfile=/tmp/celery%N.pid -Q send-sms-tasks,send-email-tasks - ;; - delivery-worker-periodic) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ - -Q periodic-tasks 2> /dev/null - ;; - delivery-worker-reporting) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q reporting-tasks 2> /dev/null - ;; - delivery-worker-priority) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q priority-tasks 2> /dev/null - ;; - # Only consume the notify-internal-tasks queue on this app so that Notify messages are processed as a priority - delivery-worker-internal) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q notify-internal-tasks 2> /dev/null - ;; - delivery-worker-broadcasts) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \ - -Q broadcast-tasks 2> /dev/null - ;; - delivery-worker-receipts) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q ses-callbacks,sms-callbacks 2> /dev/null - ;; - delivery-worker-service-callbacks) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q service-callbacks,service-callbacks-retry 2> /dev/null - ;; - delivery-worker-save-api-notifications) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=4 \ - -Q save-api-email-tasks,save-api-sms-tasks 2> /dev/null - ;; - delivery-celery-beat) - exec scripts/run_app_paas.sh celery -A run_celery.notify_celery beat --loglevel=INFO - ;; - *) - echo "Unknown notify_app_name $NOTIFY_APP_NAME" - exit 1 - ;; -esac diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh deleted file mode 100755 index 15495b873..000000000 --- a/scripts/run_multi_worker_app_paas.sh +++ /dev/null @@ -1,167 +0,0 @@ -#!/bin/bash - -set -e -o pipefail - -TERMINATE_TIMEOUT=9 -MAX_DISK_SPACE_USAGE=75 -readonly LOGS_DIR="/home/vcap/logs" - -function check_params { - if [ -z "${NOTIFY_APP_NAME}" ]; then - echo "You must set NOTIFY_APP_NAME" - exit 1 - fi - - if [ -z "${CW_APP_NAME}" ]; then - CW_APP_NAME=${NOTIFY_APP_NAME} - fi -} - -function configure_aws_logs { - # create files so that aws logs agent doesn't complain - touch ${LOGS_DIR}/gunicorn_error.log - touch ${LOGS_DIR}/app.log.json - - aws configure set plugins.cwlogs cwlogs - - cat > /home/vcap/app/awslogs.conf << EOF -[general] -state_file = ${LOGS_DIR}/awslogs-state - -[${LOGS_DIR}/app.log] -file = ${LOGS_DIR}/app.log.json -log_group_name = paas-${CW_APP_NAME}-application -log_stream_name = {hostname} - -[${LOGS_DIR}/gunicorn_error.log] -file = ${LOGS_DIR}/gunicorn_error.log -log_group_name = paas-${CW_APP_NAME}-gunicorn -log_stream_name = {hostname} -EOF -} - -# For every PID, check if it's still running. if it is, send the sigterm. then wait 9 seconds before sending sigkill -function on_exit { - echo "multi worker app exiting" - wait_time=0 - - send_signal_to_celery_processes TERM - - # check if the apps are still running every second - while [[ "$wait_time" -le "$TERMINATE_TIMEOUT" ]]; do - get_celery_pids - ensure_celery_is_running - let wait_time=wait_time+1 - sleep 1 - done - - send_signal_to_celery_processes KILL -} - -function check_disk_space { - # get something like: - # - # Filesystem Use% - # overlay 56% - # tmpfs 0% - # - # and only keep '56' - SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%") - - if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then - echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space" - echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%" - exit - fi -} - -function get_celery_pids { - # get the PIDs of the process whose parent is the root process - # print only pid and their command, get the ones with "celery" in their name - # and keep only these PIDs - - set +o pipefail # so grep returning no matches does not premature fail pipe - APP_PIDS=$(pgrep -P 1 | xargs ps -o pid=,command= -p | grep celery | cut -f1 -d/) - set -o pipefail # pipefail should be set everywhere else -} - -function send_signal_to_celery_processes { - # refresh pids to account for the case that some workers may have terminated but others not - get_celery_pids - # send signal to all remaining apps - echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: " - echo ${APP_PIDS} | xargs kill -s ${1} -} - -function start_application { - echo "Starting application..." - eval "$@" - get_celery_pids - echo "Application process pids: "${APP_PIDS} -} - -function start_aws_logs_agent { - echo "Starting aws logs agent..." - exec aws logs push --region us-west-2 --config-file /home/vcap/app/awslogs.conf & - AWSLOGS_AGENT_PID=$! - echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" -} - -function start_logs_tail { - echo "Starting logs tail..." - exec tail -n0 -f ${LOGS_DIR}/app.log.json & - LOGS_TAIL_PID=$! - echo "tail pid: ${LOGS_TAIL_PID}" -} - -function ensure_celery_is_running { - if [ "${APP_PIDS}" = "" ]; then - echo "There are no celery processes running, this container is bad" - - echo "Exporting CF information for diagnosis" - - env | grep CF - - echo "Sleeping 15 seconds for logs to get shipped" - - sleep 15 - - echo "Killing awslogs_agent and tail" - kill -9 ${AWSLOGS_AGENT_PID} - kill -9 ${LOGS_TAIL_PID} - - exit 1 - fi -} - -function run { - while true; do - check_disk_space - get_celery_pids - - ensure_celery_is_running - - for APP_PID in ${APP_PIDS}; do - kill -0 ${APP_PID} 2&>/dev/null || return 1 - done - kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent - kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail - sleep 1 - done -} - -echo "Run script pid: $$" - -check_params - -trap "on_exit" EXIT TERM - -configure_aws_logs - -# The application has to start first! -start_application "$@" - -start_aws_logs_agent -start_logs_tail - -run diff --git a/tests/test_all_queues_used.py b/tests/test_all_queues_used.py deleted file mode 100644 index c4c164e7d..000000000 --- a/tests/test_all_queues_used.py +++ /dev/null @@ -1,19 +0,0 @@ -from app.config import QueueNames - - -# NOTE 100422 pass_app_wrapper can probably be removed -def test_queue_names_set_in_paas_app_wrapper(): - with open("scripts/paas_app_wrapper.sh", 'r') as stream: - search = ' -Q ' - - watched_queues = set() - for line in stream.readlines(): - start_of_queue_arg = line.find(search) - if start_of_queue_arg > 0: - start_of_queue_names = start_of_queue_arg + len(search) - end_of_queue_names = line.find('2>') if '2>' in line else len(line) - watched_queues.update({q.strip() for q in line[start_of_queue_names:end_of_queue_names].split(',')}) - - # ses-callbacks isn't used in api (only used in SNS lambda) - ignored_queues = {'ses-callbacks'} - assert watched_queues == set(QueueNames.all_queues()) | ignored_queues