darkhelm/notifications-api
1
0
Fork 0
mirror of https://github.com/GSA/notifications-api.git synced 2025-12-13 16:52:23 -05:00
Code Issues Packages Projects Releases Wiki Activity

Terminate instance if it's running out of disk space

Browse Source
This commit is contained in:
sakisv
2020-12-23 18:48:57 +02:00
parent 2108498eb1
commit a6ecfd66b6
2 changed files with 38 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
scripts/run_app_paas.sh
View File

@@ -3,6 +3,7 @@
set -e -o pipefail set -e -o pipefail
TERMINATE_TIMEOUT=9 TERMINATE_TIMEOUT=9
MAX_DISK_SPACE_USAGE=75
readonly LOGS_DIR="/home/vcap/logs" readonly LOGS_DIR="/home/vcap/logs"
function check_params { function check_params {
@@ -69,10 +70,28 @@ function start_aws_logs_agent {
echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}" echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
} }
function check_disk_space {
# get something like:
#
# Filesystem Use%
# overlay 56%
# tmpfs 0%
#
# and only keep '56'
SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
exit
fi
}
function run { function run {
while true; do while true; do
kill -0 ${APP_PID} 2&>/dev/null || break kill -0 ${APP_PID} 2&>/dev/null || break
kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
check_disk_space
sleep 1 sleep 1
done done
} }

19
scripts/run_multi_worker_app_paas.sh
View File

@@ -3,6 +3,7 @@
set -e -o pipefail set -e -o pipefail
TERMINATE_TIMEOUT=9 TERMINATE_TIMEOUT=9
MAX_DISK_SPACE_USAGE=75
readonly LOGS_DIR="/home/vcap/logs" readonly LOGS_DIR="/home/vcap/logs"
function check_params { function check_params {
@@ -124,6 +125,23 @@ function ensure_celery_is_running {
fi fi
} }
function check_disk_space {
# get something like:
#
# Filesystem Use%
# overlay 56%
# tmpfs 0%
#
# and only keep '56'
SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
exit
fi
}
function run { function run {
while true; do while true; do
get_celery_pids get_celery_pids
@@ -135,6 +153,7 @@ function run {
done done
kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail kill -0 ${LOGS_TAIL_PID} 2&>/dev/null || start_logs_tail
check_disk_space
sleep 1 sleep 1
done done
} }