Merge pull request #3082 from alphagov/fix-sender-logging

Add disk space check for sender worker

scripts/run_multi_worker_app_paas.sh

@@ -3,6 +3,7 @@
 set -e -o pipefail
 
 TERMINATE_TIMEOUT=9
+MAX_DISK_SPACE_USAGE=75
 readonly LOGS_DIR="/home/vcap/logs"
 
 function check_params {
@@ -49,15 +50,7 @@ function on_exit {
   # check if the apps are still running every second
   while [[ "$wait_time" -le "$TERMINATE_TIMEOUT" ]]; do
     get_celery_pids
-
-    # look here for explanation regarding this syntax:
-    # https://unix.stackexchange.com/a/298942/230401
-    PROCESS_COUNT="${#APP_PIDS[@]}"
-    if [[ "${PROCESS_COUNT}" -eq "0" ]]; then
-        echo "No celery process is running any more, exiting"
-        return 0
-    fi
-
+    ensure_celery_is_running
     let wait_time=wait_time+1
     sleep 1
   done
@@ -65,6 +58,23 @@ function on_exit {
   send_signal_to_celery_processes KILL
 }
 
+function check_disk_space {
+    # get something like:
+    #
+    # Filesystem     Use%
+    # overlay         56%
+    # tmpfs            0%
+    #
+    # and only keep '56'
+    SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
+
+    if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
+        echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
+        echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
+        exit
+    fi
+}
+
 function get_celery_pids {
   # get the PIDs of the process whose parent is the root process
   # print only pid and their command, get the ones with "celery" in their name
@@ -126,6 +136,7 @@ function ensure_celery_is_running {
 
 function run {
   while true; do
+    check_disk_space
     get_celery_pids
 
     ensure_celery_is_running