Merge pull request #3076 from alphagov/fix-app-logging

Fix app logging

scripts/paas_app_wrapper.sh

@@ -22,7 +22,7 @@ case $NOTIFY_APP_NAME in
     ;;
   delivery-worker-sender)
     exec scripts/run_multi_worker_app_paas.sh celery multi start 3 -c 10 -A run_celery.notify_celery --loglevel=INFO \
-    -Q send-sms-tasks,send-email-tasks
+    --logfile=/dev/null --pidfile=/tmp/celery%N.pid -Q send-sms-tasks,send-email-tasks
     ;;
   delivery-worker-periodic)
     exec scripts/run_app_paas.sh celery -A run_celery.notify_celery worker --loglevel=INFO --concurrency=2 \

scripts/run_app_paas.sh

@@ -3,6 +3,7 @@
 set -e -o pipefail
 
 TERMINATE_TIMEOUT=9
+MAX_DISK_SPACE_USAGE=75
 readonly LOGS_DIR="/home/vcap/logs"
 
 function check_params {
@@ -69,10 +70,28 @@ function start_aws_logs_agent {
   echo "AWS logs agent pid: ${AWSLOGS_AGENT_PID}"
 }
 
+function check_disk_space {
+    # get something like:
+    #
+    # Filesystem     Use%
+    # overlay         56%
+    # tmpfs            0%
+    #
+    # and only keep '56'
+    SPACE_USAGE=$(df --output="source,pcent" | grep overlay | tr --squeeze-repeats " " | cut -f2 -d" "| cut -f1 -d"%")
+
+    if [[ "${SPACE_USAGE}" -ge "${MAX_DISK_SPACE_USAGE}" ]]; then
+        echo "Terminating ${NOTIFY_APP_NAME}, instance ${INSTANCE_INDEX} because we're running out of disk space"
+        echo "Usage: ${SPACE_USAGE}% - limit ${MAX_DISK_SPACE_USAGE}%"
+        exit
+    fi
+}
+
 function run {
   while true; do
     kill -0 ${APP_PID} 2&>/dev/null || break
     kill -0 ${AWSLOGS_AGENT_PID} 2&>/dev/null || start_aws_logs_agent
+    check_disk_space
     sleep 1
   done
 }