From 4427827b2ff7cc790d1e3400a9eeca7b8c22b991 Mon Sep 17 00:00:00 2001
From: Athanasios Voutsadakis
 <athanasios.voutsadakis@digital.cabinet-office.gov.uk>
Date: Tue, 15 Jan 2019 17:42:14 +0000
Subject: [PATCH] Handle celery PIDs more reliably

This addresses some problems that existed in the previous approach:

1. There was a race condition that could occur between the time we were
looking for the existence of the .pid files and actually reading them.

2. If for some reason the .pid file was left behind after a process had
died, the script would never know because we do:

    kill -s ${1} ${APP_PID} || true
---
 scripts/run_multi_worker_app_paas.sh | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/scripts/run_multi_worker_app_paas.sh b/scripts/run_multi_worker_app_paas.sh
index 5ddc933ec..f195e59dd 100755
--- a/scripts/run_multi_worker_app_paas.sh
+++ b/scripts/run_multi_worker_app_paas.sh
@@ -54,7 +54,7 @@ function on_exit {
     # https://unix.stackexchange.com/a/298942/230401
     PROCESS_COUNT="${#APP_PIDS[@]}"
     if [[ "${PROCESS_COUNT}" -eq "0" ]]; then
-        echo "No more .pid files found, exiting"
+        echo "No celery process is running any more, exiting"
         return 0
     fi
 
@@ -66,21 +66,18 @@ function on_exit {
 }
 
 function get_celery_pids {
-  if [[ $(ls /home/vcap/app/celery*.pid) ]]; then
-    APP_PIDS=`cat /home/vcap/app/celery*.pid`
-  else
-    APP_PIDS=()
-  fi
+  # get the PIDs of the process whose parent is the root process
+  # print only pid and their command, get the ones with "celery" in their name
+  # and keep only these PIDs
+  APP_PIDS=$(pgrep -P 1 | xargs ps -o pid=,command= -p | grep celery | cut -f1 -d/)
 }
 
 function send_signal_to_celery_processes {
   # refresh pids to account for the case that some workers may have terminated but others not
   get_celery_pids
   # send signal to all remaining apps
-  for APP_PID in ${APP_PIDS}; do
-    echo "Sending signal ${1} to process with pid ${APP_PID}"
-    kill -s ${1} ${APP_PID} || true
-  done
+  echo ${APP_PIDS} | tr -d '\n' | tr -s ' ' | xargs echo "Sending signal ${1} to processes with pids: "
+  echo ${APP_PIDS} | xargs kill -s ${1}
 }
 
 function start_application {