WIP: Synchronous/verified service actions for check_ps_service.

Author: Michael Jennings

scripts/lbnl_ps.nhc

@@ -379,21 +379,22 @@ function check_ps_blacklist() {
 }
 
 # Check to make sure a service is (or isn't) running.  Syntax:
-#  check_ps_service [-0] [-f] [-S|-r|-c|-s|-k] [-u <user>] [-d <daemon> | -m <match>] [ -e <action> | -E <action> ] <service>
+#  check_ps_service [-0] [-f] [-v|-V] [-S|-r|-c|-s|-k] [-u <user>] [-d <daemon> | -m <match>] [ -e <action> | -E <action> ] <service>
 function check_ps_service() {
-    local SERVICE OWNER MATCH DAEMON NONFATAL=0 FULLMATCH=0 RESTART=0 CYCLE=0 START=0 STOP=0 KILL=0 ACTION FOUND_ACTION
-    local THIS_PID THIS_SVC i MSG
+    local SERVICE OWNER MATCH DAEMON NONFATAL=0 FULLMATCH=0 RESTART=0 CYCLE=0 START=0 STOP=0 KILL=0 VERIFY_SYNC=0 VERIFY_CHECK=0
+    local ACTION FOUND_ACTION THIS_PID THIS_SVC i MSG RET
     local -a ARGS
 
     if [[ ${#PS_PROCS[*]} -eq 0 ]]; then
         nhc_ps_gather_data
     fi
 
     OPTIND=1
-    while getopts ":0Sfrcsku:d:m:e:E:" OPTION ; do
+    while getopts ":0VSfrcskvu:d:m:e:E:" OPTION ; do
         case "$OPTION" in
             0) NONFATAL=1 ;;
             S) START=1 ;;
+            V) VERIFY_CHECK=1 ;;
             f) FULLMATCH=1 ;;
             r) RESTART=1 ;;
             c) CYCLE=1 ;;
@@ -404,6 +405,7 @@ function check_ps_service() {
             m) MATCH="$OPTARG" ;;
             e) ACTION="$OPTARG" ;;
             E) FOUND_ACTION="$OPTARG" ;;
+            v) VERIFY_SYNC=1 ;;
             :) die 1 "$FUNCNAME:  Option -$OPTARG requires an argument." ; return 1 ;;
             \?) die 1 "$FUNCNAME:  Invalid option:  -$OPTARG" ; return 1 ;;
         esac
@@ -451,12 +453,62 @@ function check_ps_service() {
             # Logic is inverted; we DON'T want this process running, so finding it is a failure.
             MSG="$FUNCNAME:  Service $SERVICE (process $DAEMON) ${OWNER:+owned by $OWNER }running"
             if [[ "$KILL" == "1" ]]; then
-                [[ "$SHELL" != ":" ]] && kill -9 $THIS_PID
-                MSG="$MSG; killed process ID $THIS_PID"
+                if [[ "$SHELL" != ":" ]]; then
+                    kill -9 $THIS_PID
+                    RET=$?
+                    if [[ $VERIFY_SYNC -eq 1 ]]; then
+                        # VERIFY_SYNC here only means we check the return value of the kill built-in.
+                        if [[ $RET -eq 0 ]]; then
+                            log "$MSG; process ID $THIS_PID killed successfully."
+                            continue
+                        else
+                            MSG="$MSG; \"kill -9 $THIS_PID\" failed (exit code $RET)."
+                        fi
+                    elif [[ $VERIFY_CHECK -eq 1 ]]; then
+                        # VERIFY_CHECK here means we kill the PID again and make sure it's gone.
+                        # Sleep very briefly to yield CPU, hopefully ensuring signal delivery.
+                        sleep 0.01
+                        if [[ $RET -ne 0 ]]; then
+                            MSG="$MSG; \"kill -9 $THIS_PID\" failed (exit code $RET)."
+                        elif kill -0 $THIS_PID ; then
+                            MSG="$MSG; \"kill -9 $THIS_PID\" succeeded but failed to terminate process."
+                        else
+                            log "$MSG; process ID $THIS_PID terminated successfully."
+                            return 0
+                        fi
+                    else
+                        MSG="$MSG; killed process ID $THIS_PID (SIGKILL)"
+                    fi
+                else
+                    MSG="$MSG; killed process ID $THIS_PID (test mode)"
+                fi
             else
                 # $STOP must be 1
                 ${SHELL:-/bin/bash} -c "/sbin/service $SERVICE stop" &
-                MSG="$MSG; termination in progress"
+                if [[ "$SHELL" == ":" ]]; then
+                    MSG="$MSG; termination in progress"
+                elif [[ $VERIFY_SYNC -eq 1 || $VERIFY_CHECK -eq 1 ]]; then
+                    # In VERIFY mode, we must "foreground" the service action to check its return value.
+                    wait $!
+                    RET=$?
+                    if [[ $RET -ne 0 ]]; then
+                        # If the "stop" fails, both VERIFY modes do the same thing.
+                        MSG="$MSG; \"/sbin/service $SERVICE stop\" failed (exit code $RET)."
+                    elif [[ $VERIFY_CHECK -eq 1 ]]; then
+                        # VERIFY_CHECK mode requires that we also make sure the PID is really gone now.
+                        if kill -0 $THIS_PID ; then
+                            MSG="$MSG; \"/sbin/service $SERVICE stop\" succeeded but failed to stop process $THIS_PID."
+                        else
+                            log "$MSG; service $SERVICE stopped and process $THIS_PID terminated successfully."
+                            return 0
+                        fi
+                    else
+                        log "$MSG; service $SERVICE stopped successfully."
+                        return 0
+                    fi
+                else
+                    MSG="$MSG; service termination in progress"
+                fi
             fi
             if [[ $NONFATAL == 1 ]]; then
                 if [[ -n "$MSG" ]]; then