Improve handling of test SIGTERM/SIGINT (v2)
[lttng-tools.git] / tests / utils / utils.sh
index 9ece4d35200af26bb476a667bc76df5e58b931d5..71ddaa2cb5ea5a5c0e814eb307d201dcb2189817 100644 (file)
 
 SESSIOND_BIN="lttng-sessiond"
 SESSIOND_MATCH=".*lttng-sess.*"
-SESSIOND_PIDS=""
 RUNAS_BIN="lttng-runas"
 RUNAS_MATCH=".*lttng-runas.*"
 CONSUMERD_BIN="lttng-consumerd"
 CONSUMERD_MATCH=".*lttng-consumerd.*"
 RELAYD_BIN="lttng-relayd"
 RELAYD_MATCH=".*lttng-relayd.*"
-RELAYD_PIDS=""
 LTTNG_BIN="lttng"
 BABELTRACE_BIN="babeltrace"
 OUTPUT_DEST=/dev/null
@@ -48,11 +46,20 @@ export LTTNG_SESSIOND_PATH="/bin/true"
 
 source $TESTDIR/utils/tap/tap.sh
 
+if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
+       LTTNG_TEST_TEARDOWN_TIMEOUT=60
+fi
+
 function full_cleanup ()
 {
-       if [ -n "${SESSIOND_PIDS}" ] || [ -n "${RELAYD_PIDS}" ]; then
-               kill -9 ${SESSIOND_PIDS} ${RELAYD_PIDS} > /dev/null 2>&1
-       fi
+       # Try to kill daemons gracefully
+       stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+       stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
+
+       # If daemons are still present, forcibly kill them
+       stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+       stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
+       stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
 
        # Disable trap for SIGTERM since the following kill to the
        # pidgroup will be SIGTERM. Otherwise it loops.
@@ -379,26 +386,25 @@ function start_lttng_relayd_opt()
        local withtap=$1
        local opt=$2
 
-       DIR=$(readlink -f $TESTDIR)
+       DIR=$(readlink -f "$TESTDIR")
 
-       if [ -z $(pgrep $RELAYD_MATCH) ]; then
-               $DIR/../src/bin/lttng-relayd/$RELAYD_BIN -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
-               #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
+       if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
+               # shellcheck disable=SC2086
+               "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+               #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
                if [ $? -eq 1 ]; then
-                       if [ $withtap -eq "1" ]; then
+                       if [ "$withtap" -eq "1" ]; then
                                fail "Start lttng-relayd (opt: $opt)"
                        fi
                        return 1
                else
-                       if [ $withtap -eq "1" ]; then
+                       if [ "$withtap" -eq "1" ]; then
                                pass "Start lttng-relayd (opt: $opt)"
                        fi
                fi
        else
                pass "Start lttng-relayd (opt: $opt)"
        fi
-
-       RELAYD_PIDS=$(pgrep $RELAYD_MATCH)
 }
 
 function start_lttng_relayd()
@@ -414,29 +420,60 @@ function start_lttng_relayd_notap()
 function stop_lttng_relayd_opt()
 {
        local withtap=$1
+       local signal=$2
 
-       if [ $withtap -eq "1" ]; then
-               diag "Killing lttng-relayd (pid: $RELAYD_PIDS)"
+       if [ -z "$signal" ]; then
+               signal="SIGTERM"
        fi
-       kill $RELAYD_PIDS 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
-       retval=$?
 
-       if [ $? -eq 1 ]; then
-               if [ $withtap -eq "1" ]; then
+       local timeout_s=$3
+       local dtimeleft_s=
+
+       # Multiply time by 2 to simplify integer arithmetic
+       if [ -n "$timeout_s" ]; then
+               dtimeleft_s=$((timeout_s * 2))
+       fi
+
+       local retval=0
+       local pids=
+
+       pids=$(pgrep "$RELAYD_MATCH")
+       if [ -z "$pids" ]; then
+               if [ "$withtap" -eq "1" ]; then
+                       pass "No relay daemon to kill"
+               fi
+               return 0
+       fi
+
+       diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
+
+       # shellcheck disable=SC2086
+       if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+               retval=1
+               if [ "$withtap" -eq "1" ]; then
                        fail "Kill relay daemon"
                fi
-               return 1
        else
                out=1
                while [ -n "$out" ]; do
-                       out=$(pgrep $RELAYD_MATCH)
+                       out=$(pgrep "$RELAYD_MATCH")
+                       if [ -n "$dtimeleft_s" ]; then
+                               if [ $dtimeleft_s -lt 0 ]; then
+                                       out=
+                                       retval=1
+                               fi
+                               dtimeleft_s=$((dtimeleft_s - 1))
+                       fi
                        sleep 0.5
                done
-               if [ $withtap -eq "1" ]; then
-                       pass "Kill relay daemon"
+               if [ "$withtap" -eq "1" ]; then
+                       if [ "$retval" -eq "0" ]; then
+                               pass "Wait after kill relay daemon"
+                       else
+                               fail "Wait after kill relay daemon"
+                       fi
                fi
        fi
-       RELAYD_PIDS=""
        return $retval
 }
 
@@ -459,14 +496,16 @@ function start_lttng_sessiond_opt()
 
        local env_vars=""
        local consumerd=""
-       local long_bit_value=$(getconf LONG_BIT)
 
-       if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
+       local long_bit_value=
+       long_bit_value=$(getconf LONG_BIT)
+
+       if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
                # Env variable requested no session daemon
                return
        fi
 
-       DIR=$(readlink -f $TESTDIR)
+       DIR=$(readlink -f "$TESTDIR")
 
        # Get long_bit value for 32/64 consumerd
        case "$long_bit_value" in
@@ -483,32 +522,33 @@ function start_lttng_sessiond_opt()
 
        # Check for env. variable. Allow the use of LD_PRELOAD etc.
        if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
-               env_vars=${LTTNG_SESSIOND_ENV_VARS}
+               env_vars="${LTTNG_SESSIOND_ENV_VARS} "
        fi
+       env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 
-       validate_kernel_version
-       if [ $? -ne 0 ]; then
+       if ! validate_kernel_version; then
            fail "Start session daemon"
            BAIL_OUT "*** Kernel too old for session daemon tests ***"
        fi
 
-       : ${LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/config/}
+       : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
        export LTTNG_SESSION_CONFIG_XSD_PATH
 
-       if [ -z $(pgrep ${SESSIOND_MATCH}) ]; then
+       if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
                # Have a load path ?
                if [ -n "$load_path" ]; then
-                       env $env_vars $DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --load "$load_path" --background $consumerd
+                       # shellcheck disable=SC2086
+                       env $env_vars --load "$load_path" --background "$consumerd"
                else
-                       env $env_vars $DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background $consumerd
+                       # shellcheck disable=SC2086
+                       env $env_vars --background "$consumerd"
                fi
                #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
                status=$?
-               if [ $withtap -eq "1" ]; then
+               if [ "$withtap" -eq "1" ]; then
                        ok $status "Start session daemon"
                fi
        fi
-       SESSIOND_PIDS=$(pgrep $SESSIOND_MATCH)
 }
 
 function start_lttng_sessiond()
@@ -525,44 +565,98 @@ function stop_lttng_sessiond_opt()
 {
        local withtap=$1
        local signal=$2
-       local kill_opt=""
 
-       if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
+       if [ -z "$signal" ]; then
+               signal=SIGTERM
+       fi
+
+       local timeout_s=$3
+       local dtimeleft_s=
+
+       # Multiply time by 2 to simplify integer arithmetic
+       if [ -n "$timeout_s" ]; then
+               dtimeleft_s=$((timeout_s * 2))
+       fi
+
+       if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
                # Env variable requested no session daemon
-               return
+               return 0
        fi
 
-       local pids="${SESSIOND_PIDS} $(pgrep $RUNAS_MATCH)"
+       local retval=0
 
-       if [ -n "$2" ]; then
-               kill_opt="$kill_opt -s $signal"
+       local runas_pids=
+       runas_pids=$(pgrep "$RUNAS_MATCH")
+
+       local pids=
+       pids=$(pgrep "$SESSIOND_MATCH")
+
+       if [ -n "$runas_pids" ]; then
+               pids="$pids $runas_pids"
        fi
-       if [ $withtap -eq "1" ]; then
-               diag "Killing $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo $pids | tr '\n' ' ')"
+
+       if [ -z "$pids" ]; then
+               if [ "$withtap" -eq "1" ]; then
+                       pass "No session daemon to kill"
+               fi
+               return 0
        fi
-       kill $kill_opt $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 
-       if [ $? -eq 1 ]; then
-               if [ $withtap -eq "1" ]; then
+       diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
+
+       # shellcheck disable=SC2086
+       if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+               retval=1
+               if [ "$withtap" -eq "1" ]; then
                        fail "Kill sessions daemon"
                fi
        else
                out=1
                while [ -n "$out" ]; do
-                       out=$(pgrep ${SESSIOND_MATCH})
+                       out=$(pgrep "${SESSIOND_MATCH}")
+                       if [ -n "$dtimeleft_s" ]; then
+                               if [ $dtimeleft_s -lt 0 ]; then
+                                       out=
+                                       retval=1
+                               fi
+                               dtimeleft_s=$((dtimeleft_s - 1))
+                       fi
                        sleep 0.5
                done
                out=1
                while [ -n "$out" ]; do
-                       out=$(pgrep $CONSUMERD_MATCH)
+                       out=$(pgrep "$CONSUMERD_MATCH")
+                       if [ -n "$dtimeleft_s" ]; then
+                               if [ $dtimeleft_s -lt 0 ]; then
+                                       out=
+                                       retval=1
+                               fi
+                               dtimeleft_s=$((dtimeleft_s - 1))
+                       fi
                        sleep 0.5
                done
 
-               SESSIOND_PIDS=""
-               if [ $withtap -eq "1" ]; then
-                       pass "Kill session daemon"
+               if [ "$withtap" -eq "1" ]; then
+                       if [ "$retval" -eq "0" ]; then
+                               pass "Wait after kill session daemon"
+                       else
+                               fail "Wait after kill session daemon"
+                       fi
                fi
        fi
+       if [ "$signal" = "SIGKILL" ]; then
+               if [ "$(id -u)" -eq "0" ]; then
+                       local modules=
+                       modules="$(lsmod | grep ^lttng | awk '{print $1}')"
+
+                       if [ -n "$modules" ]; then
+                               diag "Unloading all LTTng modules"
+                               modprobe -r "$modules"
+                       fi
+               fi
+       fi
+
+       return $retval
 }
 
 function stop_lttng_sessiond()
@@ -579,43 +673,40 @@ function sigstop_lttng_sessiond_opt()
 {
        local withtap=$1
        local signal=SIGSTOP
-       local kill_opt=""
 
-       if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
+       if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
                # Env variable requested no session daemon
                return
        fi
 
-       PID_SESSIOND="$(pgrep ${SESSIOND_MATCH}) $(pgrep $RUNAS_MATCH)"
-
-       kill_opt="$kill_opt -s $signal"
+       PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 
-       if [ $withtap -eq "1" ]; then
-               diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo $PID_SESSIOND | tr '\n' ' ')"
+       if [ "$withtap" -eq "1" ]; then
+               diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
        fi
-       kill $kill_opt $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 
-       if [ $? -eq 1 ]; then
-               if [ $withtap -eq "1" ]; then
+       # shellcheck disable=SC2086
+       if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+               if [ "$withtap" -eq "1" ]; then
                        fail "Sending SIGSTOP to session daemon"
                fi
        else
                out=1
                while [ $out -ne 0 ]; do
-                       pid=$(pgrep $SESSIOND_MATCH)
+                       pid="$(pgrep "$SESSIOND_MATCH")"
 
                        # Wait until state becomes stopped for session
                        # daemon(s).
                        out=0
                        for sessiond_pid in $pid; do
-                               state=$(ps -p $sessiond_pid -o state= )
+                               state="$(ps -p "$sessiond_pid" -o state= )"
                                if [[ -n "$state" && "$state" != "T" ]]; then
                                        out=1
                                fi
                        done
                        sleep 0.5
                done
-               if [ $withtap -eq "1" ]; then
+               if [ "$withtap" -eq "1" ]; then
                        pass "Sending SIGSTOP to session daemon"
                fi
        fi
@@ -635,46 +726,70 @@ function stop_lttng_consumerd_opt()
 {
        local withtap=$1
        local signal=$2
-       local kill_opt=""
 
-       PID_CONSUMERD=$(pgrep $CONSUMERD_MATCH)
+       if [ -z "$signal" ]; then
+               signal=SIGTERM
+       fi
+
+       local timeout_s=$3
+       local dtimeleft_s=
 
-       if [ -n "$2" ]; then
-               kill_opt="$kill_opt -s $signal"
+       # Multiply time by 2 to simplify integer arithmetic
+       if [ -n "$timeout_s" ]; then
+               dtimeleft_s=$((timeout_s * 2))
        fi
 
-       if [ $withtap -eq "1" ]; then
-               diag "Killing $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
+       local retval=0
+
+       PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
+
+       if [ -z "$PID_CONSUMERD" ]; then
+               if [ "$withtap" -eq "1" ]; then
+                       pass "No consumer daemon to kill"
+               fi
+               return 0
        fi
 
-       kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
-       retval=$?
+       diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 
-       if [ $? -eq 1 ]; then
-               if [ $withtap -eq "1" ]; then
+       # shellcheck disable=SC2086
+       if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
+               retval=1
+               if [ "$withtap" -eq "1" ]; then
                        fail "Kill consumer daemon"
                fi
-               return 1
        else
                out=1
                while [ $out -ne 0 ]; do
-                       pid=$(pgrep $CONSUMERD_MATCH)
+                       pid="$(pgrep "$CONSUMERD_MATCH")"
 
                        # If consumerds are still present check their status.
                        # A zombie status qualifies the consumerd as *killed*
                        out=0
                        for consumer_pid in $pid; do
-                               state=$(ps -p $consumer_pid -o state= )
+                               state="$(ps -p "$consumer_pid" -o state= )"
                                if [[ -n "$state" && "$state" != "Z" ]]; then
                                        out=1
                                fi
                        done
+                       if [ -n "$dtimeleft_s" ]; then
+                               if [ $dtimeleft_s -lt 0 ]; then
+                                       out=0
+                                       retval=1
+                               fi
+                               dtimeleft_s=$((dtimeleft_s - 1))
+                       fi
                        sleep 0.5
                done
-               if [ $withtap -eq "1" ]; then
-                       pass "Kill consumer daemon"
+               if [ "$withtap" -eq "1" ]; then
+                       if [ "$retval" -eq "0" ]; then
+                               pass "Wait after kill consumer daemon"
+                       else
+                               fail "Wait after kill consumer daemon"
+                       fi
                fi
        fi
+
        return $retval
 }
 
@@ -692,40 +807,37 @@ function sigstop_lttng_consumerd_opt()
 {
        local withtap=$1
        local signal=SIGSTOP
-       local kill_opt=""
 
-       PID_CONSUMERD=$(pgrep $CONSUMERD_MATCH)
+       PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 
-       kill_opt="$kill_opt -s $signal"
+       diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 
-       if [ $withtap -eq "1" ]; then
-               diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
-       fi
-       kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
+       # shellcheck disable=SC2086
+       kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
        retval=$?
 
-       if [ $? -eq 1 ]; then
-               if [ $withtap -eq "1" ]; then
+       if [ $retval -eq 1 ]; then
+               if [ "$withtap" -eq "1" ]; then
                        fail "Sending SIGSTOP to consumer daemon"
                fi
                return 1
        else
                out=1
                while [ $out -ne 0 ]; do
-                       pid=$(pgrep $CONSUMERD_MATCH)
+                       pid="$(pgrep "$CONSUMERD_MATCH")"
 
                        # Wait until state becomes stopped for all
                        # consumers.
                        out=0
                        for consumer_pid in $pid; do
-                               state=$(ps -p $consumer_pid -o state= )
+                               state="$(ps -p "$consumer_pid" -o state= )"
                                if [[ -n "$state" && "$state" != "T" ]]; then
                                        out=1
                                fi
                        done
                        sleep 0.5
                done
-               if [ $withtap -eq "1" ]; then
+               if [ "$withtap" -eq "1" ]; then
                        pass "Sending SIGSTOP to consumer daemon"
                fi
        fi
This page took 0.031931 seconds and 5 git commands to generate.