tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 UST_EVENT_NAME="tp:tptest"
  19 KERNEL_EVENT_NAME="sched_switch"
  20 CHANNEL_NAME="testchan"
  21 HEALTH_CHECK_BIN="health_check"
  22 NUM_TESTS=106
  23 SLEEP_TIME=30
  24
  25 source $TESTDIR/utils/utils.sh
  26
  27 function report_errors
  28 {
  29         test_thread_error_string="$1"
  30         test_relayd="$2"
  31         err_no_relayd_match="Error querying relayd health"
  32
  33         # Check for health errors
  34         # Include inability to contact relayd health as an expected
  35         # error, since this can happen whenever the relayd shutdown due
  36         # to an error in any thread.
  37         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  38         if [ $test_relayd -ne 0 ]; then
  39                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  40         else
  41                 outerr=0
  42         fi
  43         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  44                 fail "Validation failure"
  45                 diag "Health returned:"
  46                 diag "stdout:"
  47                 file=${STDOUT_PATH}
  48                 while read line ; do
  49                     diag "$line"
  50                 done < ${file}
  51
  52                 diag "stderr:"
  53                 file=${STDERR_PATH}
  54                 while read line ; do
  55                     diag "$line"
  56                 done < ${file}
  57         else
  58                 pass "Validation OK"
  59         fi
  60 }
  61
  62 function test_health
  63 {
  64         test_suffix="$1"
  65         test_thread_name="$2"
  66         test_thread_error_string="$3"
  67         test_needs_root="$4"
  68         test_consumerd="$5"
  69         test_relayd="$6"
  70
  71         diag "Test health problem detection with ${test_thread_name}"
  72
  73         # Set the socket timeout to 5 so the health check detection
  74         # happens within 25 s
  75         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  76         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  77
  78         # Activate testpoints
  79         export LTTNG_TESTPOINT_ENABLE=1
  80
  81         # Activate specific thread test
  82         export ${test_thread_name}_${test_suffix}=1
  83
  84         # Spawn sessiond with preload healthexit lib
  85         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  86
  87         diag "Start session daemon"
  88         start_lttng_sessiond
  89
  90         if [ ${test_consumerd} -eq 1 ]; then
  91                 create_lttng_session_no_output $SESSION_NAME
  92
  93                 diag "With UST consumer daemons"
  94                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
  95
  96                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
  97                 {
  98                         diag "With kernel consumer daemon"
  99                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 100                 }
 101                 start_lttng_tracing_ok $SESSION_NAME
 102         fi
 103
 104         if [ ${test_relayd} -eq 1 ]; then
 105                 diag "With relay daemon"
 106                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 107
 108                 start_lttng_relayd "-o $TRACE_PATH"
 109         else
 110                 RELAYD_ARGS=
 111         fi
 112
 113         # Check health status, not caring about result
 114         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 115                 > /dev/null
 116
 117         # Wait
 118         diag "Check after running for ${SLEEP_TIME} seconds"
 119         sleep ${SLEEP_TIME}
 120
 121         # Check health status
 122         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 123                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 124
 125
 126         if [ ${test_needs_root} -eq 1 ]; then
 127                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 128                 {
 129                         report_errors "${test_thread_error_string}" "${test_relayd}"
 130                 }
 131         else
 132                 report_errors "${test_thread_error_string}" "${test_relayd}"
 133         fi
 134
 135         if [ ${test_relayd} -eq 1 ]; then
 136                 # We may fail to stop relayd here, and this is OK, since
 137                 # it may have been killed volountarily by testpoint.
 138                 stop_lttng_relayd_notap $KILL_SIGNAL
 139         fi
 140         stop_lttng_consumerd $KILL_SIGNAL
 141         stop_lttng_sessiond $KILL_SIGNAL
 142
 143         unset LTTNG_TESTPOINT_ENABLE
 144         unset ${test_thread_name}_${test_suffix}
 145         unset LD_PRELOAD
 146         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 147         unset LTTNG_RELAYD_HEALTH
 148 }
 149
 150 plan_tests $NUM_TESTS
 151
 152 print_test_banner "$TEST_DESC"
 153
 154 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 155         foundobj=1
 156 else
 157         foundobj=0
 158 fi
 159
 160 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 161
 162 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 163         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 164         "LTTNG_SESSIOND_THREAD_REG_APPS"
 165         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 166         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 167         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 168         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 169
 170         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 171         "LTTNG_CONSUMERD_THREAD_METADATA"
 172         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 173
 174         "LTTNG_RELAYD_THREAD_DISPATCHER"
 175         "LTTNG_RELAYD_THREAD_WORKER"
 176         "LTTNG_RELAYD_THREAD_LISTENER"
 177         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 178         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 179         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 180 )
 181
 182 ERROR_STRING=(
 183         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 184         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 185         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 186         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 187         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 188         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 189         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 190
 191         "Thread \"Consumer daemon channel\" is not responding"
 192         "Thread \"Consumer daemon metadata\" is not responding"
 193         "Thread \"Consumer daemon metadata timer\" is not responding"
 194
 195         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 196         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 197         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 198         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 199         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 200         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 201 )
 202
 203 # TODO
 204 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 205 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 206
 207 # TODO: test kernel consumerd specifically in addition to UST consumerd
 208
 209 # TODO: need refactoring of consumerd teardown
 210 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 211 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 212
 213 # TODO: this thread is responsible for close a file descriptor that
 214 # triggers teardown of metadata thread. We should revisit teardown of
 215 # consumerd.
 216 # "LTTNG_CONSUMERD_THREAD_DATA"
 217 # "Thread \"Consumer daemon data\" is not responding"
 218
 219 NEEDS_ROOT=(
 220         0
 221         0
 222         0
 223         0
 224         0
 225         0
 226         1
 227
 228         0
 229         0
 230         0
 231
 232         0
 233         0
 234         0
 235         0
 236         0
 237         0
 238 )
 239
 240 TEST_CONSUMERD=(
 241         0
 242         0
 243         0
 244         0
 245         0
 246         0
 247         0
 248
 249         1
 250         1
 251         1
 252
 253         1
 254         1
 255         1
 256         1
 257         1
 258         1
 259 )
 260
 261 TEST_RELAYD=(
 262         0
 263         0
 264         0
 265         0
 266         0
 267         0
 268         0
 269
 270         0
 271         0
 272         0
 273
 274         1
 275         1
 276         1
 277         1
 278         1
 279         1
 280 )
 281
 282 STDOUT_PATH=$(mktemp)
 283 STDERR_PATH=$(mktemp)
 284 TRACE_PATH=$(mktemp -d)
 285 HEALTH_PATH=$(mktemp -d)
 286
 287 if [ "$(id -u)" == "0" ]; then
 288         isroot=1
 289 else
 290         isroot=0
 291 fi
 292
 293 THREAD_COUNT=${#THREAD[@]}
 294 i=0
 295 while [ "$i" -lt "$THREAD_COUNT" ]; do
 296         test_health "${TEST_SUFFIX}" \
 297                 "${THREAD[$i]}" \
 298                 "${ERROR_STRING[$i]}" \
 299                 "${NEEDS_ROOT[$i]}" \
 300                 "${TEST_CONSUMERD[$i]}" \
 301                 "${TEST_RELAYD[$i]}"
 302         let "i++"
 303 done
 304
 305 rm -rf ${HEALTH_PATH}
 306 rm -rf ${TRACE_PATH}
 307 rm -f ${STDOUT_PATH}
 308 rm -f ${STDERR_PATH}