tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 LTTNG_BIN="lttng"
  19 UST_EVENT_NAME="tp:tptest"
  20 KERNEL_EVENT_NAME="sched_switch"
  21 CHANNEL_NAME="testchan"
  22 HEALTH_CHECK_BIN="health_check"
  23 NUM_TESTS=106
  24 SLEEP_TIME=30
  25
  26 source $TESTDIR/utils/utils.sh
  27
  28 function lttng_create_session_uri
  29 {
  30         # Create session with default path
  31         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \
  32                 -U net://localhost >/dev/null 2>&1
  33         ok $? "Create session with default path"
  34 }
  35
  36 function report_errors
  37 {
  38         test_thread_error_string="$1"
  39         test_relayd="$2"
  40         err_no_relayd_match="Error querying relayd health"
  41
  42         # Check for health errors
  43         # Include inability to contact relayd health as an expected
  44         # error, since this can happen whenever the relayd shutdown due
  45         # to an error in any thread.
  46         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  47         if [ $test_relayd -ne 0 ]; then
  48                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  49         else
  50                 outerr=0
  51         fi
  52         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  53                 fail "Validation failure"
  54                 diag "Health returned:"
  55                 diag "stdout:"
  56                 file=${STDOUT_PATH}
  57                 while read line ; do
  58                     diag "$line"
  59                 done < ${file}
  60
  61                 diag "stderr:"
  62                 file=${STDERR_PATH}
  63                 while read line ; do
  64                     diag "$line"
  65                 done < ${file}
  66         else
  67                 pass "Validation OK"
  68         fi
  69 }
  70
  71 function test_health
  72 {
  73         test_suffix="$1"
  74         test_thread_name="$2"
  75         test_thread_error_string="$3"
  76         test_needs_root="$4"
  77         test_consumerd="$5"
  78         test_relayd="$6"
  79
  80         diag "Test health problem detection with ${test_thread_name}"
  81
  82         # Set the socket timeout to 5 so the health check detection
  83         # happens within 25 s
  84         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  85         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  86
  87         # Activate testpoints
  88         export LTTNG_TESTPOINT_ENABLE=1
  89
  90         # Activate specific thread test
  91         export ${test_thread_name}_${test_suffix}=1
  92
  93         # Spawn sessiond with preload healthexit lib
  94         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  95
  96         diag "Start session daemon"
  97         start_lttng_sessiond
  98
  99         if [ ${test_consumerd} -eq 1 ]; then
 100                 create_lttng_session_no_output $SESSION_NAME
 101
 102                 diag "With UST consumer daemons"
 103                 enable_ust_lttng_event_ok $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
 104
 105                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
 106                 {
 107                         diag "With kernel consumer daemon"
 108                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 109                 }
 110                 start_lttng_tracing_ok $SESSION_NAME
 111         fi
 112
 113         if [ ${test_relayd} -eq 1 ]; then
 114                 diag "With relay daemon"
 115                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 116
 117                 start_lttng_relayd "-o $TRACE_PATH"
 118         else
 119                 RELAYD_ARGS=
 120         fi
 121
 122         # Check health status, not caring about result
 123         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 124                 > /dev/null
 125
 126         # Wait
 127         diag "Check after running for ${SLEEP_TIME} seconds"
 128         sleep ${SLEEP_TIME}
 129
 130         # Check health status
 131         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 132                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 133
 134
 135         if [ ${test_needs_root} -eq 1 ]; then
 136                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 137                 {
 138                         report_errors "${test_thread_error_string}" "${test_relayd}"
 139                 }
 140         else
 141                 report_errors "${test_thread_error_string}" "${test_relayd}"
 142         fi
 143
 144         if [ ${test_relayd} -eq 1 ]; then
 145                 # We may fail to stop relayd here, and this is OK, since
 146                 # it may have been killed volountarily by testpoint.
 147                 stop_lttng_relayd_notap $KILL_SIGNAL
 148         fi
 149         stop_lttng_consumerd $KILL_SIGNAL
 150         stop_lttng_sessiond $KILL_SIGNAL
 151
 152         unset LTTNG_TESTPOINT_ENABLE
 153         unset ${test_thread_name}_${test_suffix}
 154         unset LD_PRELOAD
 155         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 156         unset LTTNG_RELAYD_HEALTH
 157 }
 158
 159 plan_tests $NUM_TESTS
 160
 161 print_test_banner "$TEST_DESC"
 162
 163 if [ -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
 164         foundobj=1
 165 else
 166         foundobj=0
 167 fi
 168
 169 skip $foundobj "No shared object generated. Skipping all tests." $NUM_TESTS && exit 0
 170
 171 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 172         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 173         "LTTNG_SESSIOND_THREAD_REG_APPS"
 174         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 175         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 176         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 177         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 178
 179         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 180         "LTTNG_CONSUMERD_THREAD_METADATA"
 181         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 182
 183         "LTTNG_RELAYD_THREAD_DISPATCHER"
 184         "LTTNG_RELAYD_THREAD_WORKER"
 185         "LTTNG_RELAYD_THREAD_LISTENER"
 186         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 187         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 188         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 189 )
 190
 191 ERROR_STRING=(
 192         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 193         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 194         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 195         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 196         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 197         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 198         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 199
 200         "Thread \"Consumer daemon channel\" is not responding"
 201         "Thread \"Consumer daemon metadata\" is not responding"
 202         "Thread \"Consumer daemon metadata timer\" is not responding"
 203
 204         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 205         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 206         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 207         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 208         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 209         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 210 )
 211
 212 # TODO
 213 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 214 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 215
 216 # TODO: test kernel consumerd specifically in addition to UST consumerd
 217
 218 # TODO: need refactoring of consumerd teardown
 219 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 220 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 221
 222 # TODO: this thread is responsible for close a file descriptor that
 223 # triggers teardown of metadata thread. We should revisit teardown of
 224 # consumerd.
 225 # "LTTNG_CONSUMERD_THREAD_DATA"
 226 # "Thread \"Consumer daemon data\" is not responding"
 227
 228 NEEDS_ROOT=(
 229         0
 230         0
 231         0
 232         0
 233         0
 234         0
 235         1
 236
 237         0
 238         0
 239         0
 240
 241         0
 242         0
 243         0
 244         0
 245         0
 246         0
 247 )
 248
 249 TEST_CONSUMERD=(
 250         0
 251         0
 252         0
 253         0
 254         0
 255         0
 256         0
 257
 258         1
 259         1
 260         1
 261
 262         1
 263         1
 264         1
 265         1
 266         1
 267         1
 268 )
 269
 270 TEST_RELAYD=(
 271         0
 272         0
 273         0
 274         0
 275         0
 276         0
 277         0
 278
 279         0
 280         0
 281         0
 282
 283         1
 284         1
 285         1
 286         1
 287         1
 288         1
 289 )
 290
 291 STDOUT_PATH=$(mktemp)
 292 STDERR_PATH=$(mktemp)
 293 TRACE_PATH=$(mktemp -d)
 294 HEALTH_PATH=$(mktemp -d)
 295
 296 if [ "$(id -u)" == "0" ]; then
 297         isroot=1
 298 else
 299         isroot=0
 300 fi
 301
 302 THREAD_COUNT=${#THREAD[@]}
 303 i=0
 304 while [ "$i" -lt "$THREAD_COUNT" ]; do
 305         test_health "${TEST_SUFFIX}" \
 306                 "${THREAD[$i]}" \
 307                 "${ERROR_STRING[$i]}" \
 308                 "${NEEDS_ROOT[$i]}" \
 309                 "${TEST_CONSUMERD[$i]}" \
 310                 "${TEST_RELAYD[$i]}"
 311         let "i++"
 312 done
 313
 314 rm -rf ${HEALTH_PATH}
 315 rm -rf ${TRACE_PATH}
 316 rm -f ${STDOUT_PATH}
 317 rm -f ${STDERR_PATH}