tests/regression/tools/health/test_health.sh

   1 # Copyright (C) - 2012 Christian Babeux <christian.babeux@efficios.com>
   2 # Copyright (C) - 2014 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
   3 #
   4 # This program is free software; you can redistribute it and/or modify it
   5 # under the terms of the GNU General Public License, version 2 only, as
   6 # published by the Free Software Foundation.
   7 #
   8 # This program is distributed in the hope that it will be useful, but WITHOUT
   9 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 # more details.
  12 #
  13 # You should have received a copy of the GNU General Public License along with
  14 # this program; if not, write to the Free Software Foundation, Inc., 51
  15 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  16
  17 TESTDIR=${CURDIR}/../../..
  18 LTTNG_BIN="lttng"
  19 UST_EVENT_NAME="tp:tptest"
  20 KERNEL_EVENT_NAME="sched_switch"
  21 CHANNEL_NAME="testchan"
  22 HEALTH_CHECK_BIN="health_check"
  23 NUM_TESTS=90
  24 SLEEP_TIME=30
  25
  26 source $TESTDIR/utils/utils.sh
  27
  28 if [ ! -f "$CURDIR/$SESSIOND_PRELOAD" ]; then
  29         BAIL_OUT "${CURDIR}/${SESSIOND_PRELOAD} is missing."
  30 fi
  31
  32 function lttng_create_session_uri
  33 {
  34         # Create session with default path
  35         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $SESSION_NAME \
  36                 -U net://localhost >/dev/null 2>&1
  37         ok $? "Create session with default path"
  38 }
  39
  40 function report_errors
  41 {
  42         test_thread_error_string="$1"
  43         test_relayd="$2"
  44         err_no_relayd_match="Error querying relayd health"
  45
  46         # Check for health errors
  47         # Include inability to contact relayd health as an expected
  48         # error, since this can happen whenever the relayd shutdown due
  49         # to an error in any thread.
  50         out=$(grep "${test_thread_error_string}" ${STDOUT_PATH} | wc -l)
  51         if [ $test_relayd -ne 0 ]; then
  52                 outerr=$(grep "${err_no_relayd_match}" ${STDERR_PATH} | wc -l)
  53         else
  54                 outerr=0
  55         fi
  56         if [ $out -eq 0 ] && [ $outerr -eq 0 ]; then
  57                 fail "Validation failure"
  58                 diag "Health returned:"
  59                 diag "stdout:"
  60                 file=${STDOUT_PATH}
  61                 while read line ; do
  62                     diag "$line"
  63                 done < ${file}
  64
  65                 diag "stderr:"
  66                 file=${STDERR_PATH}
  67                 while read line ; do
  68                     diag "$line"
  69                 done < ${file}
  70         else
  71                 pass "Validation OK"
  72         fi
  73 }
  74
  75 function test_health
  76 {
  77         test_suffix="$1"
  78         test_thread_name="$2"
  79         test_thread_error_string="$3"
  80         test_needs_root="$4"
  81         test_consumerd="$5"
  82         test_relayd="$6"
  83
  84         diag "Test health problem detection with ${test_thread_name}"
  85
  86         # Set the socket timeout to 5 so the health check detection
  87         # happens within 25 s
  88         export LTTNG_NETWORK_SOCKET_TIMEOUT=5
  89         export LTTNG_RELAYD_HEALTH="${HEALTH_PATH}/test-health"
  90
  91         # Activate testpoints
  92         export LTTNG_TESTPOINT_ENABLE=1
  93
  94         # Activate specific thread test
  95         export ${test_thread_name}_${test_suffix}=1
  96
  97         # Spawn sessiond with preload healthexit lib
  98         export LD_PRELOAD="$CURDIR/$SESSIOND_PRELOAD"
  99
 100         diag "Start session daemon"
 101         start_lttng_sessiond
 102
 103         if [ ${test_consumerd} -eq 1 ]; then
 104                 create_lttng_session_no_output $SESSION_NAME
 105
 106                 diag "With UST consumer daemons"
 107                 enable_ust_lttng_event $SESSION_NAME $UST_EVENT_NAME $CHANNEL_NAME
 108
 109                 skip $isroot "Root access is needed. Skipping kernel consumer health check test." "1" ||
 110                 {
 111                         diag "With kernel consumer daemon"
 112                         lttng_enable_kernel_event $SESSION_NAME $KERNEL_EVENT_NAME $CHANNEL_NAME
 113                 }
 114                 start_lttng_tracing $SESSION_NAME
 115         fi
 116
 117         if [ ${test_relayd} -eq 1 ]; then
 118                 diag "With relay daemon"
 119                 RELAYD_ARGS="--relayd-path=${LTTNG_RELAYD_HEALTH}"
 120
 121                 start_lttng_relayd "-o $TRACE_PATH"
 122         else
 123                 RELAYD_ARGS=
 124         fi
 125
 126         # Check health status, not caring about result
 127         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 128                 > /dev/null
 129
 130         # Wait
 131         diag "Check after running for ${SLEEP_TIME} seconds"
 132         sleep ${SLEEP_TIME}
 133
 134         # Check health status
 135         $CURDIR/$HEALTH_CHECK_BIN ${RELAYD_ARGS} \
 136                 > ${STDOUT_PATH} 2> ${STDERR_PATH}
 137
 138
 139         if [ ${test_needs_root} -eq 1 ]; then
 140                 skip ${isroot} "Root access needed for test \"${test_thread_name}\"." "1" ||
 141                 {
 142                         report_errors "${test_thread_error_string}" "${test_relayd}"
 143                 }
 144         else
 145                 report_errors "${test_thread_error_string}" "${test_relayd}"
 146         fi
 147
 148         if [ ${test_relayd} -eq 1 ]; then
 149                 stop_lttng_relayd_nocheck
 150         fi
 151         stop_lttng_sessiond
 152
 153         unset LTTNG_TESTPOINT_ENABLE
 154         unset ${test_thread_name}_${test_suffix}
 155         unset LD_PRELOAD
 156         unset LTTNG_NETWORK_SOCKET_TIMEOUT
 157         unset LTTNG_RELAYD_HEALTH
 158 }
 159
 160 plan_tests $NUM_TESTS
 161
 162 print_test_banner "$TEST_DESC"
 163
 164 THREAD=("LTTNG_SESSIOND_THREAD_MANAGE_CLIENTS"
 165         "LTTNG_SESSIOND_THREAD_MANAGE_APPS"
 166         "LTTNG_SESSIOND_THREAD_REG_APPS"
 167         "LTTNG_SESSIOND_THREAD_HT_CLEANUP"
 168         "LTTNG_SESSIOND_THREAD_APP_MANAGE_NOTIFY"
 169         "LTTNG_SESSIOND_THREAD_APP_REG_DISPATCH"
 170         "LTTNG_SESSIOND_THREAD_MANAGE_KERNEL"
 171
 172         "LTTNG_CONSUMERD_THREAD_CHANNEL"
 173         "LTTNG_CONSUMERD_THREAD_METADATA"
 174         "LTTNG_CONSUMERD_THREAD_METADATA_TIMER"
 175
 176         "LTTNG_RELAYD_THREAD_DISPATCHER"
 177         "LTTNG_RELAYD_THREAD_WORKER"
 178         "LTTNG_RELAYD_THREAD_LISTENER"
 179         "LTTNG_RELAYD_THREAD_LIVE_DISPATCHER"
 180         "LTTNG_RELAYD_THREAD_LIVE_WORKER"
 181         "LTTNG_RELAYD_THREAD_LIVE_LISTENER"
 182 )
 183
 184 ERROR_STRING=(
 185         "Thread \"Session daemon command\" is not responding in component \"sessiond\"."
 186         "Thread \"Session daemon application manager\" is not responding in component \"sessiond\"."
 187         "Thread \"Session daemon application registration\" is not responding in component \"sessiond\"."
 188         "Thread \"Session daemon hash table cleanup\" is not responding in component \"sessiond\"."
 189         "Thread \"Session daemon application notification manager\" is not responding in component \"sessiond\"."
 190         "Thread \"Session daemon application registration dispatcher\" is not responding in component \"sessiond\"."
 191         "Thread \"Session daemon kernel\" is not responding in component \"sessiond\"."
 192
 193         "Thread \"Consumer daemon channel\" is not responding"
 194         "Thread \"Consumer daemon metadata\" is not responding"
 195         "Thread \"Consumer daemon metadata timer\" is not responding"
 196
 197         "Thread \"Relay daemon dispatcher\" is not responding in component \"relayd\"."
 198         "Thread \"Relay daemon worker\" is not responding in component \"relayd\"."
 199         "Thread \"Relay daemon listener\" is not responding in component \"relayd\"."
 200         "Thread \"Relay daemon live dispatcher\" is not responding in component \"relayd\"."
 201         "Thread \"Relay daemon live worker\" is not responding in component \"relayd\"."
 202         "Thread \"Relay daemon live listener\" is not responding in component \"relayd\"."
 203 )
 204
 205 # TODO
 206 # "LTTNG_SESSIOND_THREAD_MANAGE_CONSUMER"
 207 # "Thread \"Session daemon manage consumer\" is not responding in component \"sessiond\"."
 208
 209 # TODO: test kernel consumerd specifically in addition to UST consumerd
 210
 211 # TODO: need refactoring of consumerd teardown
 212 # "LTTNG_CONSUMERD_THREAD_SESSIOND"
 213 # "Thread \"Consumer daemon session daemon command manager\" is not responding"
 214
 215 # TODO: this thread is responsible for close a file descriptor that
 216 # triggers teardown of metadata thread. We should revisit teardown of
 217 # consumerd.
 218 # "LTTNG_CONSUMERD_THREAD_DATA"
 219 # "Thread \"Consumer daemon data\" is not responding"
 220
 221 NEEDS_ROOT=(
 222         0
 223         0
 224         0
 225         0
 226         0
 227         0
 228         1
 229
 230         0
 231         0
 232         0
 233
 234         0
 235         0
 236         0
 237         0
 238         0
 239         0
 240 )
 241
 242 TEST_CONSUMERD=(
 243         0
 244         0
 245         0
 246         0
 247         0
 248         0
 249         0
 250
 251         1
 252         1
 253         1
 254
 255         1
 256         1
 257         1
 258         1
 259         1
 260         1
 261 )
 262
 263 TEST_RELAYD=(
 264         0
 265         0
 266         0
 267         0
 268         0
 269         0
 270         0
 271
 272         0
 273         0
 274         0
 275
 276         1
 277         1
 278         1
 279         1
 280         1
 281         1
 282 )
 283
 284 STDOUT_PATH=$(mktemp)
 285 STDERR_PATH=$(mktemp)
 286 TRACE_PATH=$(mktemp -d)
 287 HEALTH_PATH=$(mktemp -d)
 288
 289 if [ "$(id -u)" == "0" ]; then
 290         isroot=1
 291 else
 292         isroot=0
 293 fi
 294
 295 THREAD_COUNT=${#THREAD[@]}
 296 i=0
 297 while [ "$i" -lt "$THREAD_COUNT" ]; do
 298         test_health "${TEST_SUFFIX}" \
 299                 "${THREAD[$i]}" \
 300                 "${ERROR_STRING[$i]}" \
 301                 "${NEEDS_ROOT[$i]}" \
 302                 "${TEST_CONSUMERD[$i]}" \
 303                 "${TEST_RELAYD[$i]}"
 304         let "i++"
 305 done
 306
 307 rm -rf ${HEALTH_PATH}
 308 rm -rf ${TRACE_PATH}
 309 rm -f ${STDOUT_PATH}
 310 rm -f ${STDERR_PATH}