tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69 }
  70
  71
  72 trap full_cleanup SIGINT SIGTERM
  73
  74 function print_ok ()
  75 {
  76         # Check if we are a terminal
  77         if [ -t 1 ]; then
  78                 echo -e "\e[1;32mOK\e[0m"
  79         else
  80                 echo -e "OK"
  81         fi
  82 }
  83
  84 function print_fail ()
  85 {
  86         # Check if we are a terminal
  87         if [ -t 1 ]; then
  88                 echo -e "\e[1;31mFAIL\e[0m"
  89         else
  90                 echo -e "FAIL"
  91         fi
  92 }
  93
  94 function print_test_banner ()
  95 {
  96         local desc="$1"
  97         diag "$desc"
  98 }
  99
 100 function validate_kernel_version ()
 101 {
 102         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 103         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 104                 return 0
 105         fi
 106         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 107                 return 0
 108         fi
 109         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 110                 return 0
 111         fi
 112         return 1
 113 }
 114
 115 # Generate a random string
 116 #  $1 = number of characters; defaults to 16
 117 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 118 function randstring()
 119 {
 120         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 121         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 122         echo
 123 }
 124
 125 # Return the number of _configured_ CPUs.
 126 function conf_proc_count()
 127 {
 128         getconf _NPROCESSORS_CONF
 129         if [ $? -ne 0 ]; then
 130                 diag "Failed to get the number of configured CPUs"
 131         fi
 132         echo
 133 }
 134
 135 # Check if base lttng-modules are present.
 136 # Bail out on failure
 137 function validate_lttng_modules_present ()
 138 {
 139         modprobe -n lttng-tracer 2>/dev/null
 140         if [ $? -ne 0  ]; then
 141                 BAIL_OUT "LTTng modules not detected."
 142         fi
 143 }
 144
 145 function enable_kernel_lttng_event
 146 {
 147         local withtap="$1"
 148         local expected_to_fail="$2"
 149         local sess_name="$3"
 150         local event_name="$4"
 151         local channel_name="$5"
 152
 153         if [ -z "$event_name" ]; then
 154                 # Enable all event if no event name specified
 155                 event_name="-a"
 156         fi
 157
 158         if [ -z "$channel_name" ]; then
 159                 # default channel if none specified
 160                 chan=""
 161         else
 162                 chan="-c $channel_name"
 163         fi
 164
 165         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 166         ret=$?
 167         if [[ $expected_to_fail -eq "1" ]]; then
 168                 test $ret -ne "0"
 169                 ret=$?
 170                 if [ $withtap -eq "1" ]; then
 171                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 172                 fi
 173         else
 174                 if [ $withtap -eq "1" ]; then
 175                         ok $ret "Enable kernel event $event_name for session $sess_name"
 176                 fi
 177         fi
 178 }
 179
 180 function enable_kernel_lttng_event_ok ()
 181 {
 182         enable_kernel_lttng_event 1 0 "$@"
 183 }
 184
 185 function enable_kernel_lttng_event_fail ()
 186 {
 187         enable_kernel_lttng_event 1 1 "$@"
 188 }
 189
 190 function enable_kernel_lttng_event_notap ()
 191 {
 192         enable_kernel_lttng_event 0 0 "$@"
 193 }
 194
 195 # Old interface
 196 function lttng_enable_kernel_event
 197 {
 198         enable_kernel_lttng_event_ok "$@"
 199 }
 200
 201 function lttng_enable_kernel_syscall()
 202 {
 203         local expected_to_fail=$1
 204         local sess_name=$2
 205         local syscall_name=$3
 206         local channel_name=$4
 207
 208         if [ -z $syscall_name ]; then
 209                 # Enable all event if no syscall name specified
 210                 syscall_name="-a"
 211         fi
 212
 213         if [ -z $channel_name ]; then
 214                 # default channel if none specified
 215                 chan=""
 216         else
 217                 chan="-c $channel_name"
 218         fi
 219
 220         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 221         ret=$?
 222         if [[ $expected_to_fail -eq "1" ]]; then
 223                 test $ret -ne "0"
 224                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 225         else
 226                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 227         fi
 228 }
 229
 230 function lttng_enable_kernel_syscall_ok()
 231 {
 232         lttng_enable_kernel_syscall 0 "$@"
 233 }
 234
 235 function lttng_enable_kernel_syscall_fail()
 236 {
 237         lttng_enable_kernel_syscall 1 "$@"
 238 }
 239
 240 function lttng_disable_kernel_syscall()
 241 {
 242         local expected_to_fail=$1
 243         local sess_name=$2
 244         local syscall_name=$3
 245         local channel_name=$4
 246
 247         if [ -z $syscall_name ]; then
 248                 # Enable all event if no syscall name specified
 249                 syscall_name="-a"
 250         fi
 251
 252         if [ -z $channel_name ]; then
 253                 # default channel if none specified
 254                 chan=""
 255         else
 256                 chan="-c $channel_name"
 257         fi
 258
 259         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 260
 261         ret=$?
 262         if [[ $expected_to_fail -eq "1" ]]; then
 263                 test $ret -ne "0"
 264                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 265         else
 266                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 267         fi
 268 }
 269
 270 function lttng_disable_kernel_syscall_ok()
 271 {
 272         lttng_disable_kernel_syscall 0 "$@"
 273 }
 274
 275 function lttng_disable_kernel_syscall_fail()
 276 {
 277         lttng_disable_kernel_syscall 1 "$@"
 278 }
 279
 280 function lttng_enable_kernel_userspace_probe_event ()
 281 {
 282         local expected_to_fail="$1"
 283         local sess_name="$2"
 284         local target="$3"
 285         local event_name="$4"
 286
 287         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 288         ret=$?
 289         if [[ $expected_to_fail -eq "1" ]]; then
 290                 test $ret -ne "0"
 291                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 292         else
 293                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 294         fi
 295 }
 296
 297 function lttng_enable_kernel_userspace_probe_event_fail ()
 298 {
 299         lttng_enable_kernel_userspace_probe_event 1 "$@"
 300 }
 301
 302 function lttng_enable_kernel_userspace_probe_event_ok ()
 303 {
 304         lttng_enable_kernel_userspace_probe_event 0 "$@"
 305 }
 306
 307 function disable_kernel_lttng_userspace_probe_event_ok ()
 308 {
 309         local sess_name="$1"
 310         local event_name="$2"
 311
 312         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 313         ok $? "Disable kernel event $target for session $sess_name"
 314 }
 315 function lttng_enable_kernel_channel()
 316 {
 317         local withtap=$1
 318         local expected_to_fail=$2
 319         local sess_name=$3
 320         local channel_name=$4
 321         local opt=$5
 322
 323         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 324         ret=$?
 325         if [[ $expected_to_fail -eq "1" ]]; then
 326                 test "$ret" -ne "0"
 327                 ret=$?
 328                 if [ $withtap -eq "1" ]; then
 329                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 330                 fi
 331         else
 332                 if [ $withtap -eq "1" ]; then
 333                         ok $ret "Enable channel $channel_name for session $sess_name"
 334                 fi
 335         fi
 336 }
 337
 338 function lttng_enable_kernel_channel_ok()
 339 {
 340         lttng_enable_kernel_channel 1 0 "$@"
 341 }
 342
 343 function lttng_enable_kernel_channel_fail()
 344 {
 345         lttng_enable_kernel_channel 1 1 "$@"
 346 }
 347
 348 function lttng_enable_kernel_channel_notap()
 349 {
 350         lttng_enable_kernel_channel 0 0 "$@"
 351 }
 352
 353 function enable_kernel_lttng_channel_ok()
 354 {
 355         lttng_enable_kernel_channel 1 0 "$@"
 356 }
 357
 358 function lttng_disable_kernel_channel()
 359 {
 360         local expected_to_fail=$1
 361         local sess_name=$2
 362         local channel_name=$3
 363
 364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 365         ret=$?
 366         if [[ $expected_to_fail -eq "1" ]]; then
 367                 test "$ret" -ne "0"
 368                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 369         else
 370                 ok $ret "Disable channel $channel_name for session $sess_name"
 371         fi
 372 }
 373
 374 function lttng_disable_kernel_channel_ok()
 375 {
 376         lttng_disable_kernel_channel 0 "$@"
 377 }
 378
 379 function lttng_disable_kernel_channel_fail()
 380 {
 381         lttng_disable_kernel_channel 1 "$@"
 382 }
 383
 384 function start_lttng_relayd_opt()
 385 {
 386         local withtap=$1
 387         local opt=$2
 388
 389         DIR=$(readlink -f "$TESTDIR")
 390
 391         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 392                 # shellcheck disable=SC2086
 393                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 394                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 395                 if [ $? -eq 1 ]; then
 396                         if [ "$withtap" -eq "1" ]; then
 397                                 fail "Start lttng-relayd (opt: $opt)"
 398                         fi
 399                         return 1
 400                 else
 401                         if [ "$withtap" -eq "1" ]; then
 402                                 pass "Start lttng-relayd (opt: $opt)"
 403                         fi
 404                 fi
 405         else
 406                 pass "Start lttng-relayd (opt: $opt)"
 407         fi
 408 }
 409
 410 function start_lttng_relayd()
 411 {
 412         start_lttng_relayd_opt 1 "$@"
 413 }
 414
 415 function start_lttng_relayd_notap()
 416 {
 417         start_lttng_relayd_opt 0 "$@"
 418 }
 419
 420 function stop_lttng_relayd_opt()
 421 {
 422         local withtap=$1
 423         local signal=$2
 424
 425         if [ -z "$signal" ]; then
 426                 signal="SIGTERM"
 427         fi
 428
 429         local timeout_s=$3
 430         local dtimeleft_s=
 431
 432         # Multiply time by 2 to simplify integer arithmetic
 433         if [ -n "$timeout_s" ]; then
 434                 dtimeleft_s=$((timeout_s * 2))
 435         fi
 436
 437         local retval=0
 438         local pids=
 439
 440         pids=$(pgrep "$RELAYD_MATCH")
 441         if [ -z "$pids" ]; then
 442                 if [ "$withtap" -eq "1" ]; then
 443                         pass "No relay daemon to kill"
 444                 fi
 445                 return 0
 446         fi
 447
 448         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 449
 450         # shellcheck disable=SC2086
 451         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 452                 retval=1
 453                 if [ "$withtap" -eq "1" ]; then
 454                         fail "Kill relay daemon"
 455                 fi
 456         else
 457                 out=1
 458                 while [ -n "$out" ]; do
 459                         out=$(pgrep "$RELAYD_MATCH")
 460                         if [ -n "$dtimeleft_s" ]; then
 461                                 if [ $dtimeleft_s -lt 0 ]; then
 462                                         out=
 463                                         retval=1
 464                                 fi
 465                                 dtimeleft_s=$((dtimeleft_s - 1))
 466                         fi
 467                         sleep 0.5
 468                 done
 469                 if [ "$withtap" -eq "1" ]; then
 470                         if [ "$retval" -eq "0" ]; then
 471                                 pass "Wait after kill relay daemon"
 472                         else
 473                                 fail "Wait after kill relay daemon"
 474                         fi
 475                 fi
 476         fi
 477         return $retval
 478 }
 479
 480 function stop_lttng_relayd()
 481 {
 482         stop_lttng_relayd_opt 1 "$@"
 483 }
 484
 485 function stop_lttng_relayd_notap()
 486 {
 487         stop_lttng_relayd_opt 0 "$@"
 488 }
 489
 490 #First arg: show tap output
 491 #Second argument: load path for automatic loading
 492 function start_lttng_sessiond_opt()
 493 {
 494         local withtap=$1
 495         local load_path=$2
 496
 497         local env_vars=""
 498         local consumerd=""
 499
 500         local long_bit_value=
 501         long_bit_value=$(getconf LONG_BIT)
 502
 503         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 504                 # Env variable requested no session daemon
 505                 return
 506         fi
 507
 508         DIR=$(readlink -f "$TESTDIR")
 509
 510         # Get long_bit value for 32/64 consumerd
 511         case "$long_bit_value" in
 512                 32)
 513                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 514                         ;;
 515                 64)
 516                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 517                         ;;
 518                 *)
 519                         return
 520                         ;;
 521         esac
 522
 523         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 524         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 525                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 526         fi
 527         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 528
 529         if ! validate_kernel_version; then
 530             fail "Start session daemon"
 531             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 532         fi
 533
 534         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 535         export LTTNG_SESSION_CONFIG_XSD_PATH
 536
 537         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 538                 # Have a load path ?
 539                 if [ -n "$load_path" ]; then
 540                         # shellcheck disable=SC2086
 541                         env $env_vars --load "$load_path" --background "$consumerd"
 542                 else
 543                         # shellcheck disable=SC2086
 544                         env $env_vars --background "$consumerd"
 545                 fi
 546                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 547                 status=$?
 548                 if [ "$withtap" -eq "1" ]; then
 549                         ok $status "Start session daemon"
 550                 fi
 551         fi
 552 }
 553
 554 function start_lttng_sessiond()
 555 {
 556         start_lttng_sessiond_opt 1 "$@"
 557 }
 558
 559 function start_lttng_sessiond_notap()
 560 {
 561         start_lttng_sessiond_opt 0 "$@"
 562 }
 563
 564 function stop_lttng_sessiond_opt()
 565 {
 566         local withtap=$1
 567         local signal=$2
 568
 569         if [ -z "$signal" ]; then
 570                 signal=SIGTERM
 571         fi
 572
 573         local timeout_s=$3
 574         local dtimeleft_s=
 575
 576         # Multiply time by 2 to simplify integer arithmetic
 577         if [ -n "$timeout_s" ]; then
 578                 dtimeleft_s=$((timeout_s * 2))
 579         fi
 580
 581         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 582                 # Env variable requested no session daemon
 583                 return 0
 584         fi
 585
 586         local retval=0
 587
 588         local runas_pids=
 589         runas_pids=$(pgrep "$RUNAS_MATCH")
 590
 591         local pids=
 592         pids=$(pgrep "$SESSIOND_MATCH")
 593
 594         if [ -n "$runas_pids" ]; then
 595                 pids="$pids $runas_pids"
 596         fi
 597
 598         if [ -z "$pids" ]; then
 599                 if [ "$withtap" -eq "1" ]; then
 600                         pass "No session daemon to kill"
 601                 fi
 602                 return 0
 603         fi
 604
 605         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 606
 607         # shellcheck disable=SC2086
 608         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 609                 retval=1
 610                 if [ "$withtap" -eq "1" ]; then
 611                         fail "Kill sessions daemon"
 612                 fi
 613         else
 614                 out=1
 615                 while [ -n "$out" ]; do
 616                         out=$(pgrep "${SESSIOND_MATCH}")
 617                         if [ -n "$dtimeleft_s" ]; then
 618                                 if [ $dtimeleft_s -lt 0 ]; then
 619                                         out=
 620                                         retval=1
 621                                 fi
 622                                 dtimeleft_s=$((dtimeleft_s - 1))
 623                         fi
 624                         sleep 0.5
 625                 done
 626                 out=1
 627                 while [ -n "$out" ]; do
 628                         out=$(pgrep "$CONSUMERD_MATCH")
 629                         if [ -n "$dtimeleft_s" ]; then
 630                                 if [ $dtimeleft_s -lt 0 ]; then
 631                                         out=
 632                                         retval=1
 633                                 fi
 634                                 dtimeleft_s=$((dtimeleft_s - 1))
 635                         fi
 636                         sleep 0.5
 637                 done
 638
 639                 if [ "$withtap" -eq "1" ]; then
 640                         if [ "$retval" -eq "0" ]; then
 641                                 pass "Wait after kill session daemon"
 642                         else
 643                                 fail "Wait after kill session daemon"
 644                         fi
 645                 fi
 646         fi
 647         if [ "$signal" = "SIGKILL" ]; then
 648                 if [ "$(id -u)" -eq "0" ]; then
 649                         local modules=
 650                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 651
 652                         if [ -n "$modules" ]; then
 653                                 diag "Unloading all LTTng modules"
 654                                 modprobe -r "$modules"
 655                         fi
 656                 fi
 657         fi
 658
 659         return $retval
 660 }
 661
 662 function stop_lttng_sessiond()
 663 {
 664         stop_lttng_sessiond_opt 1 "$@"
 665 }
 666
 667 function stop_lttng_sessiond_notap()
 668 {
 669         stop_lttng_sessiond_opt 0 "$@"
 670 }
 671
 672 function sigstop_lttng_sessiond_opt()
 673 {
 674         local withtap=$1
 675         local signal=SIGSTOP
 676
 677         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 678                 # Env variable requested no session daemon
 679                 return
 680         fi
 681
 682         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 683
 684         if [ "$withtap" -eq "1" ]; then
 685                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 686         fi
 687
 688         # shellcheck disable=SC2086
 689         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 690                 if [ "$withtap" -eq "1" ]; then
 691                         fail "Sending SIGSTOP to session daemon"
 692                 fi
 693         else
 694                 out=1
 695                 while [ $out -ne 0 ]; do
 696                         pid="$(pgrep "$SESSIOND_MATCH")"
 697
 698                         # Wait until state becomes stopped for session
 699                         # daemon(s).
 700                         out=0
 701                         for sessiond_pid in $pid; do
 702                                 state="$(ps -p "$sessiond_pid" -o state= )"
 703                                 if [[ -n "$state" && "$state" != "T" ]]; then
 704                                         out=1
 705                                 fi
 706                         done
 707                         sleep 0.5
 708                 done
 709                 if [ "$withtap" -eq "1" ]; then
 710                         pass "Sending SIGSTOP to session daemon"
 711                 fi
 712         fi
 713 }
 714
 715 function sigstop_lttng_sessiond()
 716 {
 717         sigstop_lttng_sessiond_opt 1 "$@"
 718 }
 719
 720 function sigstop_lttng_sessiond_notap()
 721 {
 722         sigstop_lttng_sessiond_opt 0 "$@"
 723 }
 724
 725 function stop_lttng_consumerd_opt()
 726 {
 727         local withtap=$1
 728         local signal=$2
 729
 730         if [ -z "$signal" ]; then
 731                 signal=SIGTERM
 732         fi
 733
 734         local timeout_s=$3
 735         local dtimeleft_s=
 736
 737         # Multiply time by 2 to simplify integer arithmetic
 738         if [ -n "$timeout_s" ]; then
 739                 dtimeleft_s=$((timeout_s * 2))
 740         fi
 741
 742         local retval=0
 743
 744         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 745
 746         if [ -z "$PID_CONSUMERD" ]; then
 747                 if [ "$withtap" -eq "1" ]; then
 748                         pass "No consumer daemon to kill"
 749                 fi
 750                 return 0
 751         fi
 752
 753         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 754
 755         # shellcheck disable=SC2086
 756         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 757                 retval=1
 758                 if [ "$withtap" -eq "1" ]; then
 759                         fail "Kill consumer daemon"
 760                 fi
 761         else
 762                 out=1
 763                 while [ $out -ne 0 ]; do
 764                         pid="$(pgrep "$CONSUMERD_MATCH")"
 765
 766                         # If consumerds are still present check their status.
 767                         # A zombie status qualifies the consumerd as *killed*
 768                         out=0
 769                         for consumer_pid in $pid; do
 770                                 state="$(ps -p "$consumer_pid" -o state= )"
 771                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 772                                         out=1
 773                                 fi
 774                         done
 775                         if [ -n "$dtimeleft_s" ]; then
 776                                 if [ $dtimeleft_s -lt 0 ]; then
 777                                         out=0
 778                                         retval=1
 779                                 fi
 780                                 dtimeleft_s=$((dtimeleft_s - 1))
 781                         fi
 782                         sleep 0.5
 783                 done
 784                 if [ "$withtap" -eq "1" ]; then
 785                         if [ "$retval" -eq "0" ]; then
 786                                 pass "Wait after kill consumer daemon"
 787                         else
 788                                 fail "Wait after kill consumer daemon"
 789                         fi
 790                 fi
 791         fi
 792
 793         return $retval
 794 }
 795
 796 function stop_lttng_consumerd()
 797 {
 798         stop_lttng_consumerd_opt 1 "$@"
 799 }
 800
 801 function stop_lttng_consumerd_notap()
 802 {
 803         stop_lttng_consumerd_opt 0 "$@"
 804 }
 805
 806 function sigstop_lttng_consumerd_opt()
 807 {
 808         local withtap=$1
 809         local signal=SIGSTOP
 810
 811         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 812
 813         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 814
 815         # shellcheck disable=SC2086
 816         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 817         retval=$?
 818
 819         if [ $retval -eq 1 ]; then
 820                 if [ "$withtap" -eq "1" ]; then
 821                         fail "Sending SIGSTOP to consumer daemon"
 822                 fi
 823                 return 1
 824         else
 825                 out=1
 826                 while [ $out -ne 0 ]; do
 827                         pid="$(pgrep "$CONSUMERD_MATCH")"
 828
 829                         # Wait until state becomes stopped for all
 830                         # consumers.
 831                         out=0
 832                         for consumer_pid in $pid; do
 833                                 state="$(ps -p "$consumer_pid" -o state= )"
 834                                 if [[ -n "$state" && "$state" != "T" ]]; then
 835                                         out=1
 836                                 fi
 837                         done
 838                         sleep 0.5
 839                 done
 840                 if [ "$withtap" -eq "1" ]; then
 841                         pass "Sending SIGSTOP to consumer daemon"
 842                 fi
 843         fi
 844         return $retval
 845 }
 846
 847 function sigstop_lttng_consumerd()
 848 {
 849         sigstop_lttng_consumerd_opt 1 "$@"
 850 }
 851
 852 function sigstop_lttng_consumerd_notap()
 853 {
 854         sigstop_lttng_consumerd_opt 0 "$@"
 855 }
 856
 857 function list_lttng_with_opts ()
 858 {
 859         local opts=$1
 860         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 861         ok $? "Lttng-tool list command with option $opts"
 862 }
 863
 864 function create_lttng_session_no_output ()
 865 {
 866         local sess_name=$1
 867         local opts="${@:2}"
 868
 869         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 870         ok $? "Create session $sess_name in no-output mode"
 871 }
 872
 873 function create_lttng_session_uri () {
 874         local sess_name=$1
 875         local uri=$2
 876         local opts="${@:3}"
 877
 878         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 879         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 880 }
 881
 882 function create_lttng_session ()
 883 {
 884         local withtap=$1
 885         local expected_to_fail=$2
 886         local sess_name=$3
 887         local trace_path=$4
 888         local opt=$5
 889
 890         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 891         ret=$?
 892         if [ $expected_to_fail -eq "1" ]; then
 893                 test "$ret" -ne "0"
 894                 ret=$?
 895                 if [ $withtap -eq "1" ]; then
 896                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 897                 fi
 898         else
 899                 if [ $withtap -eq "1" ]; then
 900                         ok $ret "Create session $sess_name in $trace_path"
 901                 fi
 902         fi
 903         return $ret
 904 }
 905
 906 function create_lttng_session_ok ()
 907 {
 908         create_lttng_session 1 0 "$@"
 909 }
 910
 911 function create_lttng_session_fail ()
 912 {
 913         create_lttng_session 1 1 "$@"
 914 }
 915
 916 function create_lttng_session_notap ()
 917 {
 918         create_lttng_session 0 0 "$@"
 919 }
 920
 921
 922 function enable_ust_lttng_channel ()
 923 {
 924         local withtap=$1
 925         local expected_to_fail=$2
 926         local sess_name=$3
 927         local channel_name=$4
 928         local opt=$5
 929
 930         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 931         ret=$?
 932         if [[ $expected_to_fail -eq "1" ]]; then
 933                 test "$ret" -ne "0"
 934                 ret=$?
 935                 if [ $withtap -eq "1" ]; then
 936                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 937                 fi
 938         else
 939                 if [ $withtap -eq "1" ]; then
 940                         ok $ret "Enable channel $channel_name for session $sess_name"
 941                 fi
 942         fi
 943         return $ret
 944 }
 945
 946 function enable_ust_lttng_channel_ok ()
 947 {
 948         enable_ust_lttng_channel 1 0 "$@"
 949 }
 950
 951 function enable_ust_lttng_channel_fail ()
 952 {
 953         enable_ust_lttng_channel 1 1 "$@"
 954 }
 955
 956 function enable_ust_lttng_channel_notap ()
 957 {
 958         enable_ust_lttng_channel 0 0 "$@"
 959 }
 960
 961 function disable_ust_lttng_channel()
 962 {
 963         local sess_name=$1
 964         local channel_name=$2
 965
 966         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 967         ok $? "Disable channel $channel_name for session $sess_name"
 968 }
 969
 970 function enable_lttng_mmap_overwrite_kernel_channel()
 971 {
 972         local sess_name=$1
 973         local channel_name=$2
 974
 975         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 976         ok $? "Enable channel $channel_name for session $sess_name"
 977 }
 978
 979 function enable_lttng_mmap_discard_small_kernel_channel()
 980 {
 981         local sess_name=$1
 982         local channel_name=$2
 983
 984         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 985         ok $? "Enable small discard channel $channel_name for session $sess_name"
 986 }
 987
 988 function enable_lttng_mmap_overwrite_small_kernel_channel()
 989 {
 990         local sess_name=$1
 991         local channel_name=$2
 992
 993         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 994         ok $? "Enable small discard channel $channel_name for session $sess_name"
 995 }
 996
 997 function enable_lttng_mmap_overwrite_ust_channel()
 998 {
 999         local sess_name=$1
1000         local channel_name=$2
1001
1002         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1003         ok $? "Enable channel $channel_name for session $sess_name"
1004 }
1005
1006 function enable_ust_lttng_event ()
1007 {
1008         local withtap=$1
1009         local expected_to_fail=$2
1010         local sess_name=$3
1011         local event_name="$4"
1012         local channel_name=$5
1013
1014         if [ -z $channel_name ]; then
1015                 # default channel if none specified
1016                 chan=""
1017         else
1018                 chan="-c $channel_name"
1019         fi
1020
1021         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1022         ret=$?
1023         if [[ $expected_to_fail -eq "1" ]]; then
1024                 test $ret -ne "0"
1025                 ret=$?
1026                 if [[ $withtap -eq "1" ]]; then
1027                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1028                 fi
1029         else
1030                 if [[ $withtap -eq "1" ]]; then
1031                         ok $ret "Enable ust event $event_name for session $sess_name"
1032                 fi
1033         fi
1034         return $ret
1035 }
1036
1037 function enable_ust_lttng_event_ok ()
1038 {
1039         enable_ust_lttng_event 1 0 "$@"
1040 }
1041
1042 function enable_ust_lttng_event_fail ()
1043 {
1044         enable_ust_lttng_event 1 1 "$@"
1045 }
1046
1047 function enable_ust_lttng_event_notap ()
1048 {
1049         enable_ust_lttng_event 0 0 "$@"
1050 }
1051
1052 function enable_jul_lttng_event()
1053 {
1054         sess_name=$1
1055         event_name="$2"
1056         channel_name=$3
1057
1058         if [ -z $channel_name ]; then
1059                 # default channel if none specified
1060                 chan=""
1061         else
1062                 chan="-c $channel_name"
1063         fi
1064
1065         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1066         ok $? "Enable JUL event $event_name for session $sess_name"
1067 }
1068
1069 function enable_jul_lttng_event_loglevel()
1070 {
1071         local sess_name=$1
1072         local event_name="$2"
1073         local loglevel=$3
1074         local channel_name=$4
1075
1076         if [ -z $channel_name ]; then
1077                 # default channel if none specified
1078                 chan=""
1079         else
1080                 chan="-c $channel_name"
1081         fi
1082
1083         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1084         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1085 }
1086
1087 function enable_log4j_lttng_event()
1088 {
1089         sess_name=$1
1090         event_name="$2"
1091         channel_name=$3
1092
1093         if [ -z $channel_name ]; then
1094                 # default channel if none specified
1095                 chan=""
1096         else
1097                 chan="-c $channel_name"
1098         fi
1099
1100         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1101         ok $? "Enable LOG4J event $event_name for session $sess_name"
1102 }
1103
1104 function enable_log4j_lttng_event_loglevel()
1105 {
1106         local sess_name=$1
1107         local event_name="$2"
1108         local loglevel=$3
1109         local channel_name=$4
1110
1111         if [ -z $channel_name ]; then
1112                 # default channel if none specified
1113                 chan=""
1114         else
1115                 chan="-c $channel_name"
1116         fi
1117
1118         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1119         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1120 }
1121
1122 function enable_python_lttng_event()
1123 {
1124         sess_name=$1
1125         event_name="$2"
1126         channel_name=$3
1127
1128         if [ -z $channel_name ]; then
1129                 # default channel if none specified
1130                 chan=""
1131         else
1132                 chan="-c $channel_name"
1133         fi
1134
1135         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1136         ok $? "Enable Python event $event_name for session $sess_name"
1137 }
1138
1139 function enable_python_lttng_event_loglevel()
1140 {
1141         local sess_name=$1
1142         local event_name="$2"
1143         local loglevel=$3
1144         local channel_name=$4
1145
1146         if [ -z $channel_name ]; then
1147                 # default channel if none specified
1148                 chan=""
1149         else
1150                 chan="-c $channel_name"
1151         fi
1152
1153         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1154         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1155 }
1156
1157 function enable_ust_lttng_event_filter()
1158 {
1159         local sess_name="$1"
1160         local event_name="$2"
1161         local filter="$3"
1162
1163         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1164         ok $? "Enable event $event_name with filtering for session $sess_name"
1165 }
1166
1167 function enable_ust_lttng_event_loglevel()
1168 {
1169         local sess_name="$1"
1170         local event_name="$2"
1171         local loglevel="$3"
1172
1173         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1174         ok $? "Enable event $event_name with loglevel $loglevel"
1175 }
1176
1177 function enable_ust_lttng_event_loglevel_only()
1178 {
1179         local sess_name="$1"
1180         local event_name="$2"
1181         local loglevel="$3"
1182
1183         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1184         ok $? "Enable event $event_name with loglevel-only $loglevel"
1185 }
1186
1187 function disable_ust_lttng_event ()
1188 {
1189         local sess_name="$1"
1190         local event_name="$2"
1191         local channel_name="$3"
1192
1193         if [ -z $channel_name ]; then
1194                 # default channel if none specified
1195                 chan=""
1196         else
1197                 chan="-c $channel_name"
1198         fi
1199
1200         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1201         ok $? "Disable event $event_name for session $sess_name"
1202 }
1203
1204 function disable_jul_lttng_event ()
1205 {
1206         local sess_name="$1"
1207         local event_name="$2"
1208
1209         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1210         ok $? "Disable JUL event $event_name for session $sess_name"
1211 }
1212
1213 function disable_log4j_lttng_event ()
1214 {
1215         local sess_name="$1"
1216         local event_name="$2"
1217
1218         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1219         ok $? "Disable LOG4J event $event_name for session $sess_name"
1220 }
1221
1222 function disable_python_lttng_event ()
1223 {
1224         local sess_name="$1"
1225         local event_name="$2"
1226
1227         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1228         ok $? "Disable Python event $event_name for session $sess_name"
1229 }
1230
1231 function start_lttng_tracing_opt ()
1232 {
1233         local withtap=$1
1234         local expected_to_fail=$2
1235         local sess_name=$3
1236
1237         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1238         ret=$?
1239         if [[ $expected_to_fail -eq "1" ]]; then
1240                 test "$ret" -ne "0"
1241                 ret=$?
1242                 if [ $withtap -eq "1" ]; then
1243                         ok $? "Start tracing for session $sess_name failed as expected"
1244                 fi
1245         else
1246                 if [ $withtap -eq "1" ]; then
1247                         ok $ret "Start tracing for session $sess_name"
1248                 fi
1249         fi
1250 }
1251
1252 function start_lttng_tracing_ok ()
1253 {
1254         start_lttng_tracing_opt 1 0 "$@"
1255 }
1256
1257 function start_lttng_tracing_fail ()
1258 {
1259         start_lttng_tracing_opt 1 1 "$@"
1260 }
1261
1262 function start_lttng_tracing_notap ()
1263 {
1264         start_lttng_tracing_opt 0 1 "$@"
1265 }
1266
1267 function stop_lttng_tracing_opt ()
1268 {
1269         local withtap=$1
1270         local expected_to_fail=$2
1271         local sess_name=$3
1272
1273         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1274         ret=$?
1275         if [[ $expected_to_fail -eq "1" ]]; then
1276                 test "$ret" -ne "0"
1277                 ret=$?
1278                 if [ $withtap -eq "1" ]; then
1279                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1280                 fi
1281         else
1282                 if [ $withtap -eq "1" ]; then
1283                         ok $ret "Stop lttng tracing for session $sess_name"
1284                 fi
1285         fi
1286 }
1287
1288 function stop_lttng_tracing_ok ()
1289 {
1290         stop_lttng_tracing_opt 1 0 "$@"
1291 }
1292
1293 function stop_lttng_tracing_fail ()
1294 {
1295         stop_lttng_tracing_opt 1 1 "$@"
1296 }
1297
1298 function stop_lttng_tracing_notap ()
1299 {
1300         stop_lttng_tracing_opt 0 0 "$@"
1301 }
1302
1303 function destroy_lttng_session ()
1304 {
1305         local withtap=$1
1306         local expected_to_fail=$2
1307         local sess_name=$3
1308
1309         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1310         ret=$?
1311         if [[ $expected_to_fail -eq "1" ]]; then
1312                 test "$ret" -ne "0"
1313                 ret=$?
1314                 if [ $withtap -eq "1" ]; then
1315                         ok $ret "Destroy session $sess_name failed as expected"
1316                 fi
1317         else
1318                 if [ $withtap -eq "1" ]; then
1319                         ok $ret "Destroy session $sess_name"
1320                 fi
1321         fi
1322 }
1323
1324 function destroy_lttng_session_ok ()
1325 {
1326         destroy_lttng_session 1 0 "$@"
1327
1328 }
1329
1330 function destroy_lttng_session_fail ()
1331 {
1332         destroy_lttng_session 1 1 "$@"
1333 }
1334
1335 function destroy_lttng_session_notap ()
1336 {
1337         destroy_lttng_session 0 0 "$@"
1338 }
1339
1340 function destroy_lttng_sessions ()
1341 {
1342         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1343         ok $? "Destroy all lttng sessions"
1344 }
1345
1346 function lttng_snapshot_add_output ()
1347 {
1348         local expected_to_fail=$1
1349         local sess_name=$2
1350         local trace_path=$3
1351
1352         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1353         ret=$?
1354         if [[ $expected_to_fail -eq 1 ]]; then
1355                 test "$ret" -ne "0"
1356                 ok $? "Added snapshot output file://$trace_path failed as expected"
1357         else
1358                 ok $ret "Added snapshot output file://$trace_path"
1359         fi
1360 }
1361
1362 function lttng_snapshot_add_output_ok ()
1363 {
1364         lttng_snapshot_add_output 0 "$@"
1365 }
1366
1367 function lttng_snapshot_add_output_fail ()
1368 {
1369         lttng_snapshot_add_output 1 "$@"
1370 }
1371
1372 function lttng_snapshot_del_output ()
1373 {
1374         local expected_to_fail=$1
1375         local sess_name=$2
1376         local id=$3
1377
1378         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1379         ret=$?
1380         if [[ $expected_to_fail -eq "1" ]]; then
1381                 test "$ret" -ne "0"
1382                 ok $? "Deleted snapshot output id $id failed as expected"
1383         else
1384                 ok $ret "Deleted snapshot output id $id"
1385         fi
1386 }
1387
1388 function lttng_snapshot_del_output_ok ()
1389 {
1390         lttng_snapshot_del_output 0 "$@"
1391 }
1392
1393 function lttng_snapshot_del_output_fail ()
1394 {
1395         lttng_snapshot_del_output 1 "$@"
1396 }
1397
1398 function lttng_snapshot_record ()
1399 {
1400         local sess_name=$1
1401         local trace_path=$2
1402
1403         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1404         ok $? "Snapshot recorded"
1405 }
1406
1407 function lttng_snapshot_list ()
1408 {
1409         local sess_name=$1
1410         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1411         ok $? "Snapshot list"
1412 }
1413
1414 function lttng_save()
1415 {
1416         local sess_name=$1
1417         local opts=$2
1418
1419         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1420         ok $? "Session saved"
1421 }
1422
1423 function lttng_load()
1424 {
1425         local expected_to_fail=$1
1426         local opts=$2
1427
1428         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1429         ret=$?
1430         if [[ $expected_to_fail -eq "1" ]]; then
1431                 test $ret -ne "0"
1432                 ok $? "Load command failed as expected with opts: $opts"
1433         else
1434                 ok $ret "Load command with opts: $opts"
1435         fi
1436 }
1437
1438 function lttng_load_ok()
1439 {
1440         lttng_load 0 "$@"
1441 }
1442
1443 function lttng_load_fail()
1444 {
1445         lttng_load 1 "$@"
1446 }
1447
1448 function lttng_track()
1449 {
1450         local expected_to_fail=$1
1451         local opts=$2
1452         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1453         ret=$?
1454         if [[ $expected_to_fail -eq "1" ]]; then
1455                 test $ret -ne "0"
1456                 ok $? "Track command failed as expected with opts: $opts"
1457         else
1458                 ok $ret "Track command with opts: $opts"
1459         fi
1460 }
1461
1462 function lttng_track_ok()
1463 {
1464         lttng_track 0 "$@"
1465 }
1466
1467 function lttng_track_fail()
1468 {
1469         lttng_track 1 "$@"
1470 }
1471
1472 function lttng_untrack()
1473 {
1474         local expected_to_fail=$1
1475         local opts=$2
1476         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1477         ret=$?
1478         if [[ $expected_to_fail -eq "1" ]]; then
1479                 test $ret -ne "0"
1480                 ok $? "Untrack command failed as expected with opts: $opts"
1481         else
1482                 ok $ret "Untrack command with opts: $opts"
1483         fi
1484 }
1485
1486 function lttng_untrack_ok()
1487 {
1488         lttng_untrack 0 "$@"
1489 }
1490
1491 function lttng_untrack_fail()
1492 {
1493         lttng_untrack 1 "$@"
1494 }
1495
1496 function lttng_track_pid_ok()
1497 {
1498         PID=$1
1499         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1500         ok $? "Lttng track pid on the kernel domain"
1501 }
1502
1503 function lttng_untrack_kernel_all_ok()
1504 {
1505         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1506         ok $? "Lttng untrack all pid on the kernel domain"
1507 }
1508
1509 function lttng_add_context_list()
1510 {
1511         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1512         ret=$?
1513         ok $ret "Context listing"
1514 }
1515
1516 function add_context_lttng()
1517 {
1518         local expected_to_fail="$1"
1519         local domain="$2"
1520         local session_name="$3"
1521         local channel_name="$4"
1522         local type="$5"
1523
1524         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1525         ret=$?
1526         if [[ $expected_to_fail -eq "1" ]]; then
1527                 test $ret -ne "0"
1528                 ok $? "Add context command failed as expected for type: $type"
1529         else
1530                 ok $ret "Add context command for type: $type"
1531         fi
1532 }
1533
1534 function add_context_ust_ok()
1535 {
1536         add_context_lttng 0 -u "$@"
1537 }
1538
1539 function add_context_ust_fail()
1540 {
1541         add_context_lttng 1 -u "$@"
1542 }
1543
1544 function add_context_kernel_ok()
1545 {
1546         add_context_lttng 0 -k "$@"
1547 }
1548
1549 function add_context_kernel_fail()
1550 {
1551         add_context_lttng 1 -k "$@"
1552 }
1553
1554 function validate_metadata_event ()
1555 {
1556         local event_name=$1
1557         local nr_event_id=$2
1558         local trace_path=$3
1559
1560         local metadata_file=$(find $trace_path | grep metadata)
1561         local metadata_path=$(dirname $metadata_file)
1562
1563         which $BABELTRACE_BIN >/dev/null
1564         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1565
1566         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1567
1568         if [ "$count" -ne "$nr_event_id" ]; then
1569                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1570                 diag "$count matching event id found in metadata"
1571         else
1572                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1573         fi
1574
1575 }
1576
1577 function trace_matches ()
1578 {
1579         local event_name=$1
1580         local nr_iter=$2
1581         local trace_path=$3
1582
1583         which $BABELTRACE_BIN >/dev/null
1584         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1585
1586         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1587
1588         if [ "$count" -ne "$nr_iter" ]; then
1589                 fail "Trace match"
1590                 diag "$count matching events found in trace"
1591         else
1592                 pass "Trace match"
1593         fi
1594 }
1595
1596 function trace_match_only()
1597 {
1598         local event_name=$1
1599         local nr_iter=$2
1600         local trace_path=$3
1601
1602         which $BABELTRACE_BIN >/dev/null
1603         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1604
1605         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1606         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1607
1608         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1609                 pass "Trace match with $total event $event_name"
1610         else
1611                 fail "Trace match"
1612                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1613         fi
1614 }
1615
1616 function validate_trace
1617 {
1618         local event_name=$1
1619         local trace_path=$2
1620
1621         which $BABELTRACE_BIN >/dev/null
1622         if [ $? -ne 0 ]; then
1623             skip 0 "Babeltrace binary not found. Skipping trace validation"
1624         fi
1625
1626         OLDIFS=$IFS
1627         IFS=","
1628         for i in $event_name; do
1629                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1630                 if [ "$traced" -ne 0 ]; then
1631                         pass "Validate trace for event $i, $traced events"
1632                 else
1633                         fail "Validate trace for event $i"
1634                         diag "Found $traced occurences of $i"
1635                 fi
1636         done
1637         ret=$?
1638         IFS=$OLDIFS
1639         return $ret
1640 }
1641
1642 function validate_trace_count
1643 {
1644         local event_name=$1
1645         local trace_path=$2
1646         local expected_count=$3
1647
1648         which $BABELTRACE_BIN >/dev/null
1649         if [ $? -ne 0 ]; then
1650             skip 0 "Babeltrace binary not found. Skipping trace validation"
1651         fi
1652
1653         cnt=0
1654         OLDIFS=$IFS
1655         IFS=","
1656         for i in $event_name; do
1657                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1658                 if [ "$traced" -ne 0 ]; then
1659                         pass "Validate trace for event $i, $traced events"
1660                 else
1661                         fail "Validate trace for event $i"
1662                         diag "Found $traced occurences of $i"
1663                 fi
1664                 cnt=$(($cnt + $traced))
1665         done
1666         IFS=$OLDIFS
1667         test $cnt -eq $expected_count
1668         ok $? "Read a total of $cnt events, expected $expected_count"
1669 }
1670
1671 function trace_first_line
1672 {
1673         local trace_path=$1
1674
1675         which $BABELTRACE_BIN >/dev/null
1676         if [ $? -ne 0 ]; then
1677             skip 0 "Babeltrace binary not found. Skipping trace validation"
1678         fi
1679
1680         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1681 }
1682
1683 function validate_trace_exp()
1684 {
1685         local event_exp=$1
1686         local trace_path=$2
1687
1688         which $BABELTRACE_BIN >/dev/null
1689         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1690
1691         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1692         if [ "$traced" -ne 0 ]; then
1693                 pass "Validate trace for expression '${event_exp}', $traced events"
1694         else
1695                 fail "Validate trace for expression '${event_exp}'"
1696                 diag "Found $traced occurences of '${event_exp}'"
1697         fi
1698         ret=$?
1699         return $ret
1700 }
1701
1702 function validate_trace_only_exp()
1703 {
1704         local event_exp=$1
1705         local trace_path=$2
1706
1707         which $BABELTRACE_BIN >/dev/null
1708         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1709
1710         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1711         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1712
1713         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1714                 pass "Trace match with $total for expression '${event_exp}'"
1715         else
1716                 fail "Trace match"
1717                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1718         fi
1719         ret=$?
1720         return $ret
1721 }
1722
1723 function validate_trace_empty()
1724 {
1725         local trace_path=$1
1726
1727         which $BABELTRACE_BIN >/dev/null
1728         if [ $? -ne 0 ]; then
1729             skip 0 "Babeltrace binary not found. Skipping trace validation"
1730         fi
1731
1732         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1733         ret=$?
1734         if [ $ret -ne 0 ]; then
1735                 fail "Failed to parse trace"
1736                 return $ret
1737         fi
1738
1739         traced=$(echo -n "$events" | wc -l)
1740         if [ "$traced" -eq 0 ]; then
1741                 pass "Validate empty trace"
1742         else
1743                 fail "Validate empty trace"
1744                 diag "Found $traced events in trace"
1745         fi
1746         ret=$?
1747         return $ret
1748 }
1749
1750 function regenerate_metadata ()
1751 {
1752         local expected_to_fail=$1
1753         local sess_name=$2
1754
1755         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1756         ret=$?
1757         if [[ $expected_to_fail -eq "1" ]]; then
1758                 test "$ret" -ne "0"
1759                 ok $? "Expected fail on regenerate metadata $sess_name"
1760         else
1761                 ok $ret "Metadata regenerate $sess_name"
1762         fi
1763 }
1764
1765 function regenerate_metadata_ok ()
1766 {
1767         regenerate_metadata 0 "$@"
1768 }
1769
1770 function regenerate_metadata_fail ()
1771 {
1772         regenerate_metadata 1 "$@"
1773 }
1774
1775 function regenerate_statedump ()
1776 {
1777         local expected_to_fail=$1
1778         local sess_name=$2
1779
1780         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1781         ret=$?
1782         if [[ $expected_to_fail -eq "1" ]]; then
1783                 test "$ret" -ne "0"
1784                 ok $? "Expected fail on regenerate statedump $sess_name"
1785         else
1786                 ok $ret "Metadata regenerate $sess_name"
1787         fi
1788 }
1789
1790 function regenerate_statedump_ok ()
1791 {
1792         regenerate_statedump 0 "$@"
1793 }
1794
1795 function regenerate_statedump_fail ()
1796 {
1797         regenerate_statedump 1 "$@"
1798 }
1799
1800 function rotate_session ()
1801 {
1802         local expected_to_fail=$1
1803         local sess_name=$2
1804
1805         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1806         ret=$?
1807         if [[ $expected_to_fail -eq "1" ]]; then
1808                 test "$ret" -ne "0"
1809                 ok $? "Expected fail on rotate session $sess_name"
1810         else
1811                 ok $ret "Rotate session $sess_name"
1812         fi
1813 }
1814
1815 function rotate_session_ok ()
1816 {
1817         rotate_session 0 "$@"
1818 }
1819
1820 function rotate_session_fail ()
1821 {
1822         rotate_session 1 "$@"
1823 }
1824
1825 function destructive_tests_enabled ()
1826 {
1827         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1828                 return 0
1829         else
1830                 return 1
1831         fi
1832 }
1833
1834 function lttng_enable_rotation_timer ()
1835 {
1836         local expected_to_fail=$1
1837         local sess_name=$2
1838         local period=$3
1839
1840         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1841         ret=$?
1842         if [[ $expected_to_fail -eq "1" ]]; then
1843                 test "$ret" -ne "0"
1844                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
1845         else
1846                 ok $ret "Set periodic rotation ($period) of session $sess_name"
1847         fi
1848 }
1849
1850 function lttng_enable_rotation_timer_ok ()
1851 {
1852         lttng_enable_rotation_timer 0 $@
1853 }
1854
1855 function lttng_enable_rotation_timer_fail ()
1856 {
1857         lttng_enable_rotation_timer 1 $@
1858 }
1859
1860 function lttng_enable_rotation_size ()
1861 {
1862         local expected_to_fail=$1
1863         local sess_name=$2
1864         local size=$3
1865
1866         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1867         ret=$?
1868         if [[ $expected_to_fail -eq "1" ]]; then
1869                 test "$ret" -ne "0"
1870                 ok $? "Expected fail on rotate session $sess_name"
1871         else
1872                 ok $ret "Rotate session $sess_name"
1873         fi
1874 }
1875
1876 function lttng_enable_rotation_size_ok ()
1877 {
1878         lttng_enable_rotation_size 0 $@
1879 }
1880
1881 function lttng_enable_rotation_size_fail ()
1882 {
1883         lttng_enable_rotation_size 1 $@
1884 }