tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18
  19 # Minimal kernel version supported for session daemon tests
  20 KERNEL_MAJOR_VERSION=2
  21 KERNEL_MINOR_VERSION=6
  22 KERNEL_PATCHLEVEL_VERSION=27
  23
  24 # We set the default UST register timeout and network and app socket timeout to
  25 # "wait forever", so that basic tests don't have to worry about hitting
  26 # timeouts on busy systems. Specialized tests should test those corner-cases.
  27 export LTTNG_UST_REGISTER_TIMEOUT=-1
  28 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  29 export LTTNG_APP_SOCKET_TIMEOUT=-1
  30
  31 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  32 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  33 # its own sessiond if none is running. It also ensures that 'lttng create'
  34 # fails when no sessiond is running.
  35 export LTTNG_SESSIOND_PATH="/bin/true"
  36
  37 source $TESTDIR/utils/tap/tap.sh
  38
  39 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  40         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  41 fi
  42
  43 function full_cleanup ()
  44 {
  45         # Try to kill daemons gracefully
  46         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  47         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  48
  49         # If daemons are still present, forcibly kill them
  50         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  51         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  52         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  53
  54         # Disable trap for SIGTERM since the following kill to the
  55         # pidgroup will be SIGTERM. Otherwise it loops.
  56         # The '-' before the pid number ($$) indicates 'kill' to signal the
  57         # whole process group.
  58         trap - SIGTERM && kill -- -$$
  59         exit 1
  60 }
  61
  62 function null_pipes ()
  63 {
  64         exec 0>/dev/null
  65         exec 1>/dev/null
  66         exec 2>/dev/null
  67 }
  68
  69 trap full_cleanup SIGINT SIGTERM
  70
  71 # perl prove closes its child pipes before giving it a chance to run its
  72 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  73 # to allow those trap handlers to proceed.
  74
  75 trap null_pipes SIGPIPE
  76
  77 function print_ok ()
  78 {
  79         # Check if we are a terminal
  80         if [ -t 1 ]; then
  81                 echo -e "\e[1;32mOK\e[0m"
  82         else
  83                 echo -e "OK"
  84         fi
  85 }
  86
  87 function print_fail ()
  88 {
  89         # Check if we are a terminal
  90         if [ -t 1 ]; then
  91                 echo -e "\e[1;31mFAIL\e[0m"
  92         else
  93                 echo -e "FAIL"
  94         fi
  95 }
  96
  97 function print_test_banner ()
  98 {
  99         local desc="$1"
 100         diag "$desc"
 101 }
 102
 103 function validate_kernel_version ()
 104 {
 105         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 106         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 107                 return 0
 108         fi
 109         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 110                 return 0
 111         fi
 112         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 113                 return 0
 114         fi
 115         return 1
 116 }
 117
 118 # Generate a random string
 119 #  $1 = number of characters; defaults to 16
 120 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 121 function randstring()
 122 {
 123         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 124         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 125         echo
 126 }
 127
 128 # Return the number of _configured_ CPUs.
 129 function conf_proc_count()
 130 {
 131         getconf _NPROCESSORS_CONF
 132         if [ $? -ne 0 ]; then
 133                 diag "Failed to get the number of configured CPUs"
 134         fi
 135         echo
 136 }
 137
 138 # Check if base lttng-modules are present.
 139 # Bail out on failure
 140 function validate_lttng_modules_present ()
 141 {
 142         modprobe -n lttng-tracer 2>/dev/null
 143         if [ $? -ne 0  ]; then
 144                 BAIL_OUT "LTTng modules not detected."
 145         fi
 146 }
 147
 148 function enable_kernel_lttng_event
 149 {
 150         local withtap="$1"
 151         local expected_to_fail="$2"
 152         local sess_name="$3"
 153         local event_name="$4"
 154         local channel_name="$5"
 155
 156         if [ -z "$event_name" ]; then
 157                 # Enable all event if no event name specified
 158                 event_name="-a"
 159         fi
 160
 161         if [ -z "$channel_name" ]; then
 162                 # default channel if none specified
 163                 chan=""
 164         else
 165                 chan="-c $channel_name"
 166         fi
 167
 168         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 169         ret=$?
 170         if [[ $expected_to_fail -eq "1" ]]; then
 171                 test $ret -ne "0"
 172                 ret=$?
 173                 if [ $withtap -eq "1" ]; then
 174                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 175                 fi
 176         else
 177                 if [ $withtap -eq "1" ]; then
 178                         ok $ret "Enable kernel event $event_name for session $sess_name"
 179                 fi
 180         fi
 181 }
 182
 183 function enable_kernel_lttng_event_ok ()
 184 {
 185         enable_kernel_lttng_event 1 0 "$@"
 186 }
 187
 188 function enable_kernel_lttng_event_fail ()
 189 {
 190         enable_kernel_lttng_event 1 1 "$@"
 191 }
 192
 193 function enable_kernel_lttng_event_notap ()
 194 {
 195         enable_kernel_lttng_event 0 0 "$@"
 196 }
 197
 198 # Old interface
 199 function lttng_enable_kernel_event
 200 {
 201         enable_kernel_lttng_event_ok "$@"
 202 }
 203
 204 function lttng_enable_kernel_syscall()
 205 {
 206         local expected_to_fail=$1
 207         local sess_name=$2
 208         local syscall_name=$3
 209         local channel_name=$4
 210
 211         if [ -z $syscall_name ]; then
 212                 # Enable all event if no syscall name specified
 213                 syscall_name="-a"
 214         fi
 215
 216         if [ -z $channel_name ]; then
 217                 # default channel if none specified
 218                 chan=""
 219         else
 220                 chan="-c $channel_name"
 221         fi
 222
 223         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 224         ret=$?
 225         if [[ $expected_to_fail -eq "1" ]]; then
 226                 test $ret -ne "0"
 227                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 228         else
 229                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 230         fi
 231 }
 232
 233 function lttng_enable_kernel_syscall_ok()
 234 {
 235         lttng_enable_kernel_syscall 0 "$@"
 236 }
 237
 238 function lttng_enable_kernel_syscall_fail()
 239 {
 240         lttng_enable_kernel_syscall 1 "$@"
 241 }
 242
 243 function lttng_disable_kernel_syscall()
 244 {
 245         local expected_to_fail=$1
 246         local sess_name=$2
 247         local syscall_name=$3
 248         local channel_name=$4
 249
 250         if [ -z $syscall_name ]; then
 251                 # Enable all event if no syscall name specified
 252                 syscall_name="-a"
 253         fi
 254
 255         if [ -z $channel_name ]; then
 256                 # default channel if none specified
 257                 chan=""
 258         else
 259                 chan="-c $channel_name"
 260         fi
 261
 262         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 263
 264         ret=$?
 265         if [[ $expected_to_fail -eq "1" ]]; then
 266                 test $ret -ne "0"
 267                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 268         else
 269                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 270         fi
 271 }
 272
 273 function lttng_disable_kernel_syscall_ok()
 274 {
 275         lttng_disable_kernel_syscall 0 "$@"
 276 }
 277
 278 function lttng_disable_kernel_syscall_fail()
 279 {
 280         lttng_disable_kernel_syscall 1 "$@"
 281 }
 282
 283 function lttng_enable_kernel_userspace_probe_event ()
 284 {
 285         local expected_to_fail="$1"
 286         local sess_name="$2"
 287         local target="$3"
 288         local event_name="$4"
 289
 290         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 291         ret=$?
 292         if [[ $expected_to_fail -eq "1" ]]; then
 293                 test $ret -ne "0"
 294                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 295         else
 296                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 297         fi
 298 }
 299
 300 function lttng_enable_kernel_userspace_probe_event_fail ()
 301 {
 302         lttng_enable_kernel_userspace_probe_event 1 "$@"
 303 }
 304
 305 function lttng_enable_kernel_userspace_probe_event_ok ()
 306 {
 307         lttng_enable_kernel_userspace_probe_event 0 "$@"
 308 }
 309
 310 function disable_kernel_lttng_userspace_probe_event_ok ()
 311 {
 312         local sess_name="$1"
 313         local event_name="$2"
 314
 315         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 316         ok $? "Disable kernel event $target for session $sess_name"
 317 }
 318 function lttng_enable_kernel_channel()
 319 {
 320         local withtap=$1
 321         local expected_to_fail=$2
 322         local sess_name=$3
 323         local channel_name=$4
 324         local opts="${@:5}"
 325
 326         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 327         ret=$?
 328         if [[ $expected_to_fail -eq "1" ]]; then
 329                 test "$ret" -ne "0"
 330                 ret=$?
 331                 if [ $withtap -eq "1" ]; then
 332                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 333                 fi
 334         else
 335                 if [ $withtap -eq "1" ]; then
 336                         ok $ret "Enable channel $channel_name for session $sess_name"
 337                 fi
 338         fi
 339 }
 340
 341 function lttng_enable_kernel_channel_ok()
 342 {
 343         lttng_enable_kernel_channel 1 0 "$@"
 344 }
 345
 346 function lttng_enable_kernel_channel_fail()
 347 {
 348         lttng_enable_kernel_channel 1 1 "$@"
 349 }
 350
 351 function lttng_enable_kernel_channel_notap()
 352 {
 353         lttng_enable_kernel_channel 0 0 "$@"
 354 }
 355
 356 function enable_kernel_lttng_channel_ok()
 357 {
 358         lttng_enable_kernel_channel 1 0 "$@"
 359 }
 360
 361 function lttng_disable_kernel_channel()
 362 {
 363         local expected_to_fail=$1
 364         local sess_name=$2
 365         local channel_name=$3
 366
 367         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 368         ret=$?
 369         if [[ $expected_to_fail -eq "1" ]]; then
 370                 test "$ret" -ne "0"
 371                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 372         else
 373                 ok $ret "Disable channel $channel_name for session $sess_name"
 374         fi
 375 }
 376
 377 function lttng_disable_kernel_channel_ok()
 378 {
 379         lttng_disable_kernel_channel 0 "$@"
 380 }
 381
 382 function lttng_disable_kernel_channel_fail()
 383 {
 384         lttng_disable_kernel_channel 1 "$@"
 385 }
 386
 387 function start_lttng_relayd_opt()
 388 {
 389         local withtap=$1
 390         local process_mode=$2
 391         local opt=$3
 392
 393         DIR=$(readlink -f "$TESTDIR")
 394
 395         if [ -z $(pgrep $RELAYD_MATCH) ]; then
 396                 # shellcheck disable=SC2086
 397                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 398                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 399                 if [ $? -eq 1 ]; then
 400                         if [ $withtap -eq "1" ]; then
 401                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 402                         fi
 403                         return 1
 404                 else
 405                         if [ $withtap -eq "1" ]; then
 406                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 407                         fi
 408                 fi
 409         else
 410                 pass "Start lttng-relayd (opt: $opt)"
 411         fi
 412 }
 413
 414 function start_lttng_relayd()
 415 {
 416         start_lttng_relayd_opt 1 "-b" "$@"
 417 }
 418
 419 function start_lttng_relayd_notap()
 420 {
 421         start_lttng_relayd_opt 0 "-b" "$@"
 422 }
 423
 424 function stop_lttng_relayd_opt()
 425 {
 426         local withtap=$1
 427         local signal=$2
 428
 429         if [ -z "$signal" ]; then
 430                 signal="SIGTERM"
 431         fi
 432
 433         local timeout_s=$3
 434         local dtimeleft_s=
 435
 436         # Multiply time by 2 to simplify integer arithmetic
 437         if [ -n "$timeout_s" ]; then
 438                 dtimeleft_s=$((timeout_s * 2))
 439         fi
 440
 441         local retval=0
 442         local pids=
 443
 444         pids=$(pgrep "$RELAYD_MATCH")
 445         if [ -z "$pids" ]; then
 446                 if [ "$withtap" -eq "1" ]; then
 447                         pass "No relay daemon to kill"
 448                 fi
 449                 return 0
 450         fi
 451
 452         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 453
 454         # shellcheck disable=SC2086
 455         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 456                 retval=1
 457                 if [ "$withtap" -eq "1" ]; then
 458                         fail "Kill relay daemon"
 459                 fi
 460         else
 461                 out=1
 462                 while [ -n "$out" ]; do
 463                         out=$(pgrep "$RELAYD_MATCH")
 464                         if [ -n "$dtimeleft_s" ]; then
 465                                 if [ $dtimeleft_s -lt 0 ]; then
 466                                         out=
 467                                         retval=1
 468                                 fi
 469                                 dtimeleft_s=$((dtimeleft_s - 1))
 470                         fi
 471                         sleep 0.5
 472                 done
 473                 if [ "$withtap" -eq "1" ]; then
 474                         if [ "$retval" -eq "0" ]; then
 475                                 pass "Wait after kill relay daemon"
 476                         else
 477                                 fail "Wait after kill relay daemon"
 478                         fi
 479                 fi
 480         fi
 481         return $retval
 482 }
 483
 484 function stop_lttng_relayd()
 485 {
 486         stop_lttng_relayd_opt 1 "$@"
 487 }
 488
 489 function stop_lttng_relayd_notap()
 490 {
 491         stop_lttng_relayd_opt 0 "$@"
 492 }
 493
 494 #First arg: show tap output
 495 #Second argument: load path for automatic loading
 496 function start_lttng_sessiond_opt()
 497 {
 498         local withtap=$1
 499         local load_path=$2
 500
 501         local env_vars=""
 502         local consumerd=""
 503
 504         local long_bit_value=
 505         long_bit_value=$(getconf LONG_BIT)
 506
 507         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 508                 # Env variable requested no session daemon
 509                 return
 510         fi
 511
 512         DIR=$(readlink -f "$TESTDIR")
 513
 514         # Get long_bit value for 32/64 consumerd
 515         case "$long_bit_value" in
 516                 32)
 517                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 518                         ;;
 519                 64)
 520                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 521                         ;;
 522                 *)
 523                         return
 524                         ;;
 525         esac
 526
 527         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 528         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 529                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 530         fi
 531         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 532
 533         if ! validate_kernel_version; then
 534             fail "Start session daemon"
 535             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 536         fi
 537
 538         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 539         export LTTNG_SESSION_CONFIG_XSD_PATH
 540
 541         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 542                 # Have a load path ?
 543                 if [ -n "$load_path" ]; then
 544                         # shellcheck disable=SC2086
 545                         env $env_vars --load "$load_path" --background "$consumerd"
 546                 else
 547                         # shellcheck disable=SC2086
 548                         env $env_vars --background "$consumerd"
 549                 fi
 550                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 551                 status=$?
 552                 if [ "$withtap" -eq "1" ]; then
 553                         ok $status "Start session daemon"
 554                 fi
 555         fi
 556 }
 557
 558 function start_lttng_sessiond()
 559 {
 560         start_lttng_sessiond_opt 1 "$@"
 561 }
 562
 563 function start_lttng_sessiond_notap()
 564 {
 565         start_lttng_sessiond_opt 0 "$@"
 566 }
 567
 568 function stop_lttng_sessiond_opt()
 569 {
 570         local withtap=$1
 571         local signal=$2
 572
 573         if [ -z "$signal" ]; then
 574                 signal=SIGTERM
 575         fi
 576
 577         local timeout_s=$3
 578         local dtimeleft_s=
 579
 580         # Multiply time by 2 to simplify integer arithmetic
 581         if [ -n "$timeout_s" ]; then
 582                 dtimeleft_s=$((timeout_s * 2))
 583         fi
 584
 585         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 586                 # Env variable requested no session daemon
 587                 return 0
 588         fi
 589
 590         local retval=0
 591
 592         local runas_pids=
 593         runas_pids=$(pgrep "$RUNAS_MATCH")
 594
 595         local pids=
 596         pids=$(pgrep "$SESSIOND_MATCH")
 597
 598         if [ -n "$runas_pids" ]; then
 599                 pids="$pids $runas_pids"
 600         fi
 601
 602         if [ -z "$pids" ]; then
 603                 if [ "$withtap" -eq "1" ]; then
 604                         pass "No session daemon to kill"
 605                 fi
 606                 return 0
 607         fi
 608
 609         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 610
 611         # shellcheck disable=SC2086
 612         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 613                 retval=1
 614                 if [ "$withtap" -eq "1" ]; then
 615                         fail "Kill sessions daemon"
 616                 fi
 617         else
 618                 out=1
 619                 while [ -n "$out" ]; do
 620                         out=$(pgrep "${SESSIOND_MATCH}")
 621                         if [ -n "$dtimeleft_s" ]; then
 622                                 if [ $dtimeleft_s -lt 0 ]; then
 623                                         out=
 624                                         retval=1
 625                                 fi
 626                                 dtimeleft_s=$((dtimeleft_s - 1))
 627                         fi
 628                         sleep 0.5
 629                 done
 630                 out=1
 631                 while [ -n "$out" ]; do
 632                         out=$(pgrep "$CONSUMERD_MATCH")
 633                         if [ -n "$dtimeleft_s" ]; then
 634                                 if [ $dtimeleft_s -lt 0 ]; then
 635                                         out=
 636                                         retval=1
 637                                 fi
 638                                 dtimeleft_s=$((dtimeleft_s - 1))
 639                         fi
 640                         sleep 0.5
 641                 done
 642
 643                 if [ "$withtap" -eq "1" ]; then
 644                         if [ "$retval" -eq "0" ]; then
 645                                 pass "Wait after kill session daemon"
 646                         else
 647                                 fail "Wait after kill session daemon"
 648                         fi
 649                 fi
 650         fi
 651         if [ "$signal" = "SIGKILL" ]; then
 652                 if [ "$(id -u)" -eq "0" ]; then
 653                         local modules=
 654                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 655
 656                         if [ -n "$modules" ]; then
 657                                 diag "Unloading all LTTng modules"
 658                                 modprobe -r "$modules"
 659                         fi
 660                 fi
 661         fi
 662
 663         return $retval
 664 }
 665
 666 function stop_lttng_sessiond()
 667 {
 668         stop_lttng_sessiond_opt 1 "$@"
 669 }
 670
 671 function stop_lttng_sessiond_notap()
 672 {
 673         stop_lttng_sessiond_opt 0 "$@"
 674 }
 675
 676 function sigstop_lttng_sessiond_opt()
 677 {
 678         local withtap=$1
 679         local signal=SIGSTOP
 680
 681         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 682                 # Env variable requested no session daemon
 683                 return
 684         fi
 685
 686         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 687
 688         if [ "$withtap" -eq "1" ]; then
 689                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 690         fi
 691
 692         # shellcheck disable=SC2086
 693         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 694                 if [ "$withtap" -eq "1" ]; then
 695                         fail "Sending SIGSTOP to session daemon"
 696                 fi
 697         else
 698                 out=1
 699                 while [ $out -ne 0 ]; do
 700                         pid="$(pgrep "$SESSIOND_MATCH")"
 701
 702                         # Wait until state becomes stopped for session
 703                         # daemon(s).
 704                         out=0
 705                         for sessiond_pid in $pid; do
 706                                 state="$(ps -p "$sessiond_pid" -o state= )"
 707                                 if [[ -n "$state" && "$state" != "T" ]]; then
 708                                         out=1
 709                                 fi
 710                         done
 711                         sleep 0.5
 712                 done
 713                 if [ "$withtap" -eq "1" ]; then
 714                         pass "Sending SIGSTOP to session daemon"
 715                 fi
 716         fi
 717 }
 718
 719 function sigstop_lttng_sessiond()
 720 {
 721         sigstop_lttng_sessiond_opt 1 "$@"
 722 }
 723
 724 function sigstop_lttng_sessiond_notap()
 725 {
 726         sigstop_lttng_sessiond_opt 0 "$@"
 727 }
 728
 729 function stop_lttng_consumerd_opt()
 730 {
 731         local withtap=$1
 732         local signal=$2
 733
 734         if [ -z "$signal" ]; then
 735                 signal=SIGTERM
 736         fi
 737
 738         local timeout_s=$3
 739         local dtimeleft_s=
 740
 741         # Multiply time by 2 to simplify integer arithmetic
 742         if [ -n "$timeout_s" ]; then
 743                 dtimeleft_s=$((timeout_s * 2))
 744         fi
 745
 746         local retval=0
 747
 748         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 749
 750         if [ -z "$PID_CONSUMERD" ]; then
 751                 if [ "$withtap" -eq "1" ]; then
 752                         pass "No consumer daemon to kill"
 753                 fi
 754                 return 0
 755         fi
 756
 757         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 758
 759         # shellcheck disable=SC2086
 760         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 761                 retval=1
 762                 if [ "$withtap" -eq "1" ]; then
 763                         fail "Kill consumer daemon"
 764                 fi
 765         else
 766                 out=1
 767                 while [ $out -ne 0 ]; do
 768                         pid="$(pgrep "$CONSUMERD_MATCH")"
 769
 770                         # If consumerds are still present check their status.
 771                         # A zombie status qualifies the consumerd as *killed*
 772                         out=0
 773                         for consumer_pid in $pid; do
 774                                 state="$(ps -p "$consumer_pid" -o state= )"
 775                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 776                                         out=1
 777                                 fi
 778                         done
 779                         if [ -n "$dtimeleft_s" ]; then
 780                                 if [ $dtimeleft_s -lt 0 ]; then
 781                                         out=0
 782                                         retval=1
 783                                 fi
 784                                 dtimeleft_s=$((dtimeleft_s - 1))
 785                         fi
 786                         sleep 0.5
 787                 done
 788                 if [ "$withtap" -eq "1" ]; then
 789                         if [ "$retval" -eq "0" ]; then
 790                                 pass "Wait after kill consumer daemon"
 791                         else
 792                                 fail "Wait after kill consumer daemon"
 793                         fi
 794                 fi
 795         fi
 796
 797         return $retval
 798 }
 799
 800 function stop_lttng_consumerd()
 801 {
 802         stop_lttng_consumerd_opt 1 "$@"
 803 }
 804
 805 function stop_lttng_consumerd_notap()
 806 {
 807         stop_lttng_consumerd_opt 0 "$@"
 808 }
 809
 810 function sigstop_lttng_consumerd_opt()
 811 {
 812         local withtap=$1
 813         local signal=SIGSTOP
 814
 815         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 816
 817         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 818
 819         # shellcheck disable=SC2086
 820         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 821         retval=$?
 822
 823         if [ $retval -eq 1 ]; then
 824                 if [ "$withtap" -eq "1" ]; then
 825                         fail "Sending SIGSTOP to consumer daemon"
 826                 fi
 827                 return 1
 828         else
 829                 out=1
 830                 while [ $out -ne 0 ]; do
 831                         pid="$(pgrep "$CONSUMERD_MATCH")"
 832
 833                         # Wait until state becomes stopped for all
 834                         # consumers.
 835                         out=0
 836                         for consumer_pid in $pid; do
 837                                 state="$(ps -p "$consumer_pid" -o state= )"
 838                                 if [[ -n "$state" && "$state" != "T" ]]; then
 839                                         out=1
 840                                 fi
 841                         done
 842                         sleep 0.5
 843                 done
 844                 if [ "$withtap" -eq "1" ]; then
 845                         pass "Sending SIGSTOP to consumer daemon"
 846                 fi
 847         fi
 848         return $retval
 849 }
 850
 851 function sigstop_lttng_consumerd()
 852 {
 853         sigstop_lttng_consumerd_opt 1 "$@"
 854 }
 855
 856 function sigstop_lttng_consumerd_notap()
 857 {
 858         sigstop_lttng_consumerd_opt 0 "$@"
 859 }
 860
 861 function list_lttng_with_opts ()
 862 {
 863         local opts=$1
 864         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 865         ok $? "Lttng-tool list command with option $opts"
 866 }
 867
 868 function create_lttng_session_no_output ()
 869 {
 870         local sess_name=$1
 871         local opts="${@:2}"
 872
 873         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 874         ok $? "Create session $sess_name in no-output mode"
 875 }
 876
 877 function create_lttng_session_uri () {
 878         local sess_name=$1
 879         local uri=$2
 880         local opts="${@:3}"
 881
 882         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 883         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 884 }
 885
 886 function create_lttng_session ()
 887 {
 888         local withtap=$1
 889         local expected_to_fail=$2
 890         local sess_name=$3
 891         local trace_path=$4
 892         local opt=$5
 893
 894         if [ -z "$trace_path" ]; then
 895                 # Use lttng-sessiond default output.
 896                 trace_path=""
 897         else
 898                 trace_path="-o $trace_path"
 899         fi
 900
 901         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create "$sess_name" $trace_path $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 902         ret=$?
 903         if [ $expected_to_fail -eq "1" ]; then
 904                 test "$ret" -ne "0"
 905                 ret=$?
 906                 if [ $withtap -eq "1" ]; then
 907                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 908                 fi
 909         else
 910                 if [ $withtap -eq "1" ]; then
 911                         ok $ret "Create session $sess_name in $trace_path"
 912                 fi
 913         fi
 914         return $ret
 915 }
 916
 917 function create_lttng_session_ok ()
 918 {
 919         create_lttng_session 1 0 "$@"
 920 }
 921
 922 function create_lttng_session_fail ()
 923 {
 924         create_lttng_session 1 1 "$@"
 925 }
 926
 927 function create_lttng_session_notap ()
 928 {
 929         create_lttng_session 0 0 "$@"
 930 }
 931
 932
 933 function enable_ust_lttng_channel ()
 934 {
 935         local withtap=$1
 936         local expected_to_fail=$2
 937         local sess_name=$3
 938         local channel_name=$4
 939         local opts="${@:5}"
 940
 941         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 942         ret=$?
 943         if [[ $expected_to_fail -eq "1" ]]; then
 944                 test "$ret" -ne "0"
 945                 ret=$?
 946                 if [ $withtap -eq "1" ]; then
 947                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 948                 fi
 949         else
 950                 if [ $withtap -eq "1" ]; then
 951                         ok $ret "Enable channel $channel_name for session $sess_name"
 952                 fi
 953         fi
 954         return $ret
 955 }
 956
 957 function enable_ust_lttng_channel_ok ()
 958 {
 959         enable_ust_lttng_channel 1 0 "$@"
 960 }
 961
 962 function enable_ust_lttng_channel_fail ()
 963 {
 964         enable_ust_lttng_channel 1 1 "$@"
 965 }
 966
 967 function enable_ust_lttng_channel_notap ()
 968 {
 969         enable_ust_lttng_channel 0 0 "$@"
 970 }
 971
 972 function disable_ust_lttng_channel()
 973 {
 974         local sess_name=$1
 975         local channel_name=$2
 976
 977         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 978         ok $? "Disable channel $channel_name for session $sess_name"
 979 }
 980
 981 function enable_lttng_mmap_overwrite_kernel_channel()
 982 {
 983         local sess_name=$1
 984         local channel_name=$2
 985
 986         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 987         ok $? "Enable channel $channel_name for session $sess_name"
 988 }
 989
 990 function enable_lttng_mmap_discard_small_kernel_channel()
 991 {
 992         local sess_name=$1
 993         local channel_name=$2
 994
 995         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 996         ok $? "Enable small discard channel $channel_name for session $sess_name"
 997 }
 998
 999 function enable_lttng_mmap_overwrite_small_kernel_channel()
1000 {
1001         local sess_name=$1
1002         local channel_name=$2
1003
1004         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1005         ok $? "Enable small discard channel $channel_name for session $sess_name"
1006 }
1007
1008 function enable_lttng_mmap_overwrite_ust_channel()
1009 {
1010         local sess_name=$1
1011         local channel_name=$2
1012
1013         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1014         ok $? "Enable channel $channel_name for session $sess_name"
1015 }
1016
1017 function enable_ust_lttng_event ()
1018 {
1019         local withtap=$1
1020         local expected_to_fail=$2
1021         local sess_name=$3
1022         local event_name="$4"
1023         local channel_name=$5
1024
1025         if [ -z $channel_name ]; then
1026                 # default channel if none specified
1027                 chan=""
1028         else
1029                 chan="-c $channel_name"
1030         fi
1031
1032         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1033         ret=$?
1034         if [[ $expected_to_fail -eq "1" ]]; then
1035                 test $ret -ne "0"
1036                 ret=$?
1037                 if [[ $withtap -eq "1" ]]; then
1038                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1039                 fi
1040         else
1041                 if [[ $withtap -eq "1" ]]; then
1042                         ok $ret "Enable ust event $event_name for session $sess_name"
1043                 fi
1044         fi
1045         return $ret
1046 }
1047
1048 function enable_ust_lttng_event_ok ()
1049 {
1050         enable_ust_lttng_event 1 0 "$@"
1051 }
1052
1053 function enable_ust_lttng_event_fail ()
1054 {
1055         enable_ust_lttng_event 1 1 "$@"
1056 }
1057
1058 function enable_ust_lttng_event_notap ()
1059 {
1060         enable_ust_lttng_event 0 0 "$@"
1061 }
1062
1063 function enable_jul_lttng_event()
1064 {
1065         sess_name=$1
1066         event_name="$2"
1067         channel_name=$3
1068
1069         if [ -z $channel_name ]; then
1070                 # default channel if none specified
1071                 chan=""
1072         else
1073                 chan="-c $channel_name"
1074         fi
1075
1076         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1077         ok $? "Enable JUL event $event_name for session $sess_name"
1078 }
1079
1080 function enable_jul_lttng_event_loglevel()
1081 {
1082         local sess_name=$1
1083         local event_name="$2"
1084         local loglevel=$3
1085         local channel_name=$4
1086
1087         if [ -z $channel_name ]; then
1088                 # default channel if none specified
1089                 chan=""
1090         else
1091                 chan="-c $channel_name"
1092         fi
1093
1094         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1095         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1096 }
1097
1098 function enable_log4j_lttng_event()
1099 {
1100         sess_name=$1
1101         event_name="$2"
1102         channel_name=$3
1103
1104         if [ -z $channel_name ]; then
1105                 # default channel if none specified
1106                 chan=""
1107         else
1108                 chan="-c $channel_name"
1109         fi
1110
1111         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1112         ok $? "Enable LOG4J event $event_name for session $sess_name"
1113 }
1114
1115 function enable_log4j_lttng_event_loglevel()
1116 {
1117         local sess_name=$1
1118         local event_name="$2"
1119         local loglevel=$3
1120         local channel_name=$4
1121
1122         if [ -z $channel_name ]; then
1123                 # default channel if none specified
1124                 chan=""
1125         else
1126                 chan="-c $channel_name"
1127         fi
1128
1129         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1130         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1131 }
1132
1133 function enable_python_lttng_event()
1134 {
1135         sess_name=$1
1136         event_name="$2"
1137         channel_name=$3
1138
1139         if [ -z $channel_name ]; then
1140                 # default channel if none specified
1141                 chan=""
1142         else
1143                 chan="-c $channel_name"
1144         fi
1145
1146         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1147         ok $? "Enable Python event $event_name for session $sess_name"
1148 }
1149
1150 function enable_python_lttng_event_loglevel()
1151 {
1152         local sess_name=$1
1153         local event_name="$2"
1154         local loglevel=$3
1155         local channel_name=$4
1156
1157         if [ -z $channel_name ]; then
1158                 # default channel if none specified
1159                 chan=""
1160         else
1161                 chan="-c $channel_name"
1162         fi
1163
1164         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1165         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1166 }
1167
1168 function enable_ust_lttng_event_filter()
1169 {
1170         local sess_name="$1"
1171         local event_name="$2"
1172         local filter="$3"
1173
1174         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1175         ok $? "Enable event $event_name with filtering for session $sess_name"
1176 }
1177
1178 function enable_ust_lttng_event_loglevel()
1179 {
1180         local sess_name="$1"
1181         local event_name="$2"
1182         local loglevel="$3"
1183
1184         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1185         ok $? "Enable event $event_name with loglevel $loglevel"
1186 }
1187
1188 function enable_ust_lttng_event_loglevel_only()
1189 {
1190         local sess_name="$1"
1191         local event_name="$2"
1192         local loglevel="$3"
1193
1194         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1195         ok $? "Enable event $event_name with loglevel-only $loglevel"
1196 }
1197
1198 function disable_ust_lttng_event ()
1199 {
1200         local sess_name="$1"
1201         local event_name="$2"
1202         local channel_name="$3"
1203
1204         if [ -z $channel_name ]; then
1205                 # default channel if none specified
1206                 chan=""
1207         else
1208                 chan="-c $channel_name"
1209         fi
1210
1211         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1212         ok $? "Disable event $event_name for session $sess_name"
1213 }
1214
1215 function disable_jul_lttng_event ()
1216 {
1217         local sess_name="$1"
1218         local event_name="$2"
1219
1220         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1221         ok $? "Disable JUL event $event_name for session $sess_name"
1222 }
1223
1224 function disable_log4j_lttng_event ()
1225 {
1226         local sess_name="$1"
1227         local event_name="$2"
1228
1229         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1230         ok $? "Disable LOG4J event $event_name for session $sess_name"
1231 }
1232
1233 function disable_python_lttng_event ()
1234 {
1235         local sess_name="$1"
1236         local event_name="$2"
1237
1238         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1239         ok $? "Disable Python event $event_name for session $sess_name"
1240 }
1241
1242 function start_lttng_tracing_opt ()
1243 {
1244         local withtap=$1
1245         local expected_to_fail=$2
1246         local sess_name=$3
1247
1248         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1249         ret=$?
1250         if [[ $expected_to_fail -eq "1" ]]; then
1251                 test "$ret" -ne "0"
1252                 ret=$?
1253                 if [ $withtap -eq "1" ]; then
1254                         ok $? "Start tracing for session $sess_name failed as expected"
1255                 fi
1256         else
1257                 if [ $withtap -eq "1" ]; then
1258                         ok $ret "Start tracing for session $sess_name"
1259                 fi
1260         fi
1261 }
1262
1263 function start_lttng_tracing_ok ()
1264 {
1265         start_lttng_tracing_opt 1 0 "$@"
1266 }
1267
1268 function start_lttng_tracing_fail ()
1269 {
1270         start_lttng_tracing_opt 1 1 "$@"
1271 }
1272
1273 function start_lttng_tracing_notap ()
1274 {
1275         start_lttng_tracing_opt 0 1 "$@"
1276 }
1277
1278 function stop_lttng_tracing_opt ()
1279 {
1280         local withtap=$1
1281         local expected_to_fail=$2
1282         local sess_name=$3
1283
1284         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1285         ret=$?
1286         if [[ $expected_to_fail -eq "1" ]]; then
1287                 test "$ret" -ne "0"
1288                 ret=$?
1289                 if [ $withtap -eq "1" ]; then
1290                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1291                 fi
1292         else
1293                 if [ $withtap -eq "1" ]; then
1294                         ok $ret "Stop lttng tracing for session $sess_name"
1295                 fi
1296         fi
1297 }
1298
1299 function stop_lttng_tracing_ok ()
1300 {
1301         stop_lttng_tracing_opt 1 0 "$@"
1302 }
1303
1304 function stop_lttng_tracing_fail ()
1305 {
1306         stop_lttng_tracing_opt 1 1 "$@"
1307 }
1308
1309 function stop_lttng_tracing_notap ()
1310 {
1311         stop_lttng_tracing_opt 0 0 "$@"
1312 }
1313
1314 function destroy_lttng_session ()
1315 {
1316         local withtap=$1
1317         local expected_to_fail=$2
1318         local sess_name=$3
1319
1320         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1321         ret=$?
1322         if [[ $expected_to_fail -eq "1" ]]; then
1323                 test "$ret" -ne "0"
1324                 ret=$?
1325                 if [ $withtap -eq "1" ]; then
1326                         ok $ret "Destroy session $sess_name failed as expected"
1327                 fi
1328         else
1329                 if [ $withtap -eq "1" ]; then
1330                         ok $ret "Destroy session $sess_name"
1331                 fi
1332         fi
1333 }
1334
1335 function destroy_lttng_session_ok ()
1336 {
1337         destroy_lttng_session 1 0 "$@"
1338
1339 }
1340
1341 function destroy_lttng_session_fail ()
1342 {
1343         destroy_lttng_session 1 1 "$@"
1344 }
1345
1346 function destroy_lttng_session_notap ()
1347 {
1348         destroy_lttng_session 0 0 "$@"
1349 }
1350
1351 function destroy_lttng_sessions ()
1352 {
1353         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1354         ok $? "Destroy all lttng sessions"
1355 }
1356
1357 function lttng_snapshot_add_output ()
1358 {
1359         local expected_to_fail=$1
1360         local sess_name=$2
1361         local trace_path=$3
1362         local opts=$4
1363
1364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1365         ret=$?
1366         if [[ $expected_to_fail -eq 1 ]]; then
1367                 test "$ret" -ne "0"
1368                 ok $? "Added snapshot output $trace_path failed as expected"
1369         else
1370                 ok $ret "Added snapshot output $trace_path"
1371         fi
1372 }
1373
1374 function lttng_snapshot_add_output_ok ()
1375 {
1376         lttng_snapshot_add_output 0 "$@"
1377 }
1378
1379 function lttng_snapshot_add_output_fail ()
1380 {
1381         lttng_snapshot_add_output 1 "$@"
1382 }
1383
1384 function lttng_snapshot_del_output ()
1385 {
1386         local expected_to_fail=$1
1387         local sess_name=$2
1388         local id=$3
1389
1390         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1391         ret=$?
1392         if [[ $expected_to_fail -eq "1" ]]; then
1393                 test "$ret" -ne "0"
1394                 ok $? "Deleted snapshot output id $id failed as expected"
1395         else
1396                 ok $ret "Deleted snapshot output id $id"
1397         fi
1398 }
1399
1400 function lttng_snapshot_del_output_ok ()
1401 {
1402         lttng_snapshot_del_output 0 "$@"
1403 }
1404
1405 function lttng_snapshot_del_output_fail ()
1406 {
1407         lttng_snapshot_del_output 1 "$@"
1408 }
1409
1410 function lttng_snapshot_record ()
1411 {
1412         local sess_name=$1
1413
1414         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1415         ok $? "Snapshot recorded"
1416 }
1417
1418 function lttng_snapshot_list ()
1419 {
1420         local sess_name=$1
1421         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1422         ok $? "Snapshot list"
1423 }
1424
1425 function lttng_save()
1426 {
1427         local sess_name=$1
1428         local opts=$2
1429
1430         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1431         ok $? "Session saved"
1432 }
1433
1434 function lttng_load()
1435 {
1436         local expected_to_fail=$1
1437         local opts=$2
1438
1439         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1440         ret=$?
1441         if [[ $expected_to_fail -eq "1" ]]; then
1442                 test $ret -ne "0"
1443                 ok $? "Load command failed as expected with opts: $opts"
1444         else
1445                 ok $ret "Load command with opts: $opts"
1446         fi
1447 }
1448
1449 function lttng_load_ok()
1450 {
1451         lttng_load 0 "$@"
1452 }
1453
1454 function lttng_load_fail()
1455 {
1456         lttng_load 1 "$@"
1457 }
1458
1459 function lttng_track()
1460 {
1461         local expected_to_fail="$1"
1462         shift 1
1463         local opts="$@"
1464         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1465         ret=$?
1466         if [[ $expected_to_fail -eq "1" ]]; then
1467                 test $ret -ne "0"
1468                 ok $? "Track command failed as expected with opts: $opts"
1469         else
1470                 ok $ret "Track command with opts: $opts"
1471         fi
1472 }
1473
1474 function lttng_track_ok()
1475 {
1476         lttng_track 0 "$@"
1477 }
1478
1479 function lttng_track_fail()
1480 {
1481         lttng_track 1 "$@"
1482 }
1483
1484 function lttng_untrack()
1485 {
1486         local expected_to_fail="$1"
1487         shift 1
1488         local opts="$@"
1489         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1490         ret=$?
1491         if [[ $expected_to_fail -eq "1" ]]; then
1492                 test $ret -ne "0"
1493                 ok $? "Untrack command failed as expected with opts: $opts"
1494         else
1495                 ok $ret "Untrack command with opts: $opts"
1496         fi
1497 }
1498
1499 function lttng_untrack_ok()
1500 {
1501         lttng_untrack 0 "$@"
1502 }
1503
1504 function lttng_untrack_fail()
1505 {
1506         lttng_untrack 1 "$@"
1507 }
1508
1509 function lttng_track_pid_ok()
1510 {
1511         PID=$1
1512         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1513         ok $? "Lttng track pid on the kernel domain"
1514 }
1515
1516 function lttng_untrack_kernel_all_ok()
1517 {
1518         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1519         ok $? "Lttng untrack all pid on the kernel domain"
1520 }
1521
1522 function lttng_track_ust_ok()
1523 {
1524         lttng_track_ok -u "$@"
1525 }
1526
1527 function lttng_track_ust_fail()
1528 {
1529         lttng_track_fail -u "$@"
1530 }
1531
1532 function lttng_track_kernel_ok()
1533 {
1534         lttng_track_ok -k "$@"
1535 }
1536
1537 function lttng_track_kernel_fail()
1538 {
1539         lttng_track_fail -k "$@"
1540 }
1541
1542 function lttng_untrack_ust_ok()
1543 {
1544         lttng_untrack_ok -u "$@"
1545 }
1546
1547 function lttng_untrack_ust_fail()
1548 {
1549         lttng_untrack_fail -u "$@"
1550 }
1551
1552 function lttng_untrack_kernel_ok()
1553 {
1554         lttng_untrack_ok -k "$@"
1555 }
1556
1557 function lttng_untrack_kernel_fail()
1558 {
1559         lttng_untrack_fail -k "$@"
1560 }
1561
1562 function lttng_add_context_list()
1563 {
1564         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1565         ret=$?
1566         ok $ret "Context listing"
1567 }
1568
1569 function add_context_lttng()
1570 {
1571         local expected_to_fail="$1"
1572         local domain="$2"
1573         local session_name="$3"
1574         local channel_name="$4"
1575         local type="$5"
1576
1577         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1578         ret=$?
1579         if [[ $expected_to_fail -eq "1" ]]; then
1580                 test $ret -ne "0"
1581                 ok $? "Add context command failed as expected for type: $type"
1582         else
1583                 ok $ret "Add context command for type: $type"
1584         fi
1585 }
1586
1587 function add_context_ust_ok()
1588 {
1589         add_context_lttng 0 -u "$@"
1590 }
1591
1592 function add_context_ust_fail()
1593 {
1594         add_context_lttng 1 -u "$@"
1595 }
1596
1597 function add_context_kernel_ok()
1598 {
1599         add_context_lttng 0 -k "$@"
1600 }
1601
1602 function add_context_kernel_fail()
1603 {
1604         add_context_lttng 1 -k "$@"
1605 }
1606
1607 function validate_directory_empty ()
1608 {
1609         local trace_path=$1
1610
1611         ls -A $local_path > /dev/null 2>&1
1612         if [ $? -eq 0 ]; then
1613                 pass "Directory empty"
1614         else
1615                 fail "Directory empty"
1616         fi
1617 }
1618
1619 function wait_live_trace_ready ()
1620 {
1621         local url=$1
1622         local zero_client_match=0
1623
1624         diag "Waiting for live trace at url: $url"
1625         while [ $zero_client_match -eq 0 ]; do
1626                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1627                 sleep 0.5
1628         done
1629         pass "Waiting for live trace at url: $url"
1630 }
1631
1632 function wait_live_viewer_connect ()
1633 {
1634         local url=$1
1635         local one_client_match=0
1636
1637         diag "Waiting for live viewers on url: $url"
1638         while [ $one_client_match -eq 0 ]; do
1639                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1640                 sleep 0.5
1641         done
1642         pass "Waiting for live viewers on url: $url"
1643 }
1644
1645 function validate_metadata_event ()
1646 {
1647         local event_name=$1
1648         local nr_event_id=$2
1649         local trace_path=$3
1650
1651         local metadata_file=$(find $trace_path | grep metadata)
1652         local metadata_path=$(dirname $metadata_file)
1653
1654         which $BABELTRACE_BIN >/dev/null
1655         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1656
1657         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1658
1659         if [ "$count" -ne "$nr_event_id" ]; then
1660                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1661                 diag "$count matching event id found in metadata"
1662         else
1663                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1664         fi
1665
1666 }
1667
1668 function trace_matches ()
1669 {
1670         local event_name=$1
1671         local nr_iter=$2
1672         local trace_path=$3
1673
1674         which $BABELTRACE_BIN >/dev/null
1675         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1676
1677         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1678
1679         if [ "$count" -ne "$nr_iter" ]; then
1680                 fail "Trace match"
1681                 diag "$count matching events found in trace"
1682         else
1683                 pass "Trace match"
1684         fi
1685 }
1686
1687 function trace_match_only()
1688 {
1689         local event_name=$1
1690         local nr_iter=$2
1691         local trace_path=$3
1692
1693         which $BABELTRACE_BIN >/dev/null
1694         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1695
1696         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1697         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1698
1699         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1700                 pass "Trace match with $total event $event_name"
1701         else
1702                 fail "Trace match"
1703                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1704         fi
1705 }
1706
1707 function validate_trace
1708 {
1709         local event_name=$1
1710         local trace_path=$2
1711
1712         which $BABELTRACE_BIN >/dev/null
1713         if [ $? -ne 0 ]; then
1714             skip 0 "Babeltrace binary not found. Skipping trace validation"
1715         fi
1716
1717         OLDIFS=$IFS
1718         IFS=","
1719         for i in $event_name; do
1720                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1721                 if [ "$traced" -ne 0 ]; then
1722                         pass "Validate trace for event $i, $traced events"
1723                 else
1724                         fail "Validate trace for event $i"
1725                         diag "Found $traced occurences of $i"
1726                 fi
1727         done
1728         ret=$?
1729         IFS=$OLDIFS
1730         return $ret
1731 }
1732
1733 function validate_trace_count
1734 {
1735         local event_name=$1
1736         local trace_path=$2
1737         local expected_count=$3
1738
1739         which $BABELTRACE_BIN >/dev/null
1740         if [ $? -ne 0 ]; then
1741             skip 0 "Babeltrace binary not found. Skipping trace validation"
1742         fi
1743
1744         cnt=0
1745         OLDIFS=$IFS
1746         IFS=","
1747         for i in $event_name; do
1748                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1749                 if [ "$traced" -ne 0 ]; then
1750                         pass "Validate trace for event $i, $traced events"
1751                 else
1752                         fail "Validate trace for event $i"
1753                         diag "Found $traced occurences of $i"
1754                 fi
1755                 cnt=$(($cnt + $traced))
1756         done
1757         IFS=$OLDIFS
1758         test $cnt -eq $expected_count
1759         ok $? "Read a total of $cnt events, expected $expected_count"
1760 }
1761
1762 function validate_trace_count_range_incl_min_excl_max
1763 {
1764         local event_name=$1
1765         local trace_path=$2
1766         local expected_min=$3
1767         local expected_max=$4
1768
1769         which $BABELTRACE_BIN >/dev/null
1770         if [ $? -ne 0 ]; then
1771             skip 0 "Babeltrace binary not found. Skipping trace validation"
1772         fi
1773
1774         cnt=0
1775         OLDIFS=$IFS
1776         IFS=","
1777         for i in $event_name; do
1778                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1779                 if [ "$traced" -ge $expected_min ]; then
1780                         pass "Validate trace for event $i, $traced events"
1781                 else
1782                         fail "Validate trace for event $i"
1783                         diag "Found $traced occurences of $i"
1784                 fi
1785                 cnt=$(($cnt + $traced))
1786         done
1787         IFS=$OLDIFS
1788         test $cnt -lt $expected_max
1789         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
1790 }
1791
1792 function trace_first_line
1793 {
1794         local trace_path=$1
1795
1796         which $BABELTRACE_BIN >/dev/null
1797         if [ $? -ne 0 ]; then
1798             skip 0 "Babeltrace binary not found. Skipping trace validation"
1799         fi
1800
1801         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1802 }
1803
1804 function validate_trace_exp()
1805 {
1806         local event_exp=$1
1807         local trace_path=$2
1808
1809         which $BABELTRACE_BIN >/dev/null
1810         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1811
1812         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1813         if [ "$traced" -ne 0 ]; then
1814                 pass "Validate trace for expression '${event_exp}', $traced events"
1815         else
1816                 fail "Validate trace for expression '${event_exp}'"
1817                 diag "Found $traced occurences of '${event_exp}'"
1818         fi
1819         ret=$?
1820         return $ret
1821 }
1822
1823 function validate_trace_only_exp()
1824 {
1825         local event_exp=$1
1826         local trace_path=$2
1827
1828         which $BABELTRACE_BIN >/dev/null
1829         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1830
1831         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1832         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1833
1834         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1835                 pass "Trace match with $total for expression '${event_exp}'"
1836         else
1837                 fail "Trace match"
1838                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1839         fi
1840         ret=$?
1841         return $ret
1842 }
1843
1844 function validate_trace_empty()
1845 {
1846         local trace_path=$1
1847
1848         which $BABELTRACE_BIN >/dev/null
1849         if [ $? -ne 0 ]; then
1850             skip 0 "Babeltrace binary not found. Skipping trace validation"
1851         fi
1852
1853         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1854         ret=$?
1855         if [ $ret -ne 0 ]; then
1856                 fail "Failed to parse trace"
1857                 return $ret
1858         fi
1859
1860         traced=$(echo -n "$events" | wc -l)
1861         if [ "$traced" -eq 0 ]; then
1862                 pass "Validate empty trace"
1863         else
1864                 fail "Validate empty trace"
1865                 diag "Found $traced events in trace"
1866         fi
1867         ret=$?
1868         return $ret
1869 }
1870
1871 function validate_folder_is_empty()
1872 {
1873         local folder=$1
1874
1875         test -z "$(ls -A "$folder")"
1876         ok $? "Folder ${folder} is empty"
1877 }
1878
1879 function validate_trace_session_ust_empty()
1880 {
1881         validate_folder_is_empty "$1"/ust
1882 }
1883
1884 function validate_trace_session_kernel_empty()
1885 {
1886         validate_trace_empty "$1"/kernel
1887 }
1888
1889 function regenerate_metadata ()
1890 {
1891         local expected_to_fail=$1
1892         local sess_name=$2
1893
1894         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1895         ret=$?
1896         if [[ $expected_to_fail -eq "1" ]]; then
1897                 test "$ret" -ne "0"
1898                 ok $? "Expected fail on regenerate metadata $sess_name"
1899         else
1900                 ok $ret "Metadata regenerate $sess_name"
1901         fi
1902 }
1903
1904 function regenerate_metadata_ok ()
1905 {
1906         regenerate_metadata 0 "$@"
1907 }
1908
1909 function regenerate_metadata_fail ()
1910 {
1911         regenerate_metadata 1 "$@"
1912 }
1913
1914 function regenerate_statedump ()
1915 {
1916         local expected_to_fail=$1
1917         local sess_name=$2
1918
1919         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1920         ret=$?
1921         if [[ $expected_to_fail -eq "1" ]]; then
1922                 test "$ret" -ne "0"
1923                 ok $? "Expected fail on regenerate statedump $sess_name"
1924         else
1925                 ok $ret "Statedump regenerate $sess_name"
1926         fi
1927 }
1928
1929 function regenerate_statedump_ok ()
1930 {
1931         regenerate_statedump 0 "$@"
1932 }
1933
1934 function regenerate_statedump_fail ()
1935 {
1936         regenerate_statedump 1 "$@"
1937 }
1938
1939 function rotate_session ()
1940 {
1941         local expected_to_fail=$1
1942         local sess_name=$2
1943
1944         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1945         ret=$?
1946         if [[ $expected_to_fail -eq "1" ]]; then
1947                 test "$ret" -ne "0"
1948                 ok $? "Expected fail on rotate session $sess_name"
1949         else
1950                 ok $ret "Rotate session $sess_name"
1951         fi
1952 }
1953
1954 function rotate_session_ok ()
1955 {
1956         rotate_session 0 "$@"
1957 }
1958
1959 function rotate_session_fail ()
1960 {
1961         rotate_session 1 "$@"
1962 }
1963
1964 function destructive_tests_enabled ()
1965 {
1966         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1967                 return 0
1968         else
1969                 return 1
1970         fi
1971 }
1972
1973 function lttng_enable_rotation_timer ()
1974 {
1975         local expected_to_fail=$1
1976         local sess_name=$2
1977         local period=$3
1978
1979         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1980         ret=$?
1981         if [[ $expected_to_fail -eq "1" ]]; then
1982                 test "$ret" -ne "0"
1983                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
1984         else
1985                 ok $ret "Set periodic rotation ($period) of session $sess_name"
1986         fi
1987 }
1988
1989 function lttng_enable_rotation_timer_ok ()
1990 {
1991         lttng_enable_rotation_timer 0 $@
1992 }
1993
1994 function lttng_enable_rotation_timer_fail ()
1995 {
1996         lttng_enable_rotation_timer 1 $@
1997 }
1998
1999 function lttng_enable_rotation_size ()
2000 {
2001         local expected_to_fail=$1
2002         local sess_name=$2
2003         local size=$3
2004
2005         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2006         ret=$?
2007         if [[ $expected_to_fail -eq "1" ]]; then
2008                 test "$ret" -ne "0"
2009                 ok $? "Expected fail on rotate session $sess_name"
2010         else
2011                 ok $ret "Rotate session $sess_name"
2012         fi
2013 }
2014
2015 function lttng_enable_rotation_size_ok ()
2016 {
2017         lttng_enable_rotation_size 0 $@
2018 }
2019
2020 function lttng_enable_rotation_size_fail ()
2021 {
2022         lttng_enable_rotation_size 1 $@
2023 }
2024
2025 function lttng_clear_session ()
2026 {
2027         local expected_to_fail=$1
2028         local sess_name=$2
2029
2030         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2031         ret=$?
2032         if [[ $expected_to_fail -eq "1" ]]; then
2033                 test "$ret" -ne "0"
2034                 ok $? "Expected fail on clear session $sess_name"
2035         else
2036                 ok $ret "Clear session $sess_name"
2037         fi
2038 }
2039
2040 function lttng_clear_session_ok ()
2041 {
2042         lttng_clear_session 0 $@
2043 }
2044
2045 function lttng_clear_session_fail ()
2046 {
2047         lttng_clear_session 1 $@
2048 }
2049
2050 function lttng_clear_all ()
2051 {
2052         $TESTDIR/../src/bin/lttng/$LTTNG_BIN clear --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
2053         ok $? "Clear all lttng sessions"
2054 }