tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 RUNAS_BIN="lttng-runas"
  19 RUNAS_MATCH=".*lttng-runas.*"
  20 CONSUMERD_BIN="lttng-consumerd"
  21 CONSUMERD_MATCH=".*lttng-consumerd.*"
  22 RELAYD_BIN="lttng-relayd"
  23 RELAYD_MATCH=".*lttng-relayd.*"
  24 LTTNG_BIN="lttng"
  25 BABELTRACE_BIN="babeltrace"
  26 OUTPUT_DEST=/dev/null
  27 ERROR_OUTPUT_DEST=/dev/null
  28
  29 # Minimal kernel version supported for session daemon tests
  30 KERNEL_MAJOR_VERSION=2
  31 KERNEL_MINOR_VERSION=6
  32 KERNEL_PATCHLEVEL_VERSION=27
  33
  34 # We set the default UST register timeout and network and app socket timeout to
  35 # "wait forever", so that basic tests don't have to worry about hitting
  36 # timeouts on busy systems. Specialized tests should test those corner-cases.
  37 export LTTNG_UST_REGISTER_TIMEOUT=-1
  38 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  39 export LTTNG_APP_SOCKET_TIMEOUT=-1
  40
  41 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  42 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  43 # its own sessiond if none is running. It also ensures that 'lttng create'
  44 # fails when no sessiond is running.
  45 export LTTNG_SESSIOND_PATH="/bin/true"
  46
  47 source $TESTDIR/utils/tap/tap.sh
  48
  49 if [ -z $LTTNG_TEST_TEARDOWN_TIMEOUT ]; then
  50         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  51 fi
  52
  53 function full_cleanup ()
  54 {
  55         # Try to kill daemons gracefully
  56         stop_lttng_relayd_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  57         stop_lttng_sessiond_notap SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  58
  59         # If daemons are still present, forcibly kill them
  60         stop_lttng_relayd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  61         stop_lttng_sessiond_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  62         stop_lttng_consumerd_notap SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  63
  64         # Disable trap for SIGTERM since the following kill to the
  65         # pidgroup will be SIGTERM. Otherwise it loops.
  66         # The '-' before the pid number ($$) indicates 'kill' to signal the
  67         # whole process group.
  68         trap - SIGTERM && kill -- -$$
  69 }
  70
  71 function null_pipes ()
  72 {
  73         exec 0>/dev/null
  74         exec 1>/dev/null
  75         exec 2>/dev/null
  76 }
  77
  78 trap full_cleanup SIGINT SIGTERM
  79
  80 # perl prove closes its child pipes before giving it a chance to run its
  81 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
  82 # to allow those trap handlers to proceed.
  83
  84 trap null_pipes SIGPIPE
  85
  86 function print_ok ()
  87 {
  88         # Check if we are a terminal
  89         if [ -t 1 ]; then
  90                 echo -e "\e[1;32mOK\e[0m"
  91         else
  92                 echo -e "OK"
  93         fi
  94 }
  95
  96 function print_fail ()
  97 {
  98         # Check if we are a terminal
  99         if [ -t 1 ]; then
 100                 echo -e "\e[1;31mFAIL\e[0m"
 101         else
 102                 echo -e "FAIL"
 103         fi
 104 }
 105
 106 function print_test_banner ()
 107 {
 108         local desc="$1"
 109         diag "$desc"
 110 }
 111
 112 function validate_kernel_version ()
 113 {
 114         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 115         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 116                 return 0
 117         fi
 118         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 119                 return 0
 120         fi
 121         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 122                 return 0
 123         fi
 124         return 1
 125 }
 126
 127 # Generate a random string
 128 #  $1 = number of characters; defaults to 16
 129 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 130 function randstring()
 131 {
 132         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 133         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 134         echo
 135 }
 136
 137 # Return the number of _configured_ CPUs.
 138 function conf_proc_count()
 139 {
 140         getconf _NPROCESSORS_CONF
 141         if [ $? -ne 0 ]; then
 142                 diag "Failed to get the number of configured CPUs"
 143         fi
 144         echo
 145 }
 146
 147 # Check if base lttng-modules are present.
 148 # Bail out on failure
 149 function validate_lttng_modules_present ()
 150 {
 151         modprobe -n lttng-tracer 2>/dev/null
 152         if [ $? -ne 0  ]; then
 153                 BAIL_OUT "LTTng modules not detected."
 154         fi
 155 }
 156
 157 function enable_kernel_lttng_event
 158 {
 159         local withtap="$1"
 160         local expected_to_fail="$2"
 161         local sess_name="$3"
 162         local event_name="$4"
 163         local channel_name="$5"
 164
 165         if [ -z "$event_name" ]; then
 166                 # Enable all event if no event name specified
 167                 event_name="-a"
 168         fi
 169
 170         if [ -z "$channel_name" ]; then
 171                 # default channel if none specified
 172                 chan=""
 173         else
 174                 chan="-c $channel_name"
 175         fi
 176
 177         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 178         ret=$?
 179         if [[ $expected_to_fail -eq "1" ]]; then
 180                 test $ret -ne "0"
 181                 ret=$?
 182                 if [ $withtap -eq "1" ]; then
 183                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 184                 fi
 185         else
 186                 if [ $withtap -eq "1" ]; then
 187                         ok $ret "Enable kernel event $event_name for session $sess_name"
 188                 fi
 189         fi
 190 }
 191
 192 function enable_kernel_lttng_event_ok ()
 193 {
 194         enable_kernel_lttng_event 1 0 "$@"
 195 }
 196
 197 function enable_kernel_lttng_event_fail ()
 198 {
 199         enable_kernel_lttng_event 1 1 "$@"
 200 }
 201
 202 function enable_kernel_lttng_event_notap ()
 203 {
 204         enable_kernel_lttng_event 0 0 "$@"
 205 }
 206
 207 # Old interface
 208 function lttng_enable_kernel_event
 209 {
 210         enable_kernel_lttng_event_ok "$@"
 211 }
 212
 213 function lttng_enable_kernel_syscall()
 214 {
 215         local expected_to_fail=$1
 216         local sess_name=$2
 217         local syscall_name=$3
 218         local channel_name=$4
 219
 220         if [ -z $syscall_name ]; then
 221                 # Enable all event if no syscall name specified
 222                 syscall_name="-a"
 223         fi
 224
 225         if [ -z $channel_name ]; then
 226                 # default channel if none specified
 227                 chan=""
 228         else
 229                 chan="-c $channel_name"
 230         fi
 231
 232         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 233         ret=$?
 234         if [[ $expected_to_fail -eq "1" ]]; then
 235                 test $ret -ne "0"
 236                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 237         else
 238                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 239         fi
 240 }
 241
 242 function lttng_enable_kernel_syscall_ok()
 243 {
 244         lttng_enable_kernel_syscall 0 "$@"
 245 }
 246
 247 function lttng_enable_kernel_syscall_fail()
 248 {
 249         lttng_enable_kernel_syscall 1 "$@"
 250 }
 251
 252 function lttng_disable_kernel_syscall()
 253 {
 254         local expected_to_fail=$1
 255         local sess_name=$2
 256         local syscall_name=$3
 257         local channel_name=$4
 258
 259         if [ -z $syscall_name ]; then
 260                 # Enable all event if no syscall name specified
 261                 syscall_name="-a"
 262         fi
 263
 264         if [ -z $channel_name ]; then
 265                 # default channel if none specified
 266                 chan=""
 267         else
 268                 chan="-c $channel_name"
 269         fi
 270
 271         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 272
 273         ret=$?
 274         if [[ $expected_to_fail -eq "1" ]]; then
 275                 test $ret -ne "0"
 276                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 277         else
 278                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 279         fi
 280 }
 281
 282 function lttng_disable_kernel_syscall_ok()
 283 {
 284         lttng_disable_kernel_syscall 0 "$@"
 285 }
 286
 287 function lttng_disable_kernel_syscall_fail()
 288 {
 289         lttng_disable_kernel_syscall 1 "$@"
 290 }
 291
 292 function lttng_enable_kernel_userspace_probe_event ()
 293 {
 294         local expected_to_fail="$1"
 295         local sess_name="$2"
 296         local target="$3"
 297         local event_name="$4"
 298
 299         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 300         ret=$?
 301         if [[ $expected_to_fail -eq "1" ]]; then
 302                 test $ret -ne "0"
 303                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 304         else
 305                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 306         fi
 307 }
 308
 309 function lttng_enable_kernel_userspace_probe_event_fail ()
 310 {
 311         lttng_enable_kernel_userspace_probe_event 1 "$@"
 312 }
 313
 314 function lttng_enable_kernel_userspace_probe_event_ok ()
 315 {
 316         lttng_enable_kernel_userspace_probe_event 0 "$@"
 317 }
 318
 319 function disable_kernel_lttng_userspace_probe_event_ok ()
 320 {
 321         local sess_name="$1"
 322         local event_name="$2"
 323
 324         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 325         ok $? "Disable kernel event $target for session $sess_name"
 326 }
 327 function lttng_enable_kernel_channel()
 328 {
 329         local withtap=$1
 330         local expected_to_fail=$2
 331         local sess_name=$3
 332         local channel_name=$4
 333         local opt=$5
 334
 335         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 336         ret=$?
 337         if [[ $expected_to_fail -eq "1" ]]; then
 338                 test "$ret" -ne "0"
 339                 ret=$?
 340                 if [ $withtap -eq "1" ]; then
 341                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 342                 fi
 343         else
 344                 if [ $withtap -eq "1" ]; then
 345                         ok $ret "Enable channel $channel_name for session $sess_name"
 346                 fi
 347         fi
 348 }
 349
 350 function lttng_enable_kernel_channel_ok()
 351 {
 352         lttng_enable_kernel_channel 1 0 "$@"
 353 }
 354
 355 function lttng_enable_kernel_channel_fail()
 356 {
 357         lttng_enable_kernel_channel 1 1 "$@"
 358 }
 359
 360 function lttng_enable_kernel_channel_notap()
 361 {
 362         lttng_enable_kernel_channel 0 0 "$@"
 363 }
 364
 365 function enable_kernel_lttng_channel_ok()
 366 {
 367         lttng_enable_kernel_channel 1 0 "$@"
 368 }
 369
 370 function lttng_disable_kernel_channel()
 371 {
 372         local expected_to_fail=$1
 373         local sess_name=$2
 374         local channel_name=$3
 375
 376         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 377         ret=$?
 378         if [[ $expected_to_fail -eq "1" ]]; then
 379                 test "$ret" -ne "0"
 380                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 381         else
 382                 ok $ret "Disable channel $channel_name for session $sess_name"
 383         fi
 384 }
 385
 386 function lttng_disable_kernel_channel_ok()
 387 {
 388         lttng_disable_kernel_channel 0 "$@"
 389 }
 390
 391 function lttng_disable_kernel_channel_fail()
 392 {
 393         lttng_disable_kernel_channel 1 "$@"
 394 }
 395
 396 function start_lttng_relayd_opt()
 397 {
 398         local withtap=$1
 399         local opt=$2
 400
 401         DIR=$(readlink -f "$TESTDIR")
 402
 403         if [ -z "$(pgrep "$RELAYD_MATCH")" ]; then
 404                 # shellcheck disable=SC2086
 405                 "$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 406                 #"$DIR/../src/bin/lttng-relayd/$RELAYD_BIN" $opt -vvv >>/tmp/relayd.log 2>&1 &
 407                 if [ $? -eq 1 ]; then
 408                         if [ "$withtap" -eq "1" ]; then
 409                                 fail "Start lttng-relayd (opt: $opt)"
 410                         fi
 411                         return 1
 412                 else
 413                         if [ "$withtap" -eq "1" ]; then
 414                                 pass "Start lttng-relayd (opt: $opt)"
 415                         fi
 416                 fi
 417         else
 418                 pass "Start lttng-relayd (opt: $opt)"
 419         fi
 420 }
 421
 422 function start_lttng_relayd()
 423 {
 424         start_lttng_relayd_opt 1 "$@"
 425 }
 426
 427 function start_lttng_relayd_notap()
 428 {
 429         start_lttng_relayd_opt 0 "$@"
 430 }
 431
 432 function stop_lttng_relayd_opt()
 433 {
 434         local withtap=$1
 435         local signal=$2
 436
 437         if [ -z "$signal" ]; then
 438                 signal="SIGTERM"
 439         fi
 440
 441         local timeout_s=$3
 442         local dtimeleft_s=
 443
 444         # Multiply time by 2 to simplify integer arithmetic
 445         if [ -n "$timeout_s" ]; then
 446                 dtimeleft_s=$((timeout_s * 2))
 447         fi
 448
 449         local retval=0
 450         local pids=
 451
 452         pids=$(pgrep "$RELAYD_MATCH")
 453         if [ -z "$pids" ]; then
 454                 if [ "$withtap" -eq "1" ]; then
 455                         pass "No relay daemon to kill"
 456                 fi
 457                 return 0
 458         fi
 459
 460         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 461
 462         # shellcheck disable=SC2086
 463         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 464                 retval=1
 465                 if [ "$withtap" -eq "1" ]; then
 466                         fail "Kill relay daemon"
 467                 fi
 468         else
 469                 out=1
 470                 while [ -n "$out" ]; do
 471                         out=$(pgrep "$RELAYD_MATCH")
 472                         if [ -n "$dtimeleft_s" ]; then
 473                                 if [ $dtimeleft_s -lt 0 ]; then
 474                                         out=
 475                                         retval=1
 476                                 fi
 477                                 dtimeleft_s=$((dtimeleft_s - 1))
 478                         fi
 479                         sleep 0.5
 480                 done
 481                 if [ "$withtap" -eq "1" ]; then
 482                         if [ "$retval" -eq "0" ]; then
 483                                 pass "Wait after kill relay daemon"
 484                         else
 485                                 fail "Wait after kill relay daemon"
 486                         fi
 487                 fi
 488         fi
 489         return $retval
 490 }
 491
 492 function stop_lttng_relayd()
 493 {
 494         stop_lttng_relayd_opt 1 "$@"
 495 }
 496
 497 function stop_lttng_relayd_notap()
 498 {
 499         stop_lttng_relayd_opt 0 "$@"
 500 }
 501
 502 #First arg: show tap output
 503 #Second argument: load path for automatic loading
 504 function start_lttng_sessiond_opt()
 505 {
 506         local withtap=$1
 507         local load_path=$2
 508
 509         local env_vars=""
 510         local consumerd=""
 511
 512         local long_bit_value=
 513         long_bit_value=$(getconf LONG_BIT)
 514
 515         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 516                 # Env variable requested no session daemon
 517                 return
 518         fi
 519
 520         DIR=$(readlink -f "$TESTDIR")
 521
 522         # Get long_bit value for 32/64 consumerd
 523         case "$long_bit_value" in
 524                 32)
 525                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 526                         ;;
 527                 64)
 528                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 529                         ;;
 530                 *)
 531                         return
 532                         ;;
 533         esac
 534
 535         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 536         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 537                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 538         fi
 539         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 540
 541         if ! validate_kernel_version; then
 542             fail "Start session daemon"
 543             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 544         fi
 545
 546         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/config/"}"
 547         export LTTNG_SESSION_CONFIG_XSD_PATH
 548
 549         if [ -z "$(pgrep "${SESSIOND_MATCH}")" ]; then
 550                 # Have a load path ?
 551                 if [ -n "$load_path" ]; then
 552                         # shellcheck disable=SC2086
 553                         env $env_vars --load "$load_path" --background "$consumerd"
 554                 else
 555                         # shellcheck disable=SC2086
 556                         env $env_vars --background "$consumerd"
 557                 fi
 558                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 559                 status=$?
 560                 if [ "$withtap" -eq "1" ]; then
 561                         ok $status "Start session daemon"
 562                 fi
 563         fi
 564 }
 565
 566 function start_lttng_sessiond()
 567 {
 568         start_lttng_sessiond_opt 1 "$@"
 569 }
 570
 571 function start_lttng_sessiond_notap()
 572 {
 573         start_lttng_sessiond_opt 0 "$@"
 574 }
 575
 576 function stop_lttng_sessiond_opt()
 577 {
 578         local withtap=$1
 579         local signal=$2
 580
 581         if [ -z "$signal" ]; then
 582                 signal=SIGTERM
 583         fi
 584
 585         local timeout_s=$3
 586         local dtimeleft_s=
 587
 588         # Multiply time by 2 to simplify integer arithmetic
 589         if [ -n "$timeout_s" ]; then
 590                 dtimeleft_s=$((timeout_s * 2))
 591         fi
 592
 593         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 594                 # Env variable requested no session daemon
 595                 return 0
 596         fi
 597
 598         local retval=0
 599
 600         local runas_pids=
 601         runas_pids=$(pgrep "$RUNAS_MATCH")
 602
 603         local pids=
 604         pids=$(pgrep "$SESSIOND_MATCH")
 605
 606         if [ -n "$runas_pids" ]; then
 607                 pids="$pids $runas_pids"
 608         fi
 609
 610         if [ -z "$pids" ]; then
 611                 if [ "$withtap" -eq "1" ]; then
 612                         pass "No session daemon to kill"
 613                 fi
 614                 return 0
 615         fi
 616
 617         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 618
 619         # shellcheck disable=SC2086
 620         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 621                 retval=1
 622                 if [ "$withtap" -eq "1" ]; then
 623                         fail "Kill sessions daemon"
 624                 fi
 625         else
 626                 out=1
 627                 while [ -n "$out" ]; do
 628                         out=$(pgrep "${SESSIOND_MATCH}")
 629                         if [ -n "$dtimeleft_s" ]; then
 630                                 if [ $dtimeleft_s -lt 0 ]; then
 631                                         out=
 632                                         retval=1
 633                                 fi
 634                                 dtimeleft_s=$((dtimeleft_s - 1))
 635                         fi
 636                         sleep 0.5
 637                 done
 638                 out=1
 639                 while [ -n "$out" ]; do
 640                         out=$(pgrep "$CONSUMERD_MATCH")
 641                         if [ -n "$dtimeleft_s" ]; then
 642                                 if [ $dtimeleft_s -lt 0 ]; then
 643                                         out=
 644                                         retval=1
 645                                 fi
 646                                 dtimeleft_s=$((dtimeleft_s - 1))
 647                         fi
 648                         sleep 0.5
 649                 done
 650
 651                 if [ "$withtap" -eq "1" ]; then
 652                         if [ "$retval" -eq "0" ]; then
 653                                 pass "Wait after kill session daemon"
 654                         else
 655                                 fail "Wait after kill session daemon"
 656                         fi
 657                 fi
 658         fi
 659         if [ "$signal" = "SIGKILL" ]; then
 660                 if [ "$(id -u)" -eq "0" ]; then
 661                         local modules=
 662                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 663
 664                         if [ -n "$modules" ]; then
 665                                 diag "Unloading all LTTng modules"
 666                                 modprobe -r "$modules"
 667                         fi
 668                 fi
 669         fi
 670
 671         return $retval
 672 }
 673
 674 function stop_lttng_sessiond()
 675 {
 676         stop_lttng_sessiond_opt 1 "$@"
 677 }
 678
 679 function stop_lttng_sessiond_notap()
 680 {
 681         stop_lttng_sessiond_opt 0 "$@"
 682 }
 683
 684 function sigstop_lttng_sessiond_opt()
 685 {
 686         local withtap=$1
 687         local signal=SIGSTOP
 688
 689         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 690                 # Env variable requested no session daemon
 691                 return
 692         fi
 693
 694         PID_SESSIOND="$(pgrep "${SESSIOND_MATCH}") $(pgrep "$RUNAS_MATCH")"
 695
 696         if [ "$withtap" -eq "1" ]; then
 697                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$PID_SESSIOND" | tr '\n' ' ')"
 698         fi
 699
 700         # shellcheck disable=SC2086
 701         if ! kill -s $signal $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 702                 if [ "$withtap" -eq "1" ]; then
 703                         fail "Sending SIGSTOP to session daemon"
 704                 fi
 705         else
 706                 out=1
 707                 while [ $out -ne 0 ]; do
 708                         pid="$(pgrep "$SESSIOND_MATCH")"
 709
 710                         # Wait until state becomes stopped for session
 711                         # daemon(s).
 712                         out=0
 713                         for sessiond_pid in $pid; do
 714                                 state="$(ps -p "$sessiond_pid" -o state= )"
 715                                 if [[ -n "$state" && "$state" != "T" ]]; then
 716                                         out=1
 717                                 fi
 718                         done
 719                         sleep 0.5
 720                 done
 721                 if [ "$withtap" -eq "1" ]; then
 722                         pass "Sending SIGSTOP to session daemon"
 723                 fi
 724         fi
 725 }
 726
 727 function sigstop_lttng_sessiond()
 728 {
 729         sigstop_lttng_sessiond_opt 1 "$@"
 730 }
 731
 732 function sigstop_lttng_sessiond_notap()
 733 {
 734         sigstop_lttng_sessiond_opt 0 "$@"
 735 }
 736
 737 function stop_lttng_consumerd_opt()
 738 {
 739         local withtap=$1
 740         local signal=$2
 741
 742         if [ -z "$signal" ]; then
 743                 signal=SIGTERM
 744         fi
 745
 746         local timeout_s=$3
 747         local dtimeleft_s=
 748
 749         # Multiply time by 2 to simplify integer arithmetic
 750         if [ -n "$timeout_s" ]; then
 751                 dtimeleft_s=$((timeout_s * 2))
 752         fi
 753
 754         local retval=0
 755
 756         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 757
 758         if [ -z "$PID_CONSUMERD" ]; then
 759                 if [ "$withtap" -eq "1" ]; then
 760                         pass "No consumer daemon to kill"
 761                 fi
 762                 return 0
 763         fi
 764
 765         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 766
 767         # shellcheck disable=SC2086
 768         if ! kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 769                 retval=1
 770                 if [ "$withtap" -eq "1" ]; then
 771                         fail "Kill consumer daemon"
 772                 fi
 773         else
 774                 out=1
 775                 while [ $out -ne 0 ]; do
 776                         pid="$(pgrep "$CONSUMERD_MATCH")"
 777
 778                         # If consumerds are still present check their status.
 779                         # A zombie status qualifies the consumerd as *killed*
 780                         out=0
 781                         for consumer_pid in $pid; do
 782                                 state="$(ps -p "$consumer_pid" -o state= )"
 783                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 784                                         out=1
 785                                 fi
 786                         done
 787                         if [ -n "$dtimeleft_s" ]; then
 788                                 if [ $dtimeleft_s -lt 0 ]; then
 789                                         out=0
 790                                         retval=1
 791                                 fi
 792                                 dtimeleft_s=$((dtimeleft_s - 1))
 793                         fi
 794                         sleep 0.5
 795                 done
 796                 if [ "$withtap" -eq "1" ]; then
 797                         if [ "$retval" -eq "0" ]; then
 798                                 pass "Wait after kill consumer daemon"
 799                         else
 800                                 fail "Wait after kill consumer daemon"
 801                         fi
 802                 fi
 803         fi
 804
 805         return $retval
 806 }
 807
 808 function stop_lttng_consumerd()
 809 {
 810         stop_lttng_consumerd_opt 1 "$@"
 811 }
 812
 813 function stop_lttng_consumerd_notap()
 814 {
 815         stop_lttng_consumerd_opt 0 "$@"
 816 }
 817
 818 function sigstop_lttng_consumerd_opt()
 819 {
 820         local withtap=$1
 821         local signal=SIGSTOP
 822
 823         PID_CONSUMERD="$(pgrep "$CONSUMERD_MATCH")"
 824
 825         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$PID_CONSUMERD" | tr '\n' ' ')"
 826
 827         # shellcheck disable=SC2086
 828         kill -s $signal $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 829         retval=$?
 830
 831         if [ $retval -eq 1 ]; then
 832                 if [ "$withtap" -eq "1" ]; then
 833                         fail "Sending SIGSTOP to consumer daemon"
 834                 fi
 835                 return 1
 836         else
 837                 out=1
 838                 while [ $out -ne 0 ]; do
 839                         pid="$(pgrep "$CONSUMERD_MATCH")"
 840
 841                         # Wait until state becomes stopped for all
 842                         # consumers.
 843                         out=0
 844                         for consumer_pid in $pid; do
 845                                 state="$(ps -p "$consumer_pid" -o state= )"
 846                                 if [[ -n "$state" && "$state" != "T" ]]; then
 847                                         out=1
 848                                 fi
 849                         done
 850                         sleep 0.5
 851                 done
 852                 if [ "$withtap" -eq "1" ]; then
 853                         pass "Sending SIGSTOP to consumer daemon"
 854                 fi
 855         fi
 856         return $retval
 857 }
 858
 859 function sigstop_lttng_consumerd()
 860 {
 861         sigstop_lttng_consumerd_opt 1 "$@"
 862 }
 863
 864 function sigstop_lttng_consumerd_notap()
 865 {
 866         sigstop_lttng_consumerd_opt 0 "$@"
 867 }
 868
 869 function list_lttng_with_opts ()
 870 {
 871         local opts=$1
 872         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 873         ok $? "Lttng-tool list command with option $opts"
 874 }
 875
 876 function create_lttng_session_no_output ()
 877 {
 878         local sess_name=$1
 879         local opts="${@:2}"
 880
 881         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 882         ok $? "Create session $sess_name in no-output mode"
 883 }
 884
 885 function create_lttng_session_uri () {
 886         local sess_name=$1
 887         local uri=$2
 888         local opts="${@:3}"
 889
 890         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 891         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 892 }
 893
 894 function create_lttng_session ()
 895 {
 896         local withtap=$1
 897         local expected_to_fail=$2
 898         local sess_name=$3
 899         local trace_path=$4
 900         local opt=$5
 901
 902         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 903         ret=$?
 904         if [ $expected_to_fail -eq "1" ]; then
 905                 test "$ret" -ne "0"
 906                 ret=$?
 907                 if [ $withtap -eq "1" ]; then
 908                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 909                 fi
 910         else
 911                 if [ $withtap -eq "1" ]; then
 912                         ok $ret "Create session $sess_name in $trace_path"
 913                 fi
 914         fi
 915         return $ret
 916 }
 917
 918 function create_lttng_session_ok ()
 919 {
 920         create_lttng_session 1 0 "$@"
 921 }
 922
 923 function create_lttng_session_fail ()
 924 {
 925         create_lttng_session 1 1 "$@"
 926 }
 927
 928 function create_lttng_session_notap ()
 929 {
 930         create_lttng_session 0 0 "$@"
 931 }
 932
 933
 934 function enable_ust_lttng_channel ()
 935 {
 936         local withtap=$1
 937         local expected_to_fail=$2
 938         local sess_name=$3
 939         local channel_name=$4
 940         local opt=$5
 941
 942         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 943         ret=$?
 944         if [[ $expected_to_fail -eq "1" ]]; then
 945                 test "$ret" -ne "0"
 946                 ret=$?
 947                 if [ $withtap -eq "1" ]; then
 948                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 949                 fi
 950         else
 951                 if [ $withtap -eq "1" ]; then
 952                         ok $ret "Enable channel $channel_name for session $sess_name"
 953                 fi
 954         fi
 955         return $ret
 956 }
 957
 958 function enable_ust_lttng_channel_ok ()
 959 {
 960         enable_ust_lttng_channel 1 0 "$@"
 961 }
 962
 963 function enable_ust_lttng_channel_fail ()
 964 {
 965         enable_ust_lttng_channel 1 1 "$@"
 966 }
 967
 968 function enable_ust_lttng_channel_notap ()
 969 {
 970         enable_ust_lttng_channel 0 0 "$@"
 971 }
 972
 973 function disable_ust_lttng_channel()
 974 {
 975         local sess_name=$1
 976         local channel_name=$2
 977
 978         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 979         ok $? "Disable channel $channel_name for session $sess_name"
 980 }
 981
 982 function enable_lttng_mmap_overwrite_kernel_channel()
 983 {
 984         local sess_name=$1
 985         local channel_name=$2
 986
 987         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 988         ok $? "Enable channel $channel_name for session $sess_name"
 989 }
 990
 991 function enable_lttng_mmap_discard_small_kernel_channel()
 992 {
 993         local sess_name=$1
 994         local channel_name=$2
 995
 996         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 997         ok $? "Enable small discard channel $channel_name for session $sess_name"
 998 }
 999
1000 function enable_lttng_mmap_overwrite_small_kernel_channel()
1001 {
1002         local sess_name=$1
1003         local channel_name=$2
1004
1005         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1006         ok $? "Enable small discard channel $channel_name for session $sess_name"
1007 }
1008
1009 function enable_lttng_mmap_overwrite_ust_channel()
1010 {
1011         local sess_name=$1
1012         local channel_name=$2
1013
1014         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1015         ok $? "Enable channel $channel_name for session $sess_name"
1016 }
1017
1018 function enable_ust_lttng_event ()
1019 {
1020         local withtap=$1
1021         local expected_to_fail=$2
1022         local sess_name=$3
1023         local event_name="$4"
1024         local channel_name=$5
1025
1026         if [ -z $channel_name ]; then
1027                 # default channel if none specified
1028                 chan=""
1029         else
1030                 chan="-c $channel_name"
1031         fi
1032
1033         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1034         ret=$?
1035         if [[ $expected_to_fail -eq "1" ]]; then
1036                 test $ret -ne "0"
1037                 ret=$?
1038                 if [[ $withtap -eq "1" ]]; then
1039                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1040                 fi
1041         else
1042                 if [[ $withtap -eq "1" ]]; then
1043                         ok $ret "Enable ust event $event_name for session $sess_name"
1044                 fi
1045         fi
1046         return $ret
1047 }
1048
1049 function enable_ust_lttng_event_ok ()
1050 {
1051         enable_ust_lttng_event 1 0 "$@"
1052 }
1053
1054 function enable_ust_lttng_event_fail ()
1055 {
1056         enable_ust_lttng_event 1 1 "$@"
1057 }
1058
1059 function enable_ust_lttng_event_notap ()
1060 {
1061         enable_ust_lttng_event 0 0 "$@"
1062 }
1063
1064 function enable_jul_lttng_event()
1065 {
1066         sess_name=$1
1067         event_name="$2"
1068         channel_name=$3
1069
1070         if [ -z $channel_name ]; then
1071                 # default channel if none specified
1072                 chan=""
1073         else
1074                 chan="-c $channel_name"
1075         fi
1076
1077         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1078         ok $? "Enable JUL event $event_name for session $sess_name"
1079 }
1080
1081 function enable_jul_lttng_event_loglevel()
1082 {
1083         local sess_name=$1
1084         local event_name="$2"
1085         local loglevel=$3
1086         local channel_name=$4
1087
1088         if [ -z $channel_name ]; then
1089                 # default channel if none specified
1090                 chan=""
1091         else
1092                 chan="-c $channel_name"
1093         fi
1094
1095         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1096         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1097 }
1098
1099 function enable_log4j_lttng_event()
1100 {
1101         sess_name=$1
1102         event_name="$2"
1103         channel_name=$3
1104
1105         if [ -z $channel_name ]; then
1106                 # default channel if none specified
1107                 chan=""
1108         else
1109                 chan="-c $channel_name"
1110         fi
1111
1112         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1113         ok $? "Enable LOG4J event $event_name for session $sess_name"
1114 }
1115
1116 function enable_log4j_lttng_event_loglevel()
1117 {
1118         local sess_name=$1
1119         local event_name="$2"
1120         local loglevel=$3
1121         local channel_name=$4
1122
1123         if [ -z $channel_name ]; then
1124                 # default channel if none specified
1125                 chan=""
1126         else
1127                 chan="-c $channel_name"
1128         fi
1129
1130         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1131         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
1132 }
1133
1134 function enable_python_lttng_event()
1135 {
1136         sess_name=$1
1137         event_name="$2"
1138         channel_name=$3
1139
1140         if [ -z $channel_name ]; then
1141                 # default channel if none specified
1142                 chan=""
1143         else
1144                 chan="-c $channel_name"
1145         fi
1146
1147         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1148         ok $? "Enable Python event $event_name for session $sess_name"
1149 }
1150
1151 function enable_python_lttng_event_loglevel()
1152 {
1153         local sess_name=$1
1154         local event_name="$2"
1155         local loglevel=$3
1156         local channel_name=$4
1157
1158         if [ -z $channel_name ]; then
1159                 # default channel if none specified
1160                 chan=""
1161         else
1162                 chan="-c $channel_name"
1163         fi
1164
1165         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1166         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1167 }
1168
1169 function enable_ust_lttng_event_filter()
1170 {
1171         local sess_name="$1"
1172         local event_name="$2"
1173         local filter="$3"
1174
1175         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1176         ok $? "Enable event $event_name with filtering for session $sess_name"
1177 }
1178
1179 function enable_ust_lttng_event_loglevel()
1180 {
1181         local sess_name="$1"
1182         local event_name="$2"
1183         local loglevel="$3"
1184
1185         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1186         ok $? "Enable event $event_name with loglevel $loglevel"
1187 }
1188
1189 function enable_ust_lttng_event_loglevel_only()
1190 {
1191         local sess_name="$1"
1192         local event_name="$2"
1193         local loglevel="$3"
1194
1195         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1196         ok $? "Enable event $event_name with loglevel-only $loglevel"
1197 }
1198
1199 function disable_ust_lttng_event ()
1200 {
1201         local sess_name="$1"
1202         local event_name="$2"
1203         local channel_name="$3"
1204
1205         if [ -z $channel_name ]; then
1206                 # default channel if none specified
1207                 chan=""
1208         else
1209                 chan="-c $channel_name"
1210         fi
1211
1212         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1213         ok $? "Disable event $event_name for session $sess_name"
1214 }
1215
1216 function disable_jul_lttng_event ()
1217 {
1218         local sess_name="$1"
1219         local event_name="$2"
1220
1221         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1222         ok $? "Disable JUL event $event_name for session $sess_name"
1223 }
1224
1225 function disable_log4j_lttng_event ()
1226 {
1227         local sess_name="$1"
1228         local event_name="$2"
1229
1230         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1231         ok $? "Disable LOG4J event $event_name for session $sess_name"
1232 }
1233
1234 function disable_python_lttng_event ()
1235 {
1236         local sess_name="$1"
1237         local event_name="$2"
1238
1239         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1240         ok $? "Disable Python event $event_name for session $sess_name"
1241 }
1242
1243 function start_lttng_tracing_opt ()
1244 {
1245         local withtap=$1
1246         local expected_to_fail=$2
1247         local sess_name=$3
1248
1249         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1250         ret=$?
1251         if [[ $expected_to_fail -eq "1" ]]; then
1252                 test "$ret" -ne "0"
1253                 ret=$?
1254                 if [ $withtap -eq "1" ]; then
1255                         ok $? "Start tracing for session $sess_name failed as expected"
1256                 fi
1257         else
1258                 if [ $withtap -eq "1" ]; then
1259                         ok $ret "Start tracing for session $sess_name"
1260                 fi
1261         fi
1262 }
1263
1264 function start_lttng_tracing_ok ()
1265 {
1266         start_lttng_tracing_opt 1 0 "$@"
1267 }
1268
1269 function start_lttng_tracing_fail ()
1270 {
1271         start_lttng_tracing_opt 1 1 "$@"
1272 }
1273
1274 function start_lttng_tracing_notap ()
1275 {
1276         start_lttng_tracing_opt 0 1 "$@"
1277 }
1278
1279 function stop_lttng_tracing_opt ()
1280 {
1281         local withtap=$1
1282         local expected_to_fail=$2
1283         local sess_name=$3
1284
1285         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1286         ret=$?
1287         if [[ $expected_to_fail -eq "1" ]]; then
1288                 test "$ret" -ne "0"
1289                 ret=$?
1290                 if [ $withtap -eq "1" ]; then
1291                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1292                 fi
1293         else
1294                 if [ $withtap -eq "1" ]; then
1295                         ok $ret "Stop lttng tracing for session $sess_name"
1296                 fi
1297         fi
1298 }
1299
1300 function stop_lttng_tracing_ok ()
1301 {
1302         stop_lttng_tracing_opt 1 0 "$@"
1303 }
1304
1305 function stop_lttng_tracing_fail ()
1306 {
1307         stop_lttng_tracing_opt 1 1 "$@"
1308 }
1309
1310 function stop_lttng_tracing_notap ()
1311 {
1312         stop_lttng_tracing_opt 0 0 "$@"
1313 }
1314
1315 function destroy_lttng_session ()
1316 {
1317         local withtap=$1
1318         local expected_to_fail=$2
1319         local sess_name=$3
1320
1321         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1322         ret=$?
1323         if [[ $expected_to_fail -eq "1" ]]; then
1324                 test "$ret" -ne "0"
1325                 ret=$?
1326                 if [ $withtap -eq "1" ]; then
1327                         ok $ret "Destroy session $sess_name failed as expected"
1328                 fi
1329         else
1330                 if [ $withtap -eq "1" ]; then
1331                         ok $ret "Destroy session $sess_name"
1332                 fi
1333         fi
1334 }
1335
1336 function destroy_lttng_session_ok ()
1337 {
1338         destroy_lttng_session 1 0 "$@"
1339
1340 }
1341
1342 function destroy_lttng_session_fail ()
1343 {
1344         destroy_lttng_session 1 1 "$@"
1345 }
1346
1347 function destroy_lttng_session_notap ()
1348 {
1349         destroy_lttng_session 0 0 "$@"
1350 }
1351
1352 function destroy_lttng_sessions ()
1353 {
1354         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1355         ok $? "Destroy all lttng sessions"
1356 }
1357
1358 function lttng_snapshot_add_output ()
1359 {
1360         local expected_to_fail=$1
1361         local sess_name=$2
1362         local trace_path=$3
1363
1364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1365         ret=$?
1366         if [[ $expected_to_fail -eq 1 ]]; then
1367                 test "$ret" -ne "0"
1368                 ok $? "Added snapshot output file://$trace_path failed as expected"
1369         else
1370                 ok $ret "Added snapshot output file://$trace_path"
1371         fi
1372 }
1373
1374 function lttng_snapshot_add_output_ok ()
1375 {
1376         lttng_snapshot_add_output 0 "$@"
1377 }
1378
1379 function lttng_snapshot_add_output_fail ()
1380 {
1381         lttng_snapshot_add_output 1 "$@"
1382 }
1383
1384 function lttng_snapshot_del_output ()
1385 {
1386         local expected_to_fail=$1
1387         local sess_name=$2
1388         local id=$3
1389
1390         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1391         ret=$?
1392         if [[ $expected_to_fail -eq "1" ]]; then
1393                 test "$ret" -ne "0"
1394                 ok $? "Deleted snapshot output id $id failed as expected"
1395         else
1396                 ok $ret "Deleted snapshot output id $id"
1397         fi
1398 }
1399
1400 function lttng_snapshot_del_output_ok ()
1401 {
1402         lttng_snapshot_del_output 0 "$@"
1403 }
1404
1405 function lttng_snapshot_del_output_fail ()
1406 {
1407         lttng_snapshot_del_output 1 "$@"
1408 }
1409
1410 function lttng_snapshot_record ()
1411 {
1412         local sess_name=$1
1413         local trace_path=$2
1414
1415         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name $trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1416         ok $? "Snapshot recorded"
1417 }
1418
1419 function lttng_snapshot_list ()
1420 {
1421         local sess_name=$1
1422         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1423         ok $? "Snapshot list"
1424 }
1425
1426 function lttng_save()
1427 {
1428         local sess_name=$1
1429         local opts=$2
1430
1431         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1432         ok $? "Session saved"
1433 }
1434
1435 function lttng_load()
1436 {
1437         local expected_to_fail=$1
1438         local opts=$2
1439
1440         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1441         ret=$?
1442         if [[ $expected_to_fail -eq "1" ]]; then
1443                 test $ret -ne "0"
1444                 ok $? "Load command failed as expected with opts: $opts"
1445         else
1446                 ok $ret "Load command with opts: $opts"
1447         fi
1448 }
1449
1450 function lttng_load_ok()
1451 {
1452         lttng_load 0 "$@"
1453 }
1454
1455 function lttng_load_fail()
1456 {
1457         lttng_load 1 "$@"
1458 }
1459
1460 function lttng_track()
1461 {
1462         local expected_to_fail=$1
1463         local opts=$2
1464         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1465         ret=$?
1466         if [[ $expected_to_fail -eq "1" ]]; then
1467                 test $ret -ne "0"
1468                 ok $? "Track command failed as expected with opts: $opts"
1469         else
1470                 ok $ret "Track command with opts: $opts"
1471         fi
1472 }
1473
1474 function lttng_track_ok()
1475 {
1476         lttng_track 0 "$@"
1477 }
1478
1479 function lttng_track_fail()
1480 {
1481         lttng_track 1 "$@"
1482 }
1483
1484 function lttng_untrack()
1485 {
1486         local expected_to_fail=$1
1487         local opts=$2
1488         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1489         ret=$?
1490         if [[ $expected_to_fail -eq "1" ]]; then
1491                 test $ret -ne "0"
1492                 ok $? "Untrack command failed as expected with opts: $opts"
1493         else
1494                 ok $ret "Untrack command with opts: $opts"
1495         fi
1496 }
1497
1498 function lttng_untrack_ok()
1499 {
1500         lttng_untrack 0 "$@"
1501 }
1502
1503 function lttng_untrack_fail()
1504 {
1505         lttng_untrack 1 "$@"
1506 }
1507
1508 function lttng_track_pid_ok()
1509 {
1510         PID=$1
1511         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1512         ok $? "Lttng track pid on the kernel domain"
1513 }
1514
1515 function lttng_untrack_kernel_all_ok()
1516 {
1517         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1518         ok $? "Lttng untrack all pid on the kernel domain"
1519 }
1520
1521 function lttng_add_context_list()
1522 {
1523         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1524         ret=$?
1525         ok $ret "Context listing"
1526 }
1527
1528 function add_context_lttng()
1529 {
1530         local expected_to_fail="$1"
1531         local domain="$2"
1532         local session_name="$3"
1533         local channel_name="$4"
1534         local type="$5"
1535
1536         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1537         ret=$?
1538         if [[ $expected_to_fail -eq "1" ]]; then
1539                 test $ret -ne "0"
1540                 ok $? "Add context command failed as expected for type: $type"
1541         else
1542                 ok $ret "Add context command for type: $type"
1543         fi
1544 }
1545
1546 function add_context_ust_ok()
1547 {
1548         add_context_lttng 0 -u "$@"
1549 }
1550
1551 function add_context_ust_fail()
1552 {
1553         add_context_lttng 1 -u "$@"
1554 }
1555
1556 function add_context_kernel_ok()
1557 {
1558         add_context_lttng 0 -k "$@"
1559 }
1560
1561 function add_context_kernel_fail()
1562 {
1563         add_context_lttng 1 -k "$@"
1564 }
1565
1566 function validate_metadata_event ()
1567 {
1568         local event_name=$1
1569         local nr_event_id=$2
1570         local trace_path=$3
1571
1572         local metadata_file=$(find $trace_path | grep metadata)
1573         local metadata_path=$(dirname $metadata_file)
1574
1575         which $BABELTRACE_BIN >/dev/null
1576         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1577
1578         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1579
1580         if [ "$count" -ne "$nr_event_id" ]; then
1581                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1582                 diag "$count matching event id found in metadata"
1583         else
1584                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1585         fi
1586
1587 }
1588
1589 function trace_matches ()
1590 {
1591         local event_name=$1
1592         local nr_iter=$2
1593         local trace_path=$3
1594
1595         which $BABELTRACE_BIN >/dev/null
1596         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1597
1598         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1599
1600         if [ "$count" -ne "$nr_iter" ]; then
1601                 fail "Trace match"
1602                 diag "$count matching events found in trace"
1603         else
1604                 pass "Trace match"
1605         fi
1606 }
1607
1608 function trace_match_only()
1609 {
1610         local event_name=$1
1611         local nr_iter=$2
1612         local trace_path=$3
1613
1614         which $BABELTRACE_BIN >/dev/null
1615         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1616
1617         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1618         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1619
1620         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1621                 pass "Trace match with $total event $event_name"
1622         else
1623                 fail "Trace match"
1624                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1625         fi
1626 }
1627
1628 function validate_trace
1629 {
1630         local event_name=$1
1631         local trace_path=$2
1632
1633         which $BABELTRACE_BIN >/dev/null
1634         if [ $? -ne 0 ]; then
1635             skip 0 "Babeltrace binary not found. Skipping trace validation"
1636         fi
1637
1638         OLDIFS=$IFS
1639         IFS=","
1640         for i in $event_name; do
1641                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1642                 if [ "$traced" -ne 0 ]; then
1643                         pass "Validate trace for event $i, $traced events"
1644                 else
1645                         fail "Validate trace for event $i"
1646                         diag "Found $traced occurences of $i"
1647                 fi
1648         done
1649         ret=$?
1650         IFS=$OLDIFS
1651         return $ret
1652 }
1653
1654 function validate_trace_count
1655 {
1656         local event_name=$1
1657         local trace_path=$2
1658         local expected_count=$3
1659
1660         which $BABELTRACE_BIN >/dev/null
1661         if [ $? -ne 0 ]; then
1662             skip 0 "Babeltrace binary not found. Skipping trace validation"
1663         fi
1664
1665         cnt=0
1666         OLDIFS=$IFS
1667         IFS=","
1668         for i in $event_name; do
1669                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1670                 if [ "$traced" -ne 0 ]; then
1671                         pass "Validate trace for event $i, $traced events"
1672                 else
1673                         fail "Validate trace for event $i"
1674                         diag "Found $traced occurences of $i"
1675                 fi
1676                 cnt=$(($cnt + $traced))
1677         done
1678         IFS=$OLDIFS
1679         test $cnt -eq $expected_count
1680         ok $? "Read a total of $cnt events, expected $expected_count"
1681 }
1682
1683 function trace_first_line
1684 {
1685         local trace_path=$1
1686
1687         which $BABELTRACE_BIN >/dev/null
1688         if [ $? -ne 0 ]; then
1689             skip 0 "Babeltrace binary not found. Skipping trace validation"
1690         fi
1691
1692         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1693 }
1694
1695 function validate_trace_exp()
1696 {
1697         local event_exp=$1
1698         local trace_path=$2
1699
1700         which $BABELTRACE_BIN >/dev/null
1701         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1702
1703         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1704         if [ "$traced" -ne 0 ]; then
1705                 pass "Validate trace for expression '${event_exp}', $traced events"
1706         else
1707                 fail "Validate trace for expression '${event_exp}'"
1708                 diag "Found $traced occurences of '${event_exp}'"
1709         fi
1710         ret=$?
1711         return $ret
1712 }
1713
1714 function validate_trace_only_exp()
1715 {
1716         local event_exp=$1
1717         local trace_path=$2
1718
1719         which $BABELTRACE_BIN >/dev/null
1720         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1721
1722         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1723         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1724
1725         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1726                 pass "Trace match with $total for expression '${event_exp}'"
1727         else
1728                 fail "Trace match"
1729                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1730         fi
1731         ret=$?
1732         return $ret
1733 }
1734
1735 function validate_trace_empty()
1736 {
1737         local trace_path=$1
1738
1739         which $BABELTRACE_BIN >/dev/null
1740         if [ $? -ne 0 ]; then
1741             skip 0 "Babeltrace binary not found. Skipping trace validation"
1742         fi
1743
1744         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1745         ret=$?
1746         if [ $ret -ne 0 ]; then
1747                 fail "Failed to parse trace"
1748                 return $ret
1749         fi
1750
1751         traced=$(echo -n "$events" | wc -l)
1752         if [ "$traced" -eq 0 ]; then
1753                 pass "Validate empty trace"
1754         else
1755                 fail "Validate empty trace"
1756                 diag "Found $traced events in trace"
1757         fi
1758         ret=$?
1759         return $ret
1760 }
1761
1762 function regenerate_metadata ()
1763 {
1764         local expected_to_fail=$1
1765         local sess_name=$2
1766
1767         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1768         ret=$?
1769         if [[ $expected_to_fail -eq "1" ]]; then
1770                 test "$ret" -ne "0"
1771                 ok $? "Expected fail on regenerate metadata $sess_name"
1772         else
1773                 ok $ret "Metadata regenerate $sess_name"
1774         fi
1775 }
1776
1777 function regenerate_metadata_ok ()
1778 {
1779         regenerate_metadata 0 "$@"
1780 }
1781
1782 function regenerate_metadata_fail ()
1783 {
1784         regenerate_metadata 1 "$@"
1785 }
1786
1787 function regenerate_statedump ()
1788 {
1789         local expected_to_fail=$1
1790         local sess_name=$2
1791
1792         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1793         ret=$?
1794         if [[ $expected_to_fail -eq "1" ]]; then
1795                 test "$ret" -ne "0"
1796                 ok $? "Expected fail on regenerate statedump $sess_name"
1797         else
1798                 ok $ret "Metadata regenerate $sess_name"
1799         fi
1800 }
1801
1802 function regenerate_statedump_ok ()
1803 {
1804         regenerate_statedump 0 "$@"
1805 }
1806
1807 function regenerate_statedump_fail ()
1808 {
1809         regenerate_statedump 1 "$@"
1810 }
1811
1812 function rotate_session ()
1813 {
1814         local expected_to_fail=$1
1815         local sess_name=$2
1816
1817         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1818         ret=$?
1819         if [[ $expected_to_fail -eq "1" ]]; then
1820                 test "$ret" -ne "0"
1821                 ok $? "Expected fail on rotate session $sess_name"
1822         else
1823                 ok $ret "Rotate session $sess_name"
1824         fi
1825 }
1826
1827 function rotate_session_ok ()
1828 {
1829         rotate_session 0 "$@"
1830 }
1831
1832 function rotate_session_fail ()
1833 {
1834         rotate_session 1 "$@"
1835 }
1836
1837 function destructive_tests_enabled ()
1838 {
1839         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1840                 return 0
1841         else
1842                 return 1
1843         fi
1844 }
1845
1846 function lttng_enable_rotation_timer ()
1847 {
1848         local expected_to_fail=$1
1849         local sess_name=$2
1850         local period=$3
1851
1852         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1853         ret=$?
1854         if [[ $expected_to_fail -eq "1" ]]; then
1855                 test "$ret" -ne "0"
1856                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
1857         else
1858                 ok $ret "Set periodic rotation ($period) of session $sess_name"
1859         fi
1860 }
1861
1862 function lttng_enable_rotation_timer_ok ()
1863 {
1864         lttng_enable_rotation_timer 0 $@
1865 }
1866
1867 function lttng_enable_rotation_timer_fail ()
1868 {
1869         lttng_enable_rotation_timer 1 $@
1870 }
1871
1872 function lttng_enable_rotation_size ()
1873 {
1874         local expected_to_fail=$1
1875         local sess_name=$2
1876         local size=$3
1877
1878         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1879         ret=$?
1880         if [[ $expected_to_fail -eq "1" ]]; then
1881                 test "$ret" -ne "0"
1882                 ok $? "Expected fail on rotate session $sess_name"
1883         else
1884                 ok $ret "Rotate session $sess_name"
1885         fi
1886 }
1887
1888 function lttng_enable_rotation_size_ok ()
1889 {
1890         lttng_enable_rotation_size 0 $@
1891 }
1892
1893 function lttng_enable_rotation_size_fail ()
1894 {
1895         lttng_enable_rotation_size 1 $@
1896 }