tests/utils/utils.sh

   1 # Copyright (C) 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # SPDX-License-Identifier: LGPL-2.1-only
   4 #
   5
   6 SESSIOND_BIN="lttng-sessiond"
   7 SESSIOND_MATCH=".*lttng-sess.*"
   8 RUNAS_BIN="lttng-runas"
   9 RUNAS_MATCH=".*lttng-runas.*"
  10 CONSUMERD_BIN="lttng-consumerd"
  11 CONSUMERD_MATCH=".*lttng-consumerd.*"
  12 RELAYD_BIN="lttng-relayd"
  13 RELAYD_MATCH=".*lttng-relayd.*"
  14 LTTNG_BIN="lttng"
  15 BABELTRACE_BIN="babeltrace2"
  16 OUTPUT_DEST=/dev/null
  17 ERROR_OUTPUT_DEST=/dev/null
  18 MI_XSD_MAJOR_VERSION=4
  19 MI_XSD_MINOR_VERSION=1
  20 MI_XSD_PATH="$TESTDIR/../src/common/mi-lttng-${MI_XSD_MAJOR_VERSION}.${MI_XSD_MINOR_VERSION}.xsd"
  21 MI_VALIDATE="$TESTDIR/utils/xml-utils/validate_xml ${MI_XSD_PATH}"
  22
  23 XML_PRETTY="$TESTDIR/utils/xml-utils/pretty_xml"
  24 XML_EXTRACT="$TESTDIR/utils/xml-utils/extract_xml"
  25 XML_NODE_CHECK="${XML_EXTRACT} -e"
  26
  27 # To match 20201127-175802
  28 date_time_pattern="[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]"
  29 # The size of a long on this system
  30 system_long_bit_size=$(getconf LONG_BIT)
  31
  32 # Minimal kernel version supported for session daemon tests
  33 KERNEL_MAJOR_VERSION=2
  34 KERNEL_MINOR_VERSION=6
  35 KERNEL_PATCHLEVEL_VERSION=27
  36
  37 # We set the default UST register timeout and network and app socket timeout to
  38 # "wait forever", so that basic tests don't have to worry about hitting
  39 # timeouts on busy systems. Specialized tests should test those corner-cases.
  40 export LTTNG_UST_REGISTER_TIMEOUT=-1
  41 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  42 export LTTNG_APP_SOCKET_TIMEOUT=-1
  43
  44 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  45 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  46 # its own sessiond if none is running. It also ensures that 'lttng create'
  47 # fails when no sessiond is running.
  48 export LTTNG_SESSIOND_PATH="/bin/true"
  49
  50 source $TESTDIR/utils/tap/tap.sh
  51
  52 if [ -z ${LTTNG_TEST_TEARDOWN_TIMEOUT+x} ]; then
  53         LTTNG_TEST_TEARDOWN_TIMEOUT=60
  54 fi
  55
  56 # Enable job monitor mode.
  57 # Here we are mostly interested in the following from the monitor mode:
  58 #    All processes run in a separate process group.
  59 # This allows us to ensure that all subprocesses from all background tasks are
  60 # cleaned up correctly using signal to process group id.
  61 set -m
  62
  63 kill_background_jobs ()
  64 {
  65         local pids
  66         pids=$(jobs -p)
  67
  68         if [ -z "$pids" ]; then
  69                 # Empty
  70                 return 0
  71         fi
  72
  73         while read -r pid;
  74         do
  75                 # Use negative number to send the signal to the process group.
  76                 # This ensure that any subprocesses receive the signal.
  77                 # /dev/null is used since there is an acceptable race between
  78                 # the moments the pids are listed and the moment we send a
  79                 # signal.
  80                 kill -SIGTERM -- "-$pid" 2>/dev/null
  81         done <<< "$pids"
  82 }
  83
  84 function cleanup ()
  85 {
  86         # Try to kill daemons gracefully
  87         stop_lttng_relayd_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  88         stop_lttng_sessiond_cleanup SIGTERM $LTTNG_TEST_TEARDOWN_TIMEOUT
  89
  90         # If daemons are still present, forcibly kill them
  91         stop_lttng_relayd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  92         stop_lttng_sessiond_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  93         stop_lttng_consumerd_cleanup SIGKILL $LTTNG_TEST_TEARDOWN_TIMEOUT
  94
  95         kill_background_jobs
  96 }
  97
  98 function full_cleanup ()
  99 {
 100         cleanup
 101         exit 1
 102 }
 103
 104 function LTTNG_BAIL_OUT ()
 105 {
 106         cleanup
 107         BAIL_OUT "$@"
 108 }
 109
 110 function null_pipes ()
 111 {
 112         exec 0>/dev/null
 113         exec 1>/dev/null
 114         exec 2>/dev/null
 115 }
 116
 117 trap full_cleanup SIGINT SIGTERM
 118
 119 # perl prove closes its child pipes before giving it a chance to run its
 120 # signal trap handlers. Redirect pipes to /dev/null if SIGPIPE is caught
 121 # to allow those trap handlers to proceed.
 122
 123 trap null_pipes SIGPIPE
 124
 125 # Check pgrep from env, default to pgrep if none
 126 if [ -z "$PGREP" ]; then
 127         PGREP=pgrep
 128 fi
 129
 130 # Due to the renaming of threads we need to use the full command (pgrep -f) to
 131 # identify the pids for multiple lttng related processes. The problem with "pgrep
 132 # -f" is that it ends up also looking at the arguments. We use a two stage
 133 # lookup. The first one is using "pgrep -f" yielding potential candidate.
 134 # The second on perform grep on the basename of the first field of the
 135 # /proc/pid/cmdline of the previously identified pids. The first field
 136 # correspond to the actual command.
 137 function lttng_pgrep ()
 138 {
 139         local pattern=$1
 140         local possible_pids
 141         local full_command_no_argument
 142         local command_basename
 143
 144         possible_pids=$($PGREP -f "$pattern")
 145         if [ -z "$possible_pids" ]; then
 146                 return 0
 147         fi
 148
 149         while IFS= read -r pid ; do
 150                 # /proc/pid/cmdline is null separated.
 151                 if full_command_no_argument=$(cut -d '' -f 1 2>/dev/null < /proc/"$pid"/cmdline); then
 152                         command_basename=$(basename "$full_command_no_argument")
 153                         if grep -q "$pattern" <<< "$command_basename"; then
 154                                 echo "$pid"
 155                         fi
 156                 fi
 157         done <<< "$possible_pids"
 158         return 0
 159 }
 160
 161 function print_ok ()
 162 {
 163         # Check if we are a terminal
 164         if [ -t 1 ]; then
 165                 echo -e "\e[1;32mOK\e[0m"
 166         else
 167                 echo -e "OK"
 168         fi
 169 }
 170
 171 function print_fail ()
 172 {
 173         # Check if we are a terminal
 174         if [ -t 1 ]; then
 175                 echo -e "\e[1;31mFAIL\e[0m"
 176         else
 177                 echo -e "FAIL"
 178         fi
 179 }
 180
 181 function print_test_banner ()
 182 {
 183         local desc="$1"
 184         diag "$desc"
 185 }
 186
 187 function validate_kernel_version ()
 188 {
 189         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
 190         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
 191                 return 0
 192         fi
 193         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 194                 return 0
 195         fi
 196         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 197                 return 0
 198         fi
 199         return 1
 200 }
 201
 202 # Generate a random string
 203 #  $1 = number of characters; defaults to 16
 204 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 205 function randstring()
 206 {
 207         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 208         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 209         echo
 210 }
 211
 212 # Return the number of _configured_ CPUs.
 213 function conf_proc_count()
 214 {
 215         getconf _NPROCESSORS_CONF
 216         if [ $? -ne 0 ]; then
 217                 diag "Failed to get the number of configured CPUs"
 218         fi
 219         echo
 220 }
 221
 222 # Check if base lttng-modules are present.
 223 # Bail out on failure
 224 function validate_lttng_modules_present ()
 225 {
 226         # Check for loadable modules.
 227         modprobe -n lttng-tracer 2>/dev/null
 228         if [ $? -eq 0 ]; then
 229                 return 0
 230         fi
 231
 232         # Check for builtin modules.
 233         ls /proc/lttng > /dev/null 2>&1
 234         if [ $? -eq 0 ]; then
 235                 return 0
 236         fi
 237
 238         LTTNG_BAIL_OUT "LTTng modules not detected."
 239 }
 240
 241 # Run the lttng binary.
 242 #
 243 # The first two arguments are stdout and stderr redirect paths, respectively.
 244 # The rest of the arguments are forwarded to the lttng binary
 245 function _run_lttng_cmd
 246 {
 247         local stdout_dest="$1"
 248         local stderr_dest="$2"
 249         shift 2
 250
 251         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN $*"
 252         $TESTDIR/../src/bin/lttng/$LTTNG_BIN "$@" 1> "$stdout_dest" 2> "$stderr_dest"
 253 }
 254
 255 function enable_kernel_lttng_event
 256 {
 257         local withtap="$1"
 258         local expected_to_fail="$2"
 259         local sess_name="$3"
 260         local event_name="$4"
 261         local channel_name="$5"
 262
 263         if [ -z "$event_name" ]; then
 264                 # Enable all event if no event name specified
 265                 event_name="-a"
 266         fi
 267
 268         if [ -z "$channel_name" ]; then
 269                 # default channel if none specified
 270                 chan=""
 271         else
 272                 chan="-c $channel_name"
 273         fi
 274
 275         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 276                 enable-event "$event_name" $chan -s $sess_name -k
 277         ret=$?
 278         if [[ $expected_to_fail -eq "1" ]]; then
 279                 test $ret -ne "0"
 280                 ret=$?
 281                 if [ $withtap -eq "1" ]; then
 282                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 283                 fi
 284         else
 285                 if [ $withtap -eq "1" ]; then
 286                         ok $ret "Enable kernel event $event_name for session $sess_name"
 287                 fi
 288         fi
 289 }
 290
 291 function enable_kernel_lttng_event_ok ()
 292 {
 293         enable_kernel_lttng_event 1 0 "$@"
 294 }
 295
 296 function enable_kernel_lttng_event_fail ()
 297 {
 298         enable_kernel_lttng_event 1 1 "$@"
 299 }
 300
 301 function enable_kernel_lttng_event_notap ()
 302 {
 303         enable_kernel_lttng_event 0 0 "$@"
 304 }
 305
 306 # Old interface
 307 function lttng_enable_kernel_event
 308 {
 309         enable_kernel_lttng_event_ok "$@"
 310 }
 311
 312 function lttng_enable_kernel_syscall()
 313 {
 314         local expected_to_fail=$1
 315         local sess_name=$2
 316         local syscall_name=$3
 317         local channel_name=$4
 318
 319         if [ -z $syscall_name ]; then
 320                 # Enable all event if no syscall name specified
 321                 syscall_name="-a"
 322         fi
 323
 324         if [ -z $channel_name ]; then
 325                 # default channel if none specified
 326                 chan=""
 327         else
 328                 chan="-c $channel_name"
 329         fi
 330
 331         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 332                 enable-event --syscall "$syscall_name" $chan -s $sess_name -k
 333         ret=$?
 334         if [[ $expected_to_fail -eq "1" ]]; then
 335                 test $ret -ne "0"
 336                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 337         else
 338                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 339         fi
 340 }
 341
 342 function lttng_enable_kernel_syscall_ok()
 343 {
 344         lttng_enable_kernel_syscall 0 "$@"
 345 }
 346
 347 function lttng_enable_kernel_syscall_fail()
 348 {
 349         lttng_enable_kernel_syscall 1 "$@"
 350 }
 351
 352 function lttng_disable_kernel_syscall()
 353 {
 354         local expected_to_fail=$1
 355         local sess_name=$2
 356         local syscall_name=$3
 357         local channel_name=$4
 358
 359         if [ -z $syscall_name ]; then
 360                 # Enable all event if no syscall name specified
 361                 syscall_name="-a"
 362         fi
 363
 364         if [ -z $channel_name ]; then
 365                 # default channel if none specified
 366                 chan=""
 367         else
 368                 chan="-c $channel_name"
 369         fi
 370
 371         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 372                 disable-event --syscall "$syscall_name" $chan -s $sess_name -k
 373
 374         ret=$?
 375         if [[ $expected_to_fail -eq "1" ]]; then
 376                 test $ret -ne "0"
 377                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 378         else
 379                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 380         fi
 381 }
 382
 383 function lttng_disable_kernel_syscall_ok()
 384 {
 385         lttng_disable_kernel_syscall 0 "$@"
 386 }
 387
 388 function lttng_disable_kernel_syscall_fail()
 389 {
 390         lttng_disable_kernel_syscall 1 "$@"
 391 }
 392
 393 function lttng_enable_kernel_function_event ()
 394 {
 395         local expected_to_fail="$1"
 396         local sess_name="$2"
 397         local target="$3"
 398         local event_name="$4"
 399
 400         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --function="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 401         ret=$?
 402         if [[ $expected_to_fail -eq "1" ]]; then
 403                 test $ret -ne "0"
 404                 ok $? "Enable kernel function event for session $sess_name failed as expected"
 405         else
 406                 ok $ret "Enable kernel function event for session $sess_name"
 407         fi
 408 }
 409
 410 function lttng_enable_kernel_function_event_ok ()
 411 {
 412         lttng_enable_kernel_function_event 0 "$@"
 413 }
 414
 415 function lttng_enable_kernel_userspace_probe_event ()
 416 {
 417         local expected_to_fail="$1"
 418         local sess_name="$2"
 419         local target="$3"
 420         local event_name="$4"
 421
 422         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" enable-event --kernel --userspace-probe="$target" "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 423         ret=$?
 424         if [[ $expected_to_fail -eq "1" ]]; then
 425                 test $ret -ne "0"
 426                 ok $? "Enable kernel userspace probe event for session $sess_name failed as expected"
 427         else
 428                 ok $ret "Enable kernel userspace probe event for session $sess_name"
 429         fi
 430 }
 431
 432 function lttng_enable_kernel_userspace_probe_event_fail ()
 433 {
 434         lttng_enable_kernel_userspace_probe_event 1 "$@"
 435 }
 436
 437 function lttng_enable_kernel_userspace_probe_event_ok ()
 438 {
 439         lttng_enable_kernel_userspace_probe_event 0 "$@"
 440 }
 441
 442 function disable_kernel_lttng_userspace_probe_event_ok ()
 443 {
 444         local sess_name="$1"
 445         local event_name="$2"
 446
 447         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" disable-event --kernel "$event_name" -s "$sess_name" > "$OUTPUT_DEST" 2> "$ERROR_OUTPUT_DEST"
 448         ok $? "Disable kernel event $target for session $sess_name"
 449 }
 450 function lttng_enable_kernel_channel()
 451 {
 452         local withtap=$1
 453         local expected_to_fail=$2
 454         local sess_name=$3
 455         local channel_name=$4
 456         local opts="${@:5}"
 457
 458         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 459                 enable-channel -k $channel_name -s $sess_name $opts
 460         ret=$?
 461         if [[ $expected_to_fail -eq "1" ]]; then
 462                 test "$ret" -ne "0"
 463                 ret=$?
 464                 if [ $withtap -eq "1" ]; then
 465                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 466                 fi
 467         else
 468                 if [ $withtap -eq "1" ]; then
 469                         ok $ret "Enable channel $channel_name for session $sess_name"
 470                 fi
 471         fi
 472 }
 473
 474 function lttng_enable_kernel_channel_ok()
 475 {
 476         lttng_enable_kernel_channel 1 0 "$@"
 477 }
 478
 479 function lttng_enable_kernel_channel_fail()
 480 {
 481         lttng_enable_kernel_channel 1 1 "$@"
 482 }
 483
 484 function lttng_enable_kernel_channel_notap()
 485 {
 486         lttng_enable_kernel_channel 0 0 "$@"
 487 }
 488
 489 function enable_kernel_lttng_channel_ok()
 490 {
 491         lttng_enable_kernel_channel 1 0 "$@"
 492 }
 493
 494 function lttng_disable_kernel_channel()
 495 {
 496         local expected_to_fail=$1
 497         local sess_name=$2
 498         local channel_name=$3
 499
 500         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
 501                 disable-channel -k $channel_name -s $sess_name
 502         ret=$?
 503         if [[ $expected_to_fail -eq "1" ]]; then
 504                 test "$ret" -ne "0"
 505                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 506         else
 507                 ok $ret "Disable channel $channel_name for session $sess_name"
 508         fi
 509 }
 510
 511 function lttng_disable_kernel_channel_ok()
 512 {
 513         lttng_disable_kernel_channel 0 "$@"
 514 }
 515
 516 function lttng_disable_kernel_channel_fail()
 517 {
 518         lttng_disable_kernel_channel 1 "$@"
 519 }
 520
 521 function start_lttng_relayd_opt()
 522 {
 523         local withtap=$1
 524         local process_mode=$2
 525         local opt=$3
 526
 527         DIR=$(readlink -f "$TESTDIR")
 528
 529         if [ -z $(lttng_pgrep "$RELAYD_MATCH") ]; then
 530                 # shellcheck disable=SC2086
 531                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN $process_mode $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 532                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 533                 if [ $? -eq 1 ]; then
 534                         if [ $withtap -eq "1" ]; then
 535                                 fail "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 536                         fi
 537                         return 1
 538                 else
 539                         if [ $withtap -eq "1" ]; then
 540                                 pass "Start lttng-relayd (process mode: $process_mode opt: $opt)"
 541                         fi
 542                 fi
 543         else
 544                 pass "Start lttng-relayd (opt: $opt)"
 545         fi
 546 }
 547
 548 function start_lttng_relayd()
 549 {
 550         start_lttng_relayd_opt 1 "-b" "$@"
 551 }
 552
 553 function start_lttng_relayd_notap()
 554 {
 555         start_lttng_relayd_opt 0 "-b" "$@"
 556 }
 557
 558 function stop_lttng_relayd_opt()
 559 {
 560         local withtap=$1
 561         local is_cleanup=$2
 562         local signal=$3
 563         local timeout_s=$4
 564         local dtimeleft_s=
 565         local retval=0
 566         local pids
 567
 568         if [ -z "$signal" ]; then
 569                 signal="SIGTERM"
 570         fi
 571
 572
 573         # Multiply time by 2 to simplify integer arithmetic
 574         if [ -n "$timeout_s" ]; then
 575                 dtimeleft_s=$((timeout_s * 2))
 576         fi
 577
 578
 579         pids=$(lttng_pgrep "$RELAYD_MATCH")
 580         if [ -z "$pids" ]; then
 581                 if [ "$is_cleanup" -eq 1 ]; then
 582                         :
 583                 elif [ "$withtap" -eq "1" ]; then
 584                         fail "No relay daemon to kill"
 585                 else
 586                         LTTNG_BAIL_OUT "No relay daemon to kill"
 587                 fi
 588                 return 0
 589         fi
 590
 591         diag "Killing (signal $signal) lttng-relayd (pid: $pids)"
 592
 593         # shellcheck disable=SC2086
 594         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 595                 retval=1
 596                 if [ "$withtap" -eq "1" ]; then
 597                         fail "Kill relay daemon"
 598                 fi
 599         else
 600                 out=1
 601                 while [ -n "$out" ]; do
 602                         out=$(lttng_pgrep "$RELAYD_MATCH")
 603                         if [ -n "$dtimeleft_s" ]; then
 604                                 if [ $dtimeleft_s -lt 0 ]; then
 605                                         out=
 606                                         retval=1
 607                                 fi
 608                                 dtimeleft_s=$((dtimeleft_s - 1))
 609                         fi
 610                         sleep 0.5
 611                 done
 612                 if [ "$withtap" -eq "1" ]; then
 613                         if [ "$retval" -eq "0" ]; then
 614                                 pass "Wait after kill relay daemon"
 615                         else
 616                                 fail "Wait after kill relay daemon"
 617                         fi
 618                 fi
 619         fi
 620         return $retval
 621 }
 622
 623 function stop_lttng_relayd()
 624 {
 625         stop_lttng_relayd_opt 1 0 "$@"
 626 }
 627
 628 function stop_lttng_relayd_notap()
 629 {
 630         stop_lttng_relayd_opt 0 0 "$@"
 631 }
 632
 633 function stop_lttng_relayd_cleanup()
 634 {
 635         stop_lttng_relayd_opt 0 1 "$@"
 636 }
 637
 638 #First arg: show tap output
 639 #Second argument: load path for automatic loading
 640 function start_lttng_sessiond_opt()
 641 {
 642         local withtap=$1
 643         local load_path=$2
 644
 645         # The rest of the arguments will be passed directly to lttng-sessiond.
 646         shift 2
 647
 648         local env_vars=""
 649         local consumerd=""
 650
 651         local long_bit_value=
 652         long_bit_value=$(getconf LONG_BIT)
 653
 654         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 655                 # Env variable requested no session daemon
 656                 return
 657         fi
 658
 659         DIR=$(readlink -f "$TESTDIR")
 660
 661         # Get long_bit value for 32/64 consumerd
 662         case "$long_bit_value" in
 663                 32)
 664                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 665                         ;;
 666                 64)
 667                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 668                         ;;
 669                 *)
 670                         return
 671                         ;;
 672         esac
 673
 674         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 675         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 676                 env_vars="${LTTNG_SESSIOND_ENV_VARS} "
 677         fi
 678         env_vars="${env_vars}$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN"
 679
 680         if ! validate_kernel_version; then
 681             fail "Start session daemon"
 682             LTTNG_BAIL_OUT "*** Kernel too old for session daemon tests ***"
 683         fi
 684
 685         diag "export LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/"
 686         : "${LTTNG_SESSION_CONFIG_XSD_PATH="${DIR}/../src/common/"}"
 687         export LTTNG_SESSION_CONFIG_XSD_PATH
 688
 689         if [ -z "$(lttng_pgrep "${SESSIOND_MATCH}")" ]; then
 690                 # Have a load path ?
 691                 if [ -n "$load_path" ]; then
 692                         diag "env $env_vars --load $load_path --background $consumerd $@"
 693                         # shellcheck disable=SC2086
 694                         env $env_vars --load "$load_path" --background "$consumerd" "$@"
 695                 else
 696                         diag "env $env_vars --background $consumerd $@"
 697                         # shellcheck disable=SC2086
 698                         env $env_vars --background "$consumerd" "$@"
 699                 fi
 700                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 701                 status=$?
 702                 if [ "$withtap" -eq "1" ]; then
 703                         ok $status "Start session daemon"
 704                 fi
 705         fi
 706 }
 707
 708 function start_lttng_sessiond()
 709 {
 710         start_lttng_sessiond_opt 1 "$@"
 711 }
 712
 713 function start_lttng_sessiond_notap()
 714 {
 715         start_lttng_sessiond_opt 0 "$@"
 716 }
 717
 718 function stop_lttng_sessiond_opt()
 719 {
 720         local withtap=$1
 721         local is_cleanup=$2
 722         local signal=$3
 723         local timeout_s=$4
 724         local dtimeleft_s=
 725         local retval=0
 726         local runas_pids
 727         local pids
 728
 729         if [ -z "$signal" ]; then
 730                 signal=SIGTERM
 731         fi
 732
 733         # Multiply time by 2 to simplify integer arithmetic
 734         if [ -n "$timeout_s" ]; then
 735                 dtimeleft_s=$((timeout_s * 2))
 736         fi
 737
 738         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 739                 # Env variable requested no session daemon
 740                 return 0
 741         fi
 742
 743         runas_pids=$(lttng_pgrep "$RUNAS_MATCH")
 744         pids=$(lttng_pgrep "$SESSIOND_MATCH")
 745
 746         if [ -n "$runas_pids" ]; then
 747                 pids="$pids $runas_pids"
 748         fi
 749
 750         if [ -z "$pids" ]; then
 751                 if [ "$is_cleanup" -eq 1 ]; then
 752                         :
 753                 elif [ "$withtap" -eq "1" ]; then
 754                         fail "No session daemon to kill"
 755                 else
 756                         LTTNG_BAIL_OUT "No session daemon to kill"
 757                 fi
 758                 return 0
 759         fi
 760
 761         diag "Killing (signal $signal) $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 762
 763         # shellcheck disable=SC2086
 764         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 765                 retval=1
 766                 if [ "$withtap" -eq "1" ]; then
 767                         fail "Kill sessions daemon"
 768                 fi
 769         else
 770                 out=1
 771                 while [ -n "$out" ]; do
 772                         out=$(lttng_pgrep "${SESSIOND_MATCH}")
 773                         if [ -n "$dtimeleft_s" ]; then
 774                                 if [ $dtimeleft_s -lt 0 ]; then
 775                                         out=
 776                                         retval=1
 777                                 fi
 778                                 dtimeleft_s=$((dtimeleft_s - 1))
 779                         fi
 780                         sleep 0.5
 781                 done
 782                 out=1
 783                 while [ -n "$out" ]; do
 784                         out=$(lttng_pgrep "$CONSUMERD_MATCH")
 785                         if [ -n "$dtimeleft_s" ]; then
 786                                 if [ $dtimeleft_s -lt 0 ]; then
 787                                         out=
 788                                         retval=1
 789                                 fi
 790                                 dtimeleft_s=$((dtimeleft_s - 1))
 791                         fi
 792                         sleep 0.5
 793                 done
 794
 795                 if [ "$withtap" -eq "1" ]; then
 796                         if [ "$retval" -eq "0" ]; then
 797                                 pass "Wait after kill session daemon"
 798                         else
 799                                 fail "Wait after kill session daemon"
 800                         fi
 801                 fi
 802         fi
 803         if [ "$signal" = "SIGKILL" ]; then
 804                 if [ "$(id -u)" -eq "0" ]; then
 805                         local modules=
 806                         modules="$(lsmod | grep ^lttng | awk '{print $1}')"
 807
 808                         if [ -n "$modules" ]; then
 809                                 diag "Unloading all LTTng modules"
 810                                 modprobe --remove "$modules"
 811                         fi
 812                 fi
 813         fi
 814
 815         return $retval
 816 }
 817
 818 function stop_lttng_sessiond()
 819 {
 820         stop_lttng_sessiond_opt 1 0 "$@"
 821 }
 822
 823 function stop_lttng_sessiond_notap()
 824 {
 825         stop_lttng_sessiond_opt 0 0 "$@"
 826 }
 827
 828 function stop_lttng_sessiond_cleanup()
 829 {
 830         stop_lttng_sessiond_opt 0 1 "$@"
 831 }
 832
 833 function sigstop_lttng_sessiond_opt()
 834 {
 835         local withtap=$1
 836         local signal=SIGSTOP
 837         local pids
 838
 839         if [ -n "$TEST_NO_SESSIOND" ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 840                 # Env variable requested no session daemon
 841                 return
 842         fi
 843
 844         pids="$(lttng_pgrep "${SESSIOND_MATCH}") $(lttng_pgrep "$RUNAS_MATCH")"
 845
 846         if [ "$withtap" -eq "1" ]; then
 847                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 848         fi
 849
 850         # shellcheck disable=SC2086
 851         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 852                 if [ "$withtap" -eq "1" ]; then
 853                         fail "Sending SIGSTOP to session daemon"
 854                 fi
 855         else
 856                 out=1
 857                 while [ $out -ne 0 ]; do
 858                         pids="$(lttng_pgrep "$SESSIOND_MATCH")"
 859
 860                         # Wait until state becomes stopped for session
 861                         # daemon(s).
 862                         out=0
 863                         for sessiond_pid in $pids; do
 864                                 state="$(ps -p "$sessiond_pid" -o state= )"
 865                                 if [[ -n "$state" && "$state" != "T" ]]; then
 866                                         out=1
 867                                 fi
 868                         done
 869                         sleep 0.5
 870                 done
 871                 if [ "$withtap" -eq "1" ]; then
 872                         pass "Sending SIGSTOP to session daemon"
 873                 fi
 874         fi
 875 }
 876
 877 function sigstop_lttng_sessiond()
 878 {
 879         sigstop_lttng_sessiond_opt 1 "$@"
 880 }
 881
 882 function sigstop_lttng_sessiond_notap()
 883 {
 884         sigstop_lttng_sessiond_opt 0 "$@"
 885 }
 886
 887 function stop_lttng_consumerd_opt()
 888 {
 889         local withtap=$1
 890         local is_cleanup=$2
 891         local signal=$3
 892         local timeout_s=$4
 893         local dtimeleft_s=
 894         local retval=0
 895         local pids
 896
 897         if [ -z "$signal" ]; then
 898                 signal=SIGTERM
 899         fi
 900
 901         # Multiply time by 2 to simplify integer arithmetic
 902         if [ -n "$timeout_s" ]; then
 903                 dtimeleft_s=$((timeout_s * 2))
 904         fi
 905
 906         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 907
 908         if [ -z "$pids" ]; then
 909                 if [ "$is_cleanup" -eq 1 ]; then
 910                         :
 911                 elif [ "$withtap" -eq "1" ]; then
 912                         fail "No consumerd daemon to kill"
 913                 else
 914                         LTTNG_BAIL_OUT "No consumerd daemon to kill"
 915                 fi
 916                 return 0
 917         fi
 918
 919         diag "Killing (signal $signal) $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 920
 921         # shellcheck disable=SC2086
 922         if ! kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST; then
 923                 retval=1
 924                 if [ "$withtap" -eq "1" ]; then
 925                         fail "Kill consumer daemon"
 926                 fi
 927         else
 928                 out=1
 929                 while [ $out -ne 0 ]; do
 930                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 931
 932                         # If consumerds are still present check their status.
 933                         # A zombie status qualifies the consumerd as *killed*
 934                         out=0
 935                         for consumer_pid in $pids; do
 936                                 state="$(ps -p "$consumer_pid" -o state= )"
 937                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 938                                         out=1
 939                                 fi
 940                         done
 941                         if [ -n "$dtimeleft_s" ]; then
 942                                 if [ $dtimeleft_s -lt 0 ]; then
 943                                         out=0
 944                                         retval=1
 945                                 fi
 946                                 dtimeleft_s=$((dtimeleft_s - 1))
 947                         fi
 948                         sleep 0.5
 949                 done
 950                 if [ "$withtap" -eq "1" ]; then
 951                         if [ "$retval" -eq "0" ]; then
 952                                 pass "Wait after kill consumer daemon"
 953                         else
 954                                 fail "Wait after kill consumer daemon"
 955                         fi
 956                 fi
 957         fi
 958
 959         return $retval
 960 }
 961
 962 function stop_lttng_consumerd()
 963 {
 964         stop_lttng_consumerd_opt 1 0 "$@"
 965 }
 966
 967 function stop_lttng_consumerd_notap()
 968 {
 969         stop_lttng_consumerd_opt 0 0 "$@"
 970 }
 971
 972 function stop_lttng_consumerd_cleanup()
 973 {
 974         stop_lttng_consumerd_opt 0 1 "$@"
 975 }
 976
 977 function sigstop_lttng_consumerd_opt()
 978 {
 979         local withtap=$1
 980         local signal=SIGSTOP
 981         local pids
 982
 983         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
 984
 985         diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo "$pids" | tr '\n' ' ')"
 986
 987         # shellcheck disable=SC2086
 988         kill -s $signal $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 989         retval=$?
 990
 991         if [ $retval -eq 1 ]; then
 992                 if [ "$withtap" -eq "1" ]; then
 993                         fail "Sending SIGSTOP to consumer daemon"
 994                 fi
 995                 return 1
 996         else
 997                 out=1
 998                 while [ $out -ne 0 ]; do
 999                         pids="$(lttng_pgrep "$CONSUMERD_MATCH")"
1000
1001                         # Wait until state becomes stopped for all
1002                         # consumers.
1003                         out=0
1004                         for consumer_pid in $pids; do
1005                                 state="$(ps -p "$consumer_pid" -o state= )"
1006                                 if [[ -n "$state" && "$state" != "T" ]]; then
1007                                         out=1
1008                                 fi
1009                         done
1010                         sleep 0.5
1011                 done
1012                 if [ "$withtap" -eq "1" ]; then
1013                         pass "Sending SIGSTOP to consumer daemon"
1014                 fi
1015         fi
1016         return $retval
1017 }
1018
1019 function sigstop_lttng_consumerd()
1020 {
1021         sigstop_lttng_consumerd_opt 1 "$@"
1022 }
1023
1024 function sigstop_lttng_consumerd_notap()
1025 {
1026         sigstop_lttng_consumerd_opt 0 "$@"
1027 }
1028
1029 function list_lttng_with_opts ()
1030 {
1031         local ret
1032         local withtap=$1
1033         shift
1034         local opts=$1
1035         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1036                 list $opts
1037         ret=$?
1038         if [ $withtap -eq "1" ]; then
1039                 ok $ret "Lttng-tool list command with option $opts"
1040         fi
1041 }
1042
1043 function list_lttng_ok ()
1044 {
1045         list_lttng_with_opts 1 "$@"
1046 }
1047
1048 function list_lttng_notap ()
1049 {
1050         list_lttng_with_opts 0 "$@"
1051 }
1052
1053 function create_lttng_session_no_output ()
1054 {
1055         local sess_name=$1
1056         local opts="${@:2}"
1057         local trace_format=("--trace-format" "ctf1")
1058
1059         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1060                 create "${trace_format[@]}" $sess_name --no-output $opts
1061         ok $? "Create session $sess_name in no-output mode"
1062 }
1063
1064 function create_lttng_session_uri () {
1065         local sess_name=$1
1066         local uri=$2
1067         local opts="${@:3}"
1068         local trace_format=("--trace-format" "ctf1")
1069
1070         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1071                 create "${trace_format[@]}" $sess_name -U $uri $opts
1072         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
1073 }
1074
1075 function create_lttng_session ()
1076 {
1077         local withtap=$1
1078         local expected_to_fail=$2
1079         local sess_name=$3
1080         local trace_path=$4
1081         local opt=$5
1082         local trace_format=("--trace-format" "ctf1")
1083
1084         if [ -z "$trace_path" ]; then
1085                 # Use lttng-sessiond default output.
1086                 trace_path=""
1087         else
1088                 trace_path="-o $trace_path"
1089         fi
1090
1091         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1092                 create "${trace_format[@]}" "$sess_name" $trace_path $opt
1093         ret=$?
1094         if [ $expected_to_fail -eq "1" ]; then
1095                 test "$ret" -ne "0"
1096                 ret=$?
1097                 if [ $withtap -eq "1" ]; then
1098                         ok $ret "Create session $sess_name in $trace_path failed as expected"
1099                 fi
1100         else
1101                 if [ $withtap -eq "1" ]; then
1102                         ok $ret "Create session $sess_name in $trace_path"
1103                 fi
1104         fi
1105         return $ret
1106 }
1107
1108 function create_lttng_session_ok ()
1109 {
1110         create_lttng_session 1 0 "$@"
1111 }
1112
1113 function create_lttng_session_fail ()
1114 {
1115         create_lttng_session 1 1 "$@"
1116 }
1117
1118 function create_lttng_session_notap ()
1119 {
1120         create_lttng_session 0 0 "$@"
1121 }
1122
1123
1124 function enable_ust_lttng_channel ()
1125 {
1126         local withtap=$1
1127         local expected_to_fail=$2
1128         local sess_name=$3
1129         local channel_name=$4
1130         local opts="${@:5}"
1131
1132         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1133                 enable-channel -u $channel_name -s $sess_name $opts
1134         ret=$?
1135         if [[ $expected_to_fail -eq "1" ]]; then
1136                 test "$ret" -ne "0"
1137                 ret=$?
1138                 if [ $withtap -eq "1" ]; then
1139                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
1140                 fi
1141         else
1142                 if [ $withtap -eq "1" ]; then
1143                         ok $ret "Enable channel $channel_name for session $sess_name"
1144                 fi
1145         fi
1146         return $ret
1147 }
1148
1149 function enable_ust_lttng_channel_ok ()
1150 {
1151         enable_ust_lttng_channel 1 0 "$@"
1152 }
1153
1154 function enable_ust_lttng_channel_fail ()
1155 {
1156         enable_ust_lttng_channel 1 1 "$@"
1157 }
1158
1159 function enable_ust_lttng_channel_notap ()
1160 {
1161         enable_ust_lttng_channel 0 0 "$@"
1162 }
1163
1164 function disable_ust_lttng_channel()
1165 {
1166         local sess_name=$1
1167         local channel_name=$2
1168
1169         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1170                 disable-channel -u $channel_name -s $sess_name
1171         ok $? "Disable channel $channel_name for session $sess_name"
1172 }
1173
1174 function enable_lttng_mmap_overwrite_kernel_channel()
1175 {
1176         local sess_name=$1
1177         local channel_name=$2
1178
1179         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1180                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite
1181         ok $? "Enable channel $channel_name for session $sess_name"
1182 }
1183
1184 function enable_lttng_mmap_discard_small_kernel_channel()
1185 {
1186         local sess_name=$1
1187         local channel_name=$2
1188
1189         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1190                 enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1191         ok $? "Enable small discard channel $channel_name for session $sess_name"
1192 }
1193
1194 function enable_lttng_mmap_overwrite_small_kernel_channel()
1195 {
1196         local sess_name=$1
1197         local channel_name=$2
1198
1199         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1200                 enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2
1201         ok $? "Enable small discard channel $channel_name for session $sess_name"
1202 }
1203
1204 function enable_lttng_mmap_overwrite_ust_channel()
1205 {
1206         local sess_name=$1
1207         local channel_name=$2
1208
1209         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1210                 enable-channel -s $sess_name $channel_name -u --output mmap --overwrite
1211         ok $? "Enable channel $channel_name for session $sess_name"
1212 }
1213
1214 function enable_ust_lttng_event ()
1215 {
1216         local withtap=$1
1217         local expected_to_fail=$2
1218         local sess_name=$3
1219         local event_name="$4"
1220         local channel_name=$5
1221
1222         if [ -z $channel_name ]; then
1223                 # default channel if none specified
1224                 chan=""
1225         else
1226                 chan="-c $channel_name"
1227         fi
1228
1229         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1230                 enable-event "$event_name" $chan -s "$sess_name" -u
1231         ret=$?
1232         if [[ $expected_to_fail -eq "1" ]]; then
1233                 test $ret -ne "0"
1234                 ret=$?
1235                 if [[ $withtap -eq "1" ]]; then
1236                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
1237                 fi
1238         else
1239                 if [[ $withtap -eq "1" ]]; then
1240                         ok $ret "Enable ust event $event_name for session $sess_name"
1241                 fi
1242         fi
1243         return $ret
1244 }
1245
1246 function enable_ust_lttng_event_ok ()
1247 {
1248         enable_ust_lttng_event 1 0 "$@"
1249 }
1250
1251 function enable_ust_lttng_event_fail ()
1252 {
1253         enable_ust_lttng_event 1 1 "$@"
1254 }
1255
1256 function enable_ust_lttng_event_notap ()
1257 {
1258         enable_ust_lttng_event 0 0 "$@"
1259 }
1260
1261 function enable_jul_lttng_event()
1262 {
1263         sess_name=$1
1264         event_name="$2"
1265         channel_name=$3
1266
1267         if [ -z $channel_name ]; then
1268                 # default channel if none specified
1269                 chan=""
1270         else
1271                 chan="-c $channel_name"
1272         fi
1273
1274         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1275                 enable-event "$event_name" $chan -s $sess_name -j
1276         ok $? "Enable JUL event $event_name for session $sess_name"
1277 }
1278
1279 function enable_jul_lttng_event_loglevel()
1280 {
1281         local sess_name=$1
1282         local event_name="$2"
1283         local loglevel=$3
1284         local channel_name=$4
1285
1286         if [ -z $channel_name ]; then
1287                 # default channel if none specified
1288                 chan=""
1289         else
1290                 chan="-c $channel_name"
1291         fi
1292
1293         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1294                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j
1295         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
1296 }
1297
1298 function enable_log4j_lttng_event()
1299 {
1300         local sess_name=$1
1301         local event_name=$2
1302         local channel_name=$3
1303
1304         local chan_opt=()
1305
1306         # default channel if none specified
1307         if [ -n "$channel_name" ]; then
1308                 chan_opt=("-c" "$channel_name")
1309         fi
1310
1311         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1312                 enable-event "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1313         ok $? "Enable LOG4J event '$event_name' for session '$sess_name'"
1314 }
1315
1316 function enable_log4j_lttng_event_filter()
1317 {
1318         local sess_name=$1
1319         local event_name=$2
1320         local filter=$3
1321
1322         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1323                 enable-event "$event_name" -s "$sess_name" --log4j --filter "$filter"
1324         ok $? "Enable LOG4J event '$event_name' with filter '$filter' for session '$sess_name'"
1325 }
1326
1327 function enable_log4j_lttng_event_filter_loglevel_only()
1328 {
1329         local sess_name=$1
1330         local event_name=$2
1331         local filter=$3
1332         local loglevel=$4
1333
1334         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1335                 enable-event --loglevel-only "$loglevel" "$event_name" -s "$sess_name" -l --filter "$filter"
1336         ok $? "Enable LOG4J event '$event_name' with filter '$filter' and loglevel-only '$loglevel' for session '$sess_name'"
1337 }
1338
1339 function enable_log4j_lttng_event_loglevel()
1340 {
1341         local sess_name=$1
1342         local event_name=$2
1343         local loglevel=$3
1344         local channel_name=$4
1345
1346
1347         # default channel if none specified
1348         if [ -n "$channel_name" ]; then
1349                 chan_opt=("-c" "$channel_name")
1350         fi
1351
1352         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1353                 enable-event --loglevel "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1354         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel '$loglevel'"
1355 }
1356
1357 function enable_log4j_lttng_event_loglevel_only()
1358 {
1359         local sess_name=$1
1360         local event_name=$2
1361         local loglevel=$3
1362         local channel_name=$4
1363
1364         local chan_opt=()
1365
1366         # default channel if none specified
1367         if [ -n "$channel_name" ]; then
1368                 chan_opt=("-c" "$channel_name")
1369         fi
1370
1371         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1372                 enable-event --loglevel-only "$loglevel" "$event_name" "${chan_opt[@]}" -s "$sess_name" --log4j
1373         ok $? "Enable LOG4J event '$event_name' for session '$sess_name' with loglevel-only '$loglevel'"
1374 }
1375
1376 function enable_python_lttng_event()
1377 {
1378         sess_name=$1
1379         event_name="$2"
1380         channel_name=$3
1381
1382         if [ -z $channel_name ]; then
1383                 # default channel if none specified
1384                 chan=""
1385         else
1386                 chan="-c $channel_name"
1387         fi
1388
1389         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1390                 enable-event "$event_name" $chan -s $sess_name -p
1391         ok $? "Enable Python event $event_name for session $sess_name"
1392 }
1393
1394 function enable_python_lttng_event_loglevel()
1395 {
1396         local sess_name=$1
1397         local event_name="$2"
1398         local loglevel=$3
1399         local channel_name=$4
1400
1401         if [ -z $channel_name ]; then
1402                 # default channel if none specified
1403                 chan=""
1404         else
1405                 chan="-c $channel_name"
1406         fi
1407
1408         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1409                 enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p
1410         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1411 }
1412
1413 function enable_ust_lttng_event_filter()
1414 {
1415         local sess_name="$1"
1416         local event_name="$2"
1417         local filter="$3"
1418         local channel_name=$4
1419
1420         if [ -z $channel_name ]; then
1421                 # default channel if none specified
1422                 chan=""
1423         else
1424                 chan="-c $channel_name"
1425         fi
1426
1427         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1428                 enable-event $chan "$event_name" -s $sess_name -u --filter "$filter"
1429         ok $? "Enable event $event_name with filtering for session $sess_name"
1430 }
1431
1432 function enable_ust_lttng_event_loglevel()
1433 {
1434         local sess_name="$1"
1435         local event_name="$2"
1436         local loglevel="$3"
1437
1438         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1439                 enable-event "$event_name" -s $sess_name -u --loglevel $loglevel
1440         ok $? "Enable event $event_name with loglevel $loglevel"
1441 }
1442
1443 function enable_ust_lttng_event_loglevel_only()
1444 {
1445         local sess_name="$1"
1446         local event_name="$2"
1447         local loglevel="$3"
1448
1449         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1450                 enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel
1451         ok $? "Enable event $event_name with loglevel-only $loglevel"
1452 }
1453
1454 function disable_ust_lttng_event ()
1455 {
1456         local sess_name="$1"
1457         local event_name="$2"
1458         local channel_name="$3"
1459
1460         if [ -z $channel_name ]; then
1461                 # default channel if none specified
1462                 chan=""
1463         else
1464                 chan="-c $channel_name"
1465         fi
1466
1467         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1468                 disable-event "$event_name" -s $sess_name $chan -u
1469         ok $? "Disable event $event_name for session $sess_name"
1470 }
1471
1472 function disable_jul_lttng_event ()
1473 {
1474         local sess_name="$1"
1475         local event_name="$2"
1476
1477         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1478         ok $? "Disable JUL event $event_name for session $sess_name"
1479 }
1480
1481 function disable_log4j_lttng_event ()
1482 {
1483         local sess_name="$1"
1484         local event_name="$2"
1485
1486         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1487                 disable-event "$event_name" -s "$sess_name" --log4j
1488         ok $? "Disable LOG4J event '$event_name' for session '$sess_name'"
1489 }
1490
1491 function disable_python_lttng_event ()
1492 {
1493         local sess_name="$1"
1494         local event_name="$2"
1495
1496         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1497                 disable-event "$event_name" -s $sess_name -p
1498         ok $? "Disable Python event $event_name for session $sess_name"
1499 }
1500
1501 function start_lttng_tracing_opt ()
1502 {
1503         local withtap=$1
1504         local expected_to_fail=$2
1505         local sess_name=$3
1506
1507         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1508                 start $sess_name
1509         ret=$?
1510         if [[ $expected_to_fail -eq "1" ]]; then
1511                 test "$ret" -ne "0"
1512                 ret=$?
1513                 if [ $withtap -eq "1" ]; then
1514                         ok $? "Start tracing for session $sess_name failed as expected"
1515                 fi
1516         else
1517                 if [ $withtap -eq "1" ]; then
1518                         ok $ret "Start tracing for session $sess_name"
1519                 fi
1520         fi
1521 }
1522
1523 function start_lttng_tracing_ok ()
1524 {
1525         start_lttng_tracing_opt 1 0 "$@"
1526 }
1527
1528 function start_lttng_tracing_fail ()
1529 {
1530         start_lttng_tracing_opt 1 1 "$@"
1531 }
1532
1533 function start_lttng_tracing_notap ()
1534 {
1535         start_lttng_tracing_opt 0 1 "$@"
1536 }
1537
1538 function stop_lttng_tracing_opt ()
1539 {
1540         local withtap=$1
1541         local expected_to_fail=$2
1542         local sess_name=$3
1543
1544         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1545                 stop $sess_name
1546         ret=$?
1547         if [[ $expected_to_fail -eq "1" ]]; then
1548                 test "$ret" -ne "0"
1549                 ret=$?
1550                 if [ $withtap -eq "1" ]; then
1551                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1552                 fi
1553         else
1554                 if [ $withtap -eq "1" ]; then
1555                         ok $ret "Stop lttng tracing for session $sess_name"
1556                 fi
1557         fi
1558 }
1559
1560 function stop_lttng_tracing_ok ()
1561 {
1562         stop_lttng_tracing_opt 1 0 "$@"
1563 }
1564
1565 function stop_lttng_tracing_fail ()
1566 {
1567         stop_lttng_tracing_opt 1 1 "$@"
1568 }
1569
1570 function stop_lttng_tracing_notap ()
1571 {
1572         stop_lttng_tracing_opt 0 0 "$@"
1573 }
1574
1575 function destroy_lttng_session ()
1576 {
1577         local withtap=$1
1578         local expected_to_fail=$2
1579         local sess_name=$3
1580
1581         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1582                 destroy $sess_name
1583         ret=$?
1584         if [[ $expected_to_fail -eq "1" ]]; then
1585                 test "$ret" -ne "0"
1586                 ret=$?
1587                 if [ $withtap -eq "1" ]; then
1588                         ok $ret "Destroy session $sess_name failed as expected"
1589                 fi
1590         else
1591                 if [ $withtap -eq "1" ]; then
1592                         ok $ret "Destroy session $sess_name"
1593                 fi
1594         fi
1595 }
1596
1597 function destroy_lttng_session_ok ()
1598 {
1599         destroy_lttng_session 1 0 "$@"
1600
1601 }
1602
1603 function destroy_lttng_session_fail ()
1604 {
1605         destroy_lttng_session 1 1 "$@"
1606 }
1607
1608 function destroy_lttng_session_notap ()
1609 {
1610         destroy_lttng_session 0 0 "$@"
1611 }
1612
1613 function destroy_lttng_sessions ()
1614 {
1615         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1616                 destroy --all
1617         ok $? "Destroy all lttng sessions"
1618 }
1619
1620 function lttng_snapshot_add_output ()
1621 {
1622         local expected_to_fail=$1
1623         local sess_name=$2
1624         local trace_path=$3
1625         local opts=$4
1626
1627         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1628                 snapshot add-output -s $sess_name $trace_path $opts
1629         ret=$?
1630         if [[ $expected_to_fail -eq 1 ]]; then
1631                 test "$ret" -ne "0"
1632                 ok $? "Added snapshot output $trace_path failed as expected"
1633         else
1634                 ok $ret "Added snapshot output $trace_path"
1635         fi
1636 }
1637
1638 function lttng_snapshot_add_output_ok ()
1639 {
1640         lttng_snapshot_add_output 0 "$@"
1641 }
1642
1643 function lttng_snapshot_add_output_fail ()
1644 {
1645         lttng_snapshot_add_output 1 "$@"
1646 }
1647
1648 function lttng_snapshot_del_output ()
1649 {
1650         local expected_to_fail=$1
1651         local sess_name=$2
1652         local id=$3
1653
1654         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1655                 snapshot del-output -s $sess_name $id
1656         ret=$?
1657         if [[ $expected_to_fail -eq "1" ]]; then
1658                 test "$ret" -ne "0"
1659                 ok $? "Deleted snapshot output id $id failed as expected"
1660         else
1661                 ok $ret "Deleted snapshot output id $id"
1662         fi
1663 }
1664
1665 function lttng_snapshot_del_output_ok ()
1666 {
1667         lttng_snapshot_del_output 0 "$@"
1668 }
1669
1670 function lttng_snapshot_del_output_fail ()
1671 {
1672         lttng_snapshot_del_output 1 "$@"
1673 }
1674
1675 function lttng_snapshot_record ()
1676 {
1677         local sess_name=$1
1678         local trace_path=$2
1679
1680         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1681                 snapshot record -s "$sess_name" "$trace_path"
1682         ok $? "Snapshot recorded"
1683 }
1684
1685 function lttng_snapshot_list ()
1686 {
1687         local sess_name=$1
1688         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1689                 snapshot list-output -s $sess_name
1690         ok $? "Snapshot list"
1691 }
1692
1693 function lttng_save()
1694 {
1695         local sess_name=$1
1696         local opts=$2
1697
1698         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1699                 save $sess_name $opts
1700         ok $? "Session saved"
1701 }
1702
1703 function lttng_load()
1704 {
1705         local expected_to_fail=$1
1706         local opts=$2
1707
1708         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1709                 load $opts
1710         ret=$?
1711         if [[ $expected_to_fail -eq "1" ]]; then
1712                 test $ret -ne "0"
1713                 ok $? "Load command failed as expected with opts: $opts"
1714         else
1715                 ok $ret "Load command with opts: $opts"
1716         fi
1717 }
1718
1719 function lttng_load_ok()
1720 {
1721         lttng_load 0 "$@"
1722 }
1723
1724 function lttng_load_fail()
1725 {
1726         lttng_load 1 "$@"
1727 }
1728
1729 function lttng_track()
1730 {
1731         local expected_to_fail="$1"
1732         shift 1
1733         local opts="$@"
1734         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1735         ret=$?
1736         if [[ $expected_to_fail -eq "1" ]]; then
1737                 test $ret -ne "0"
1738                 ok $? "Track command failed as expected with opts: $opts"
1739         else
1740                 ok $ret "Track command with opts: $opts"
1741         fi
1742 }
1743
1744 function lttng_track_ok()
1745 {
1746         lttng_track 0 "$@"
1747 }
1748
1749 function lttng_track_fail()
1750 {
1751         lttng_track 1 "$@"
1752 }
1753
1754 function lttng_untrack()
1755 {
1756         local expected_to_fail="$1"
1757         shift 1
1758         local opts="$@"
1759         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1760         ret=$?
1761         if [[ $expected_to_fail -eq "1" ]]; then
1762                 test $ret -ne "0"
1763                 ok $? "Untrack command failed as expected with opts: $opts"
1764         else
1765                 ok $ret "Untrack command with opts: $opts"
1766         fi
1767 }
1768
1769 function lttng_untrack_ok()
1770 {
1771         lttng_untrack 0 "$@"
1772 }
1773
1774 function lttng_untrack_fail()
1775 {
1776         lttng_untrack 1 "$@"
1777 }
1778
1779 function lttng_track_pid_ok()
1780 {
1781         PID=$1
1782         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1783         ok $? "Lttng track pid on the kernel domain"
1784 }
1785
1786 function lttng_untrack_kernel_all_ok()
1787 {
1788         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1789         ok $? "Lttng untrack all pid on the kernel domain"
1790 }
1791
1792 function lttng_track_ust_ok()
1793 {
1794         lttng_track_ok -u "$@"
1795 }
1796
1797 function lttng_track_ust_fail()
1798 {
1799         lttng_track_fail -u "$@"
1800 }
1801
1802 function lttng_track_kernel_ok()
1803 {
1804         lttng_track_ok -k "$@"
1805 }
1806
1807 function lttng_track_kernel_fail()
1808 {
1809         lttng_track_fail -k "$@"
1810 }
1811
1812 function lttng_untrack_ust_ok()
1813 {
1814         lttng_untrack_ok -u "$@"
1815 }
1816
1817 function lttng_untrack_ust_fail()
1818 {
1819         lttng_untrack_fail -u "$@"
1820 }
1821
1822 function lttng_untrack_kernel_ok()
1823 {
1824         lttng_untrack_ok -k "$@"
1825 }
1826
1827 function lttng_untrack_kernel_fail()
1828 {
1829         lttng_untrack_fail -k "$@"
1830 }
1831
1832 function lttng_add_context_list()
1833 {
1834         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1835                 add-context --list
1836         ret=$?
1837         ok $ret "Context listing"
1838 }
1839
1840 function add_context_lttng()
1841 {
1842         local expected_to_fail="$1"
1843         local domain="$2"
1844         local session_name="$3"
1845         local channel_name="$4"
1846         local type="$5"
1847
1848         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
1849                 add-context -s $session_name -c $channel_name -t $type $domain
1850         ret=$?
1851         if [[ $expected_to_fail -eq "1" ]]; then
1852                 test $ret -ne "0"
1853                 ok $? "Add context command failed as expected for type: $type"
1854         else
1855                 ok $ret "Add context command for type: $type"
1856         fi
1857 }
1858
1859 function add_context_ust_ok()
1860 {
1861         add_context_lttng 0 -u "$@"
1862 }
1863
1864 function add_context_ust_fail()
1865 {
1866         add_context_lttng 1 -u "$@"
1867 }
1868
1869 function add_context_kernel_ok()
1870 {
1871         add_context_lttng 0 -k "$@"
1872 }
1873
1874 function add_context_kernel_fail()
1875 {
1876         add_context_lttng 1 -k "$@"
1877 }
1878
1879 function wait_live_trace_ready ()
1880 {
1881         local url=$1
1882         local zero_client_match=0
1883
1884         diag "Waiting for live trace at url: $url"
1885         while [ $zero_client_match -eq 0 ]; do
1886                 zero_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "0 client(s) connected" | wc -l)
1887                 sleep 0.5
1888         done
1889         pass "Waiting for live trace at url: $url"
1890 }
1891
1892 function wait_live_viewer_connect ()
1893 {
1894         local url=$1
1895         local one_client_match=0
1896
1897         diag "Waiting for live viewers on url: $url"
1898         while [ $one_client_match -eq 0 ]; do
1899                 one_client_match=$($BABELTRACE_BIN -i lttng-live $url | grep "1 client(s) connected" | wc -l)
1900                 sleep 0.5
1901         done
1902         pass "Waiting for live viewers on url: $url"
1903 }
1904
1905 function bail_out_if_no_babeltrace()
1906 {
1907         which "$BABELTRACE_BIN" >/dev/null
1908         if [ $? -ne 0 ]; then
1909                 LTTNG_BAIL_OUT "\"$BABELTRACE_BIN\" binary not found. Skipping tests"
1910         fi
1911 }
1912
1913 function validate_metadata_event ()
1914 {
1915         local event_name=$1
1916         local nr_event_id=$2
1917         local trace_path=$3
1918
1919         local metadata_file=$(find $trace_path -name "metadata")
1920         local metadata_path=$(dirname $metadata_file)
1921
1922         bail_out_if_no_babeltrace
1923
1924         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1925
1926         if [ "$count" -ne "$nr_event_id" ]; then
1927                 fail "Metadata match with the metadata of $nr_event_id event(s) named $event_name"
1928                 diag "$count matching event names found in metadata"
1929         else
1930                 pass "Metadata match with the metadata of $nr_event_id event(s) named $event_name"
1931         fi
1932
1933 }
1934
1935 function trace_matches ()
1936 {
1937         local event_name=$1
1938         local nr_iter=$2
1939         local trace_path=$3
1940
1941         bail_out_if_no_babeltrace
1942
1943         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1944
1945         if [ "$count" -ne "$nr_iter" ]; then
1946                 fail "Trace match"
1947                 diag "$count matching events found in trace"
1948         else
1949                 pass "Trace match"
1950         fi
1951 }
1952
1953 function trace_match_only()
1954 {
1955         local event_name=$1
1956         local nr_iter=$2
1957         local trace_path=$3
1958
1959         bail_out_if_no_babeltrace
1960         #which "$BABELTRACE_BIN" >/dev/null
1961         #skip $? -ne 0 "\"$BABELTRACE_BIN\" binary not found. Skipping trace comparison"
1962
1963         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1964         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1965
1966         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1967                 pass "Trace match with $total event $event_name"
1968         else
1969                 fail "Trace match"
1970                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1971         fi
1972 }
1973
1974 function validate_trace
1975 {
1976         local event_name=$1
1977         local trace_path=$2
1978
1979         bail_out_if_no_babeltrace
1980
1981         OLDIFS=$IFS
1982         IFS=","
1983         for i in $event_name; do
1984                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1985                 if [ "$traced" -ne 0 ]; then
1986                         pass "Validate trace for event $i, $traced events"
1987                 else
1988                         fail "Validate trace for event $i"
1989                         diag "Found $traced occurrences of $i"
1990                 fi
1991         done
1992         ret=$?
1993         IFS=$OLDIFS
1994         return $ret
1995 }
1996
1997 function validate_trace_count
1998 {
1999         local event_name=$1
2000         local trace_path=$2
2001         local expected_count=$3
2002
2003         bail_out_if_no_babeltrace
2004
2005         cnt=0
2006         OLDIFS=$IFS
2007         IFS=","
2008         for i in $event_name; do
2009                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2010                 if [ "$traced" -ne 0 ]; then
2011                         pass "Validate trace for event $i, $traced events"
2012                 else
2013                         fail "Validate trace for event $i"
2014                         diag "Found $traced occurrences of $i"
2015                 fi
2016                 cnt=$(($cnt + $traced))
2017         done
2018         IFS=$OLDIFS
2019         test $cnt -eq $expected_count
2020         ok $? "Read a total of $cnt events, expected $expected_count"
2021 }
2022
2023 function validate_trace_count_range_incl_min_excl_max
2024 {
2025         local event_name=$1
2026         local trace_path=$2
2027         local expected_min=$3
2028         local expected_max=$4
2029
2030         bail_out_if_no_babeltrace
2031
2032         cnt=0
2033         OLDIFS=$IFS
2034         IFS=","
2035         for i in $event_name; do
2036                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
2037                 if [ "$traced" -ge $expected_min ]; then
2038                         pass "Validate trace for event $i, $traced events"
2039                 else
2040                         fail "Validate trace for event $i"
2041                         diag "Found $traced occurrences of $i"
2042                 fi
2043                 cnt=$(($cnt + $traced))
2044         done
2045         IFS=$OLDIFS
2046         test $cnt -lt $expected_max
2047         ok $? "Read a total of $cnt events, expected between [$expected_min, $expected_max["
2048 }
2049
2050 function trace_first_line
2051 {
2052         local trace_path=$1
2053
2054         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
2055 }
2056
2057 function validate_trace_exp()
2058 {
2059         local event_exp=$1
2060         local trace_path=$2
2061
2062         bail_out_if_no_babeltrace
2063
2064         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
2065         if [ "$traced" -ne 0 ]; then
2066                 pass "Validate trace for expression '${event_exp}', $traced events"
2067         else
2068                 fail "Validate trace for expression '${event_exp}'"
2069                 diag "Found $traced occurrences of '${event_exp}'"
2070         fi
2071         ret=$?
2072         return $ret
2073 }
2074
2075 function validate_trace_only_exp()
2076 {
2077         local event_exp=$1
2078         local trace_path=$2
2079
2080         bail_out_if_no_babeltrace
2081
2082         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
2083         local total=$($BABELTRACE_BIN $trace_path | wc -l)
2084
2085         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
2086                 pass "Trace match with $total for expression '${event_exp}'"
2087         else
2088                 fail "Trace match"
2089                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
2090         fi
2091         ret=$?
2092         return $ret
2093 }
2094
2095 function validate_trace_empty()
2096 {
2097         local trace_path=$1
2098
2099         bail_out_if_no_babeltrace
2100
2101         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
2102         ret=$?
2103         if [ $ret -ne 0 ]; then
2104                 fail "Failed to parse trace"
2105                 return $ret
2106         fi
2107
2108         traced=$(echo -n "$events" | wc -l)
2109         if [ "$traced" -eq 0 ]; then
2110                 pass "Validate empty trace"
2111         else
2112                 fail "Validate empty trace"
2113                 diag "Found $traced events in trace"
2114         fi
2115         ret=$?
2116         return $ret
2117 }
2118
2119 function validate_directory_empty ()
2120 {
2121         local trace_path="$1"
2122
2123         # Do not double quote `$trace_path` below as we want wildcards to be
2124         # expanded.
2125         files="$(ls -A $trace_path)"
2126         ret=$?
2127         if [ $ret -ne 0 ]; then
2128                 fail "Failed to list content of directory \"$trace_path\""
2129                 return $ret
2130         fi
2131
2132         nb_files="$(echo -n "$files" | wc -l)"
2133         ok $nb_files "Directory \"$trace_path\" is empty"
2134 }
2135
2136 function validate_trace_session_ust_empty()
2137 {
2138         validate_directory_empty "$1"/ust
2139 }
2140
2141 function validate_trace_session_kernel_empty()
2142 {
2143         validate_trace_empty "$1"/kernel
2144 }
2145
2146 function regenerate_metadata ()
2147 {
2148         local expected_to_fail=$1
2149         local sess_name=$2
2150
2151         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2152                 regenerate metadata -s $sess_name
2153         ret=$?
2154         if [[ $expected_to_fail -eq "1" ]]; then
2155                 test "$ret" -ne "0"
2156                 ok $? "Expected fail on regenerate metadata $sess_name"
2157         else
2158                 ok $ret "Metadata regenerate $sess_name"
2159         fi
2160 }
2161
2162 function regenerate_metadata_ok ()
2163 {
2164         regenerate_metadata 0 "$@"
2165 }
2166
2167 function regenerate_metadata_fail ()
2168 {
2169         regenerate_metadata 1 "$@"
2170 }
2171
2172 function regenerate_statedump ()
2173 {
2174         local expected_to_fail=$1
2175         local sess_name=$2
2176
2177         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2178                 regenerate statedump -s $sess_name
2179         ret=$?
2180         if [[ $expected_to_fail -eq "1" ]]; then
2181                 test "$ret" -ne "0"
2182                 ok $? "Expected fail on regenerate statedump $sess_name"
2183         else
2184                 ok $ret "Statedump regenerate $sess_name"
2185         fi
2186 }
2187
2188 function regenerate_statedump_ok ()
2189 {
2190         regenerate_statedump 0 "$@"
2191 }
2192
2193 function regenerate_statedump_fail ()
2194 {
2195         regenerate_statedump 1 "$@"
2196 }
2197
2198 function rotate_session ()
2199 {
2200         local expected_to_fail=$1
2201         local sess_name=$2
2202
2203         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2204                 rotate $sess_name
2205         ret=$?
2206         if [[ $expected_to_fail -eq "1" ]]; then
2207                 test "$ret" -ne "0"
2208                 ok $? "Expected fail on rotate session $sess_name"
2209         else
2210                 ok $ret "Rotate session $sess_name"
2211         fi
2212 }
2213
2214 function rotate_session_ok ()
2215 {
2216         rotate_session 0 "$@"
2217 }
2218
2219 function rotate_session_fail ()
2220 {
2221         rotate_session 1 "$@"
2222 }
2223
2224 function destructive_tests_enabled ()
2225 {
2226         if [ "$LTTNG_ENABLE_DESTRUCTIVE_TESTS" = "will-break-my-system" ]; then
2227                 return 0
2228         else
2229                 return 1
2230         fi
2231 }
2232
2233 function lttng_enable_rotation_timer ()
2234 {
2235         local expected_to_fail=$1
2236         local sess_name=$2
2237         local period=$3
2238
2239         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2240                 enable-rotation -s $sess_name --timer $period
2241         ret=$?
2242         if [[ $expected_to_fail -eq "1" ]]; then
2243                 test "$ret" -ne "0"
2244                 ok $? "Expected fail when setting periodic rotation ($period) of session $sess_name"
2245         else
2246                 ok $ret "Set periodic rotation ($period) of session $sess_name"
2247         fi
2248 }
2249
2250 function lttng_enable_rotation_timer_ok ()
2251 {
2252         lttng_enable_rotation_timer 0 $@
2253 }
2254
2255 function lttng_enable_rotation_timer_fail ()
2256 {
2257         lttng_enable_rotation_timer 1 $@
2258 }
2259
2260 function lttng_enable_rotation_size ()
2261 {
2262         local expected_to_fail=$1
2263         local sess_name=$2
2264         local size=$3
2265
2266         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2267                 enable-rotation -s $sess_name --size $size
2268         ret=$?
2269         if [[ $expected_to_fail -eq "1" ]]; then
2270                 test "$ret" -ne "0"
2271                 ok $? "Expected to fail to set a periodic rotation of session $sess_name" "every " $size " bytes"
2272         else
2273                 ok $ret "Set a scheduled rotation of session $sess_name" "every " $size " bytes"
2274         fi
2275 }
2276
2277 function lttng_enable_rotation_size_ok ()
2278 {
2279         lttng_enable_rotation_size 0 $@
2280 }
2281
2282 function lttng_enable_rotation_size_fail ()
2283 {
2284         lttng_enable_rotation_size 1 $@
2285 }
2286
2287 function lttng_clear_session ()
2288 {
2289         local expected_to_fail=$1
2290         local sess_name=$2
2291
2292         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2293                 clear $sess_name
2294         ret=$?
2295         if [[ $expected_to_fail -eq "1" ]]; then
2296                 test "$ret" -ne "0"
2297                 ok $? "Expected fail on clear session $sess_name"
2298         else
2299                 ok $ret "Clear session $sess_name"
2300         fi
2301 }
2302
2303 function lttng_clear_session_ok ()
2304 {
2305         lttng_clear_session 0 $@
2306 }
2307
2308 function lttng_clear_session_fail ()
2309 {
2310         lttng_clear_session 1 $@
2311 }
2312
2313 function lttng_clear_all ()
2314 {
2315         _run_lttng_cmd "$OUTPUT_DEST" "$ERROR_OUTPUT_DEST" \
2316                 clear --all
2317         ok $? "Clear all lttng sessions"
2318 }
2319
2320 function lttng_add_trigger()
2321 {
2322         local expected_to_fail="$1"
2323         local trigger_name="$2"
2324         shift 2
2325         local args=("$@")
2326
2327         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name $trigger_name ${args[*]}"
2328         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-trigger --name "$trigger_name" "${args[@]}" 1> /dev/null 2> /dev/null
2329         ret=$?
2330         if [[ $expected_to_fail -eq "1" ]]; then
2331                 test "$ret" -ne "0"
2332                 ok $? "Add trigger $trigger_name failed as expected"
2333         else
2334                 ok $ret "Add trigger $trigger_name"
2335         fi
2336 }
2337
2338 function lttng_remove_trigger()
2339 {
2340         local expected_to_fail="$1"
2341         local trigger_name="$2"
2342         shift 2
2343
2344         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN remove-trigger $trigger_name $*"
2345         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" remove-trigger "$trigger_name" "$@" 1> /dev/null 2> /dev/null
2346         ret=$?
2347         if [[ $expected_to_fail -eq "1" ]]; then
2348                 test "$ret" -ne "0"
2349                 ok $? "Remove trigger $trigger_name failed as expected"
2350         else
2351                 ok $ret "Remove trigger $trigger_name"
2352         fi
2353 }
2354
2355 function lttng_add_trigger_ok()
2356 {
2357         lttng_add_trigger 0 "$@"
2358 }
2359
2360 function lttng_add_trigger_fail()
2361 {
2362         lttng_add_trigger 1 "$@"
2363 }
2364
2365 function lttng_remove_trigger_ok()
2366 {
2367         lttng_remove_trigger 0 "$@"
2368 }
2369
2370 function list_triggers_matches_ok ()
2371 {
2372         local tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2373         local tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2374
2375         local test_name="$1"
2376         local expected_stdout_file="$2"
2377
2378         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN list-triggers"
2379
2380         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" list-triggers > "${tmp_stdout}" 2> "${tmp_stderr}"
2381         ok $? "${test_name}: exit code is 0"
2382
2383         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2384         ok $? "${test_name}: expected stdout"
2385
2386         diff -u /dev/null "${tmp_stderr}"
2387         ok $? "${test_name}: expected stderr"
2388
2389         rm -f "${tmp_stdout}"
2390         rm -f "${tmp_stderr}"
2391 }
2392
2393 function list_triggers_matches_mi_ok ()
2394 {
2395         local tmp_stdout
2396         local tmp_stdout_raw
2397         local tmp_stderr
2398
2399         local test_name="$1"
2400         local expected_stdout_file="$2"
2401
2402         tmp_stdout_raw=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2403         tmp_stdout=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stdout.XXXXXX")
2404         tmp_stderr=$(mktemp --tmpdir -t "tmp.${FUNCNAME[0]}_stderr.XXXXXX")
2405
2406         diag "$TESTDIR/../src/bin/lttng/$LTTNG_BIN --mi xml list-triggers"
2407
2408         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" --mi=xml list-triggers > "${tmp_stdout_raw}" 2> "${tmp_stderr}"
2409         ok $? "${test_name}: exit code is 0"
2410
2411         # Pretty-fy xml before further test.
2412         $XML_PRETTY < "${tmp_stdout_raw}" > "${tmp_stdout}"
2413
2414         $MI_VALIDATE "${tmp_stdout}"
2415         ok $? "list-trigger mi is valid"
2416
2417         diff -u "${expected_stdout_file}" "${tmp_stdout}"
2418         ok $? "${test_name}: expected stdout"
2419
2420         diff -u /dev/null "${tmp_stderr}"
2421         ok $? "${test_name}: expected stderr"
2422
2423         rm -f "${tmp_stdout}"
2424         rm -f "${tmp_stdout_raw}"
2425         rm -f "${tmp_stderr}"
2426 }
2427
2428 function validate_path_pattern ()
2429 {
2430         local message=$1
2431         local pattern=$2
2432         # Base path is only used in error case and is used to list the content
2433         # of the base path.
2434         local base_path=$3
2435
2436
2437         [ -f $pattern ]
2438         ret=$?
2439         ok $ret "$message"
2440
2441         if [ "$ret" -ne "0" ]; then
2442                 diag "Path pattern expected: $pattern"
2443                 # List the tracepath for more info. We use find as a recursive
2444                 # directory lister.
2445                 diag "The base path content:"
2446                 find "$base_path" -print
2447         fi
2448 }
2449
2450 function validate_trace_path_ust_uid ()
2451 {
2452         local trace_path=$1
2453         local session_name=$2
2454         local uid=$UID
2455         local pattern="$trace_path/$session_name-$date_time_pattern/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2456
2457         validate_path_pattern "UST per-uid trace path is valid" "$pattern" "$trace_path"
2458 }
2459
2460 function validate_trace_path_ust_uid_network ()
2461 {
2462         local trace_path=$1
2463         local session_name=$2
2464         local base_path=$3
2465         local uid=$UID
2466         local hostname=$HOSTNAME
2467         local pattern
2468         local ret
2469
2470         # If the session was given a network base path (e.g
2471         # 127.0.0.1/my/custom/path on creation, there is no session name
2472         # component to the path on the relayd side. Caller can simply not pass a
2473         # session name for this scenario.
2474         if [ -n "$session_name" ]; then
2475                 session_name="$session_name-$date_time_pattern"
2476                 if [ -n "$base_path" ]; then
2477                         fail "Session name and base path are mutually exclusive"
2478                         return
2479                 fi
2480         fi
2481
2482         pattern="$trace_path/$hostname/$base_path/$session_name/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2483
2484         validate_path_pattern "UST per-uid network trace path is valid" "$pattern" "$trace_path"
2485 }
2486
2487 function validate_trace_path_ust_uid_snapshot_network ()
2488 {
2489         local trace_path=$1
2490         local session_name=$2
2491         local snapshot_name=$3
2492         local snapshot_number=$4
2493         local base_path=$5
2494         local hostname=$HOSTNAME
2495         local uid=$UID
2496         local pattern
2497         local ret
2498
2499         # If the session/output was given a network base path (e.g
2500         # 127.0.0.1/my/custom/path on creation, there is no session name
2501         # component to the path on the relayd side. Caller can simply not pass a
2502         # session name for this scenario.
2503         if [ -n "$session_name" ]; then
2504                 session_name="$session_name-$date_time_pattern"
2505                 if [ -n "$base_path" ]; then
2506                         fail "Session name and base path are mutually exclusive"
2507                         return
2508                 fi
2509         fi
2510
2511         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2512
2513         validate_path_pattern "UST per-uid network snapshot trace path is valid" "$pattern" "$trace_path"
2514 }
2515
2516 function validate_trace_path_ust_uid_snapshot ()
2517 {
2518         local trace_path=$1
2519         local session_name=$2
2520         local snapshot_name=$3
2521         local snapshot_number=$4
2522         local base_path=$5
2523         local uid=$UID
2524         local pattern
2525         local ret
2526
2527         # If the session/output was given a network base path (e.g
2528         # 127.0.0.1/my/custom/path) on creation, there is no session name
2529         # component to the path on the relayd side. Caller can simply not pass a
2530         # session name for this scenario.
2531         if [ -n "$session_name" ]; then
2532                 session_name="$session_name-$date_time_pattern"
2533                 if [ -n "$base_path" ]; then
2534                         fail "Session name and base path are mutually exclusive"
2535                         return
2536                 fi
2537         fi
2538
2539         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/ust/uid/$uid/${system_long_bit_size}-bit/metadata"
2540
2541         validate_path_pattern "UST per-uid snapshot trace path is valid" "$pattern" "$trace_path"
2542 }
2543
2544 function validate_trace_path_ust_pid ()
2545 {
2546         local trace_path=$1
2547         local session_name=$2
2548         local app_string=$3
2549         local pid=$4
2550         local pattern
2551         local ret
2552
2553         # If the session was given a trace path on creation, there is no session
2554         # name component to the path. Caller can simply not pass a session name
2555         # for this scenario.
2556         if [ -n "$session_name" ]; then
2557                 session_name="$session_name-$date_time_pattern"
2558         fi
2559
2560         pattern="$trace_path/$session_name/ust/pid/$pid/$app_string-*-$date_time_pattern/metadata"
2561
2562         validate_path_pattern "UST per-pid trace path is valid" "$pattern" "$trace_path"
2563 }
2564
2565 function validate_trace_path_kernel ()
2566 {
2567         local trace_path=$1
2568         local session_name=$2
2569         local pattern
2570
2571         # If the session was given a trace path on creation, there is no session
2572         # name component to the path. Caller can simply not pass a session name
2573         # for this scenario.
2574         if [ -n "$session_name" ]; then
2575                 session_name="$session_name-$date_time_pattern"
2576         fi
2577
2578         pattern="$trace_path/$session_name/kernel/metadata"
2579
2580         validate_path_pattern "Kernel trace path is valid" "$pattern" "$trace_path"
2581 }
2582
2583 function validate_trace_path_kernel_network ()
2584 {
2585         local trace_path=$1
2586         local session_name=$2
2587         local hostname=$HOSTNAME
2588         local pattern="$trace_path/$hostname/$session_name-$date_time_pattern/kernel/metadata"
2589
2590         validate_path_pattern "Kernel network trace path is valid" "$pattern" "$trace_path"
2591 }
2592
2593 function validate_trace_path_kernel_snapshot ()
2594 {
2595         local trace_path=$1
2596         local session_name=$2
2597         local snapshot_name=$3
2598         local snapshot_number=$4
2599         local base_path=$5
2600         local pattern
2601         local ret
2602
2603         # If the session/output was given a network base path (e.g
2604         # 127.0.0.1/my/custom/path on creation, there is no session name
2605         # component to the path on the relayd side. Caller can simply not pass a
2606         # session name for this scenario.
2607         if [ -n "$session_name" ]; then
2608                 session_name="$session_name-$date_time_pattern"
2609                 if [ -n "$base_path" ]; then
2610                         fail "Session name and base path are mutually exclusive"
2611                         return
2612                 fi
2613         fi
2614
2615         pattern="$trace_path/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2616
2617         validate_path_pattern "Kernel snapshot trace path is valid" "$pattern" "$trace_path"
2618 }
2619
2620 function validate_trace_path_kernel_snapshot_network ()
2621 {
2622         local trace_path=$1
2623         local session_name=$2
2624         local snapshot_name=$3
2625         local snapshot_number=$4
2626         local base_path=$5
2627         local hostname=$HOSTNAME
2628         local pattern
2629         local ret
2630
2631         # If the session/output was given a network base path (e.g
2632         # 127.0.0.1/my/custom/path on creation, there is no session name
2633         # component to the path on the relayd side. Caller can simply not pass a
2634         # session name for this scenario.
2635         if [ -n "$session_name" ]; then
2636                 session_name="$session_name-$date_time_pattern"
2637                 if [ -n "$base_path" ]; then
2638                         fail "Session name and base path are mutually exclusive"
2639                         return
2640                 fi
2641         fi
2642
2643         pattern="$trace_path/$hostname/$base_path/$session_name/$snapshot_name-$date_time_pattern-$snapshot_number/kernel/metadata"
2644
2645         validate_path_pattern "Kernel network snapshot trace path is valid" "$pattern" "$trace_path"
2646 }