tests/utils/utils.sh

   1 # Copyright (C) - 2012 David Goulet <dgoulet@efficios.com>
   2 #
   3 # This library is free software; you can redistribute it and/or modify it under
   4 # the terms of the GNU Lesser General Public License as published by the Free
   5 # Software Foundation; version 2.1 of the License.
   6 #
   7 # This library is distributed in the hope that it will be useful, but WITHOUT
   8 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   9 # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
  10 # details.
  11 #
  12 # You should have received a copy of the GNU Lesser General Public License
  13 # along with this library; if not, write to the Free Software Foundation, Inc.,
  14 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
  15
  16 SESSIOND_BIN="lttng-sessiond"
  17 SESSIOND_MATCH=".*lttng-sess.*"
  18 SESSIOND_PIDS=""
  19 RUNAS_BIN="lttng-runas"
  20 RUNAS_MATCH=".*lttng-runas.*"
  21 CONSUMERD_BIN="lttng-consumerd"
  22 CONSUMERD_MATCH=".*lttng-consumerd.*"
  23 RELAYD_BIN="lttng-relayd"
  24 RELAYD_MATCH=".*lttng-relayd.*"
  25 RELAYD_PIDS=""
  26 LTTNG_BIN="lttng"
  27 BABELTRACE_BIN="babeltrace"
  28 OUTPUT_DEST=/dev/null
  29 ERROR_OUTPUT_DEST=/dev/null
  30
  31 # Minimal kernel version supported for session daemon tests
  32 KERNEL_MAJOR_VERSION=2
  33 KERNEL_MINOR_VERSION=6
  34 KERNEL_PATCHLEVEL_VERSION=27
  35
  36 # We set the default UST register timeout and network and app socket timeout to
  37 # "wait forever", so that basic tests don't have to worry about hitting
  38 # timeouts on busy systems. Specialized tests should test those corner-cases.
  39 export LTTNG_UST_REGISTER_TIMEOUT=-1
  40 export LTTNG_NETWORK_SOCKET_TIMEOUT=-1
  41 export LTTNG_APP_SOCKET_TIMEOUT=-1
  42
  43 # We set the default lttng-sessiond path to /bin/true to prevent the spawning
  44 # of a daemonized sessiond. This is necessary since 'lttng create' will spawn
  45 # its own sessiond if none is running. It also ensures that 'lttng create'
  46 # fails when no sessiond is running.
  47 export LTTNG_SESSIOND_PATH="/bin/true"
  48
  49 source $TESTDIR/utils/tap/tap.sh
  50
  51 function full_cleanup ()
  52 {
  53         if [ -n "${SESSIOND_PIDS}" ] || [ -n "${RELAYD_PIDS}" ]; then
  54                 kill -9 ${SESSIOND_PIDS} ${RELAYD_PIDS} > /dev/null 2>&1
  55         fi
  56
  57         # Disable trap for SIGTERM since the following kill to the
  58         # pidgroup will be SIGTERM. Otherwise it loops.
  59         # The '-' before the pid number ($$) indicates 'kill' to signal the
  60         # whole process group.
  61         trap - SIGTERM && kill -- -$$
  62 }
  63
  64
  65 trap full_cleanup SIGINT SIGTERM
  66
  67 function print_ok ()
  68 {
  69         # Check if we are a terminal
  70         if [ -t 1 ]; then
  71                 echo -e "\e[1;32mOK\e[0m"
  72         else
  73                 echo -e "OK"
  74         fi
  75 }
  76
  77 function print_fail ()
  78 {
  79         # Check if we are a terminal
  80         if [ -t 1 ]; then
  81                 echo -e "\e[1;31mFAIL\e[0m"
  82         else
  83                 echo -e "FAIL"
  84         fi
  85 }
  86
  87 function print_test_banner ()
  88 {
  89         local desc="$1"
  90         diag "$desc"
  91 }
  92
  93 function validate_kernel_version ()
  94 {
  95         local kern_version=($(uname -r | awk -F. '{ printf("%d.%d.%d\n",$1,$2,$3); }' | tr '.' '\n'))
  96         if [ ${kern_version[0]} -gt $KERNEL_MAJOR_VERSION ]; then
  97                 return 0
  98         fi
  99         if [ ${kern_version[1]} -gt $KERNEL_MINOR_VERSION ]; then
 100                 return 0
 101         fi
 102         if [ ${kern_version[2]} -ge $KERNEL_PATCHLEVEL_VERSION ]; then
 103                 return 0
 104         fi
 105         return 1
 106 }
 107
 108 # Generate a random string
 109 #  $1 = number of characters; defaults to 16
 110 #  $2 = include special characters; 1 = yes, 0 = no; defaults to yes
 111 function randstring()
 112 {
 113         [ "$2" == "0" ] && CHAR="[:alnum:]" || CHAR="[:graph:]"
 114         cat /dev/urandom 2>/dev/null | tr -cd "$CHAR" 2>/dev/null | head -c ${1:-16} 2>/dev/null
 115         echo
 116 }
 117
 118 # Return the number of _configured_ CPUs.
 119 function conf_proc_count()
 120 {
 121         getconf _NPROCESSORS_CONF
 122         if [ $? -ne 0 ]; then
 123                 diag "Failed to get the number of configured CPUs"
 124         fi
 125         echo
 126 }
 127
 128 # Check if base lttng-modules are present.
 129 # Bail out on failure
 130 function validate_lttng_modules_present ()
 131 {
 132         modprobe -n lttng-tracer 2>/dev/null
 133         if [ $? -ne 0  ]; then
 134                 BAIL_OUT "LTTng modules not detected."
 135         fi
 136 }
 137
 138 function enable_kernel_lttng_event
 139 {
 140         local withtap="$1"
 141         local expected_to_fail="$2"
 142         local sess_name="$3"
 143         local event_name="$4"
 144         local channel_name="$5"
 145
 146         if [ -z "$event_name" ]; then
 147                 # Enable all event if no event name specified
 148                 event_name="-a"
 149         fi
 150
 151         if [ -z "$channel_name" ]; then
 152                 # default channel if none specified
 153                 chan=""
 154         else
 155                 chan="-c $channel_name"
 156         fi
 157
 158         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 159         ret=$?
 160         if [[ $expected_to_fail -eq "1" ]]; then
 161                 test $ret -ne "0"
 162                 ret=$?
 163                 if [ $withtap -eq "1" ]; then
 164                         ok $ret "Enable kernel event $event_name for session $session_name on channel $channel_name failed as expected"
 165                 fi
 166         else
 167                 if [ $withtap -eq "1" ]; then
 168                         ok $ret "Enable kernel event $event_name for session $sess_name"
 169                 fi
 170         fi
 171 }
 172
 173 function enable_kernel_lttng_event_ok ()
 174 {
 175         enable_kernel_lttng_event 1 0 "$@"
 176 }
 177
 178 function enable_kernel_lttng_event_fail ()
 179 {
 180         enable_kernel_lttng_event 1 1 "$@"
 181 }
 182
 183 function enable_kernel_lttng_event_notap ()
 184 {
 185         enable_kernel_lttng_event 0 0 "$@"
 186 }
 187
 188 # Old interface
 189 function lttng_enable_kernel_event
 190 {
 191         enable_kernel_lttng_event_ok "$@"
 192 }
 193
 194 function lttng_enable_kernel_syscall()
 195 {
 196         local expected_to_fail=$1
 197         local sess_name=$2
 198         local syscall_name=$3
 199         local channel_name=$4
 200
 201         if [ -z $syscall_name ]; then
 202                 # Enable all event if no syscall name specified
 203                 syscall_name="-a"
 204         fi
 205
 206         if [ -z $channel_name ]; then
 207                 # default channel if none specified
 208                 chan=""
 209         else
 210                 chan="-c $channel_name"
 211         fi
 212
 213         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 214         ret=$?
 215         if [[ $expected_to_fail -eq "1" ]]; then
 216                 test $ret -ne "0"
 217                 ok $? "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name fail as expected"
 218         else
 219                 ok $ret "Enable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 220         fi
 221 }
 222
 223 function lttng_enable_kernel_syscall_ok()
 224 {
 225         lttng_enable_kernel_syscall 0 "$@"
 226 }
 227
 228 function lttng_enable_kernel_syscall_fail()
 229 {
 230         lttng_enable_kernel_syscall 1 "$@"
 231 }
 232
 233 function lttng_disable_kernel_syscall()
 234 {
 235         local expected_to_fail=$1
 236         local sess_name=$2
 237         local syscall_name=$3
 238         local channel_name=$4
 239
 240         if [ -z $syscall_name ]; then
 241                 # Enable all event if no syscall name specified
 242                 syscall_name="-a"
 243         fi
 244
 245         if [ -z $channel_name ]; then
 246                 # default channel if none specified
 247                 chan=""
 248         else
 249                 chan="-c $channel_name"
 250         fi
 251
 252         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event --syscall "$syscall_name" $chan -s $sess_name -k 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 253
 254         ret=$?
 255         if [[ $expected_to_fail -eq "1" ]]; then
 256                 test $ret -ne "0"
 257                 ok $? "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name failed as expected"
 258         else
 259                 ok $ret "Disable kernel syscall $syscall_name for session $sess_name on channel $channel_name"
 260         fi
 261 }
 262
 263 function lttng_disable_kernel_syscall_ok()
 264 {
 265         lttng_disable_kernel_syscall 0 "$@"
 266 }
 267
 268 function lttng_disable_kernel_syscall_fail()
 269 {
 270         lttng_disable_kernel_syscall 1 "$@"
 271 }
 272
 273 function lttng_enable_kernel_channel()
 274 {
 275         local withtap=$1
 276         local expected_to_fail=$2
 277         local sess_name=$3
 278         local channel_name=$4
 279         local opt=$5
 280
 281         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -k $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 282         ret=$?
 283         if [[ $expected_to_fail -eq "1" ]]; then
 284                 test "$ret" -ne "0"
 285                 ret=$?
 286                 if [ $withtap -eq "1" ]; then
 287                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 288                 fi
 289         else
 290                 if [ $withtap -eq "1" ]; then
 291                         ok $ret "Enable channel $channel_name for session $sess_name"
 292                 fi
 293         fi
 294 }
 295
 296 function lttng_enable_kernel_channel_ok()
 297 {
 298         lttng_enable_kernel_channel 1 0 "$@"
 299 }
 300
 301 function lttng_enable_kernel_channel_fail()
 302 {
 303         lttng_enable_kernel_channel 1 1 "$@"
 304 }
 305
 306 function lttng_enable_kernel_channel_notap()
 307 {
 308         lttng_enable_kernel_channel 0 0 "$@"
 309 }
 310
 311 function enable_kernel_lttng_channel_ok()
 312 {
 313         lttng_enable_kernel_channel 1 0 "$@"
 314 }
 315
 316 function lttng_disable_kernel_channel()
 317 {
 318         local expected_to_fail=$1
 319         local sess_name=$2
 320         local channel_name=$3
 321
 322         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -k $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 323         ret=$?
 324         if [[ $expected_to_fail -eq "1" ]]; then
 325                 test "$ret" -ne "0"
 326                 ok $? "Disable channel $channel_name for session $sess_name failed as expected"
 327         else
 328                 ok $ret "Disable channel $channel_name for session $sess_name"
 329         fi
 330 }
 331
 332 function lttng_disable_kernel_channel_ok()
 333 {
 334         lttng_disable_kernel_channel 0 "$@"
 335 }
 336
 337 function lttng_disable_kernel_channel_fail()
 338 {
 339         lttng_disable_kernel_channel 1 "$@"
 340 }
 341
 342 function start_lttng_relayd_opt()
 343 {
 344         local withtap=$1
 345         local opt=$2
 346
 347         DIR=$(readlink -f $TESTDIR)
 348
 349         if [ -z $(pgrep $RELAYD_MATCH) ]; then
 350                 $DIR/../src/bin/lttng-relayd/$RELAYD_BIN -b $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 351                 #$DIR/../src/bin/lttng-relayd/$RELAYD_BIN $opt -vvv >>/tmp/relayd.log 2>&1 &
 352                 if [ $? -eq 1 ]; then
 353                         if [ $withtap -eq "1" ]; then
 354                                 fail "Start lttng-relayd (opt: $opt)"
 355                         fi
 356                         return 1
 357                 else
 358                         if [ $withtap -eq "1" ]; then
 359                                 pass "Start lttng-relayd (opt: $opt)"
 360                         fi
 361                 fi
 362         else
 363                 pass "Start lttng-relayd (opt: $opt)"
 364         fi
 365
 366         RELAYD_PIDS=$(pgrep $RELAYD_MATCH)
 367 }
 368
 369 function start_lttng_relayd()
 370 {
 371         start_lttng_relayd_opt 1 "$@"
 372 }
 373
 374 function start_lttng_relayd_notap()
 375 {
 376         start_lttng_relayd_opt 0 "$@"
 377 }
 378
 379 function stop_lttng_relayd_opt()
 380 {
 381         local withtap=$1
 382
 383         if [ $withtap -eq "1" ]; then
 384                 diag "Killing lttng-relayd (pid: $RELAYD_PIDS)"
 385         fi
 386         kill $RELAYD_PIDS 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 387         retval=$?
 388
 389         if [ $? -eq 1 ]; then
 390                 if [ $withtap -eq "1" ]; then
 391                         fail "Kill relay daemon"
 392                 fi
 393                 return 1
 394         else
 395                 out=1
 396                 while [ -n "$out" ]; do
 397                         out=$(pgrep $RELAYD_MATCH)
 398                         sleep 0.5
 399                 done
 400                 if [ $withtap -eq "1" ]; then
 401                         pass "Kill relay daemon"
 402                 fi
 403         fi
 404         RELAYD_PIDS=""
 405         return $retval
 406 }
 407
 408 function stop_lttng_relayd()
 409 {
 410         stop_lttng_relayd_opt 1 "$@"
 411 }
 412
 413 function stop_lttng_relayd_notap()
 414 {
 415         stop_lttng_relayd_opt 0 "$@"
 416 }
 417
 418 #First arg: show tap output
 419 #Second argument: load path for automatic loading
 420 function start_lttng_sessiond_opt()
 421 {
 422         local withtap=$1
 423         local load_path=$2
 424
 425         local env_vars=""
 426         local consumerd=""
 427         local long_bit_value=$(getconf LONG_BIT)
 428
 429         if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 430                 # Env variable requested no session daemon
 431                 return
 432         fi
 433
 434         DIR=$(readlink -f $TESTDIR)
 435
 436         # Get long_bit value for 32/64 consumerd
 437         case "$long_bit_value" in
 438                 32)
 439                         consumerd="--consumerd32-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 440                         ;;
 441                 64)
 442                         consumerd="--consumerd64-path=$DIR/../src/bin/lttng-consumerd/lttng-consumerd"
 443                         ;;
 444                 *)
 445                         return
 446                         ;;
 447         esac
 448
 449         # Check for env. variable. Allow the use of LD_PRELOAD etc.
 450         if [[ "x${LTTNG_SESSIOND_ENV_VARS}" != "x" ]]; then
 451                 env_vars=${LTTNG_SESSIOND_ENV_VARS}
 452         fi
 453
 454         validate_kernel_version
 455         if [ $? -ne 0 ]; then
 456             fail "Start session daemon"
 457             BAIL_OUT "*** Kernel too old for session daemon tests ***"
 458         fi
 459
 460         : ${LTTNG_SESSION_CONFIG_XSD_PATH=${DIR}/../src/common/config/}
 461         export LTTNG_SESSION_CONFIG_XSD_PATH
 462
 463         if [ -z $(pgrep ${SESSIOND_MATCH}) ]; then
 464                 # Have a load path ?
 465                 if [ -n "$load_path" ]; then
 466                         env $env_vars $DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --load "$load_path" --background $consumerd
 467                 else
 468                         env $env_vars $DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background $consumerd
 469                 fi
 470                 #$DIR/../src/bin/lttng-sessiond/$SESSIOND_BIN --background --consumerd32-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --consumerd64-path="$DIR/../src/bin/lttng-consumerd/lttng-consumerd" --verbose-consumer >>/tmp/sessiond.log 2>&1
 471                 status=$?
 472                 if [ $withtap -eq "1" ]; then
 473                         ok $status "Start session daemon"
 474                 fi
 475         fi
 476         SESSIOND_PIDS=$(pgrep $SESSIOND_MATCH)
 477 }
 478
 479 function start_lttng_sessiond()
 480 {
 481         start_lttng_sessiond_opt 1 "$@"
 482 }
 483
 484 function start_lttng_sessiond_notap()
 485 {
 486         start_lttng_sessiond_opt 0 "$@"
 487 }
 488
 489 function stop_lttng_sessiond_opt()
 490 {
 491         local withtap=$1
 492         local signal=$2
 493         local kill_opt=""
 494
 495         if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 496                 # Env variable requested no session daemon
 497                 return
 498         fi
 499
 500         local pids="${SESSIOND_PIDS} $(pgrep $RUNAS_MATCH)"
 501
 502         if [ -n "$2" ]; then
 503                 kill_opt="$kill_opt -s $signal"
 504         fi
 505         if [ $withtap -eq "1" ]; then
 506                 diag "Killing $SESSIOND_BIN and lt-$SESSIOND_BIN pids: $(echo $pids | tr '\n' ' ')"
 507         fi
 508         kill $kill_opt $pids 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 509
 510         if [ $? -eq 1 ]; then
 511                 if [ $withtap -eq "1" ]; then
 512                         fail "Kill sessions daemon"
 513                 fi
 514         else
 515                 out=1
 516                 while [ -n "$out" ]; do
 517                         out=$(pgrep ${SESSIOND_MATCH})
 518                         sleep 0.5
 519                 done
 520                 out=1
 521                 while [ -n "$out" ]; do
 522                         out=$(pgrep $CONSUMERD_MATCH)
 523                         sleep 0.5
 524                 done
 525
 526                 SESSIOND_PIDS=""
 527                 if [ $withtap -eq "1" ]; then
 528                         pass "Kill session daemon"
 529                 fi
 530         fi
 531 }
 532
 533 function stop_lttng_sessiond()
 534 {
 535         stop_lttng_sessiond_opt 1 "$@"
 536 }
 537
 538 function stop_lttng_sessiond_notap()
 539 {
 540         stop_lttng_sessiond_opt 0 "$@"
 541 }
 542
 543 function sigstop_lttng_sessiond_opt()
 544 {
 545         local withtap=$1
 546         local signal=SIGSTOP
 547         local kill_opt=""
 548
 549         if [ -n $TEST_NO_SESSIOND ] && [ "$TEST_NO_SESSIOND" == "1" ]; then
 550                 # Env variable requested no session daemon
 551                 return
 552         fi
 553
 554         PID_SESSIOND="$(pgrep ${SESSIOND_MATCH}) $(pgrep $RUNAS_MATCH)"
 555
 556         kill_opt="$kill_opt -s $signal"
 557
 558         if [ $withtap -eq "1" ]; then
 559                 diag "Sending SIGSTOP to lt-$SESSIOND_BIN and $SESSIOND_BIN pids: $(echo $PID_SESSIOND | tr '\n' ' ')"
 560         fi
 561         kill $kill_opt $PID_SESSIOND 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 562
 563         if [ $? -eq 1 ]; then
 564                 if [ $withtap -eq "1" ]; then
 565                         fail "Sending SIGSTOP to session daemon"
 566                 fi
 567         else
 568                 out=1
 569                 while [ $out -ne 0 ]; do
 570                         pid=$(pgrep $SESSIOND_MATCH)
 571
 572                         # Wait until state becomes stopped for session
 573                         # daemon(s).
 574                         out=0
 575                         for sessiond_pid in $pid; do
 576                                 state=$(ps -p $sessiond_pid -o state= )
 577                                 if [[ -n "$state" && "$state" != "T" ]]; then
 578                                         out=1
 579                                 fi
 580                         done
 581                         sleep 0.5
 582                 done
 583                 if [ $withtap -eq "1" ]; then
 584                         pass "Sending SIGSTOP to session daemon"
 585                 fi
 586         fi
 587 }
 588
 589 function sigstop_lttng_sessiond()
 590 {
 591         sigstop_lttng_sessiond_opt 1 "$@"
 592 }
 593
 594 function sigstop_lttng_sessiond_notap()
 595 {
 596         sigstop_lttng_sessiond_opt 0 "$@"
 597 }
 598
 599 function stop_lttng_consumerd_opt()
 600 {
 601         local withtap=$1
 602         local signal=$2
 603         local kill_opt=""
 604
 605         PID_CONSUMERD=$(pgrep $CONSUMERD_MATCH)
 606
 607         if [ -n "$2" ]; then
 608                 kill_opt="$kill_opt -s $signal"
 609         fi
 610
 611         if [ $withtap -eq "1" ]; then
 612                 diag "Killing $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
 613         fi
 614         kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 615         retval=$?
 616         set +x
 617
 618         if [ $? -eq 1 ]; then
 619                 if [ $withtap -eq "1" ]; then
 620                         fail "Kill consumer daemon"
 621                 fi
 622                 return 1
 623         else
 624                 out=1
 625                 while [ $out -ne 0 ]; do
 626                         pid=$(pgrep $CONSUMERD_MATCH)
 627
 628                         # If consumerds are still present check their status.
 629                         # A zombie status qualifies the consumerd as *killed*
 630                         out=0
 631                         for consumer_pid in $pid; do
 632                                 state=$(ps -p $consumer_pid -o state= )
 633                                 if [[ -n "$state" && "$state" != "Z" ]]; then
 634                                         out=1
 635                                 fi
 636                         done
 637                         sleep 0.5
 638                 done
 639                 if [ $withtap -eq "1" ]; then
 640                         pass "Kill consumer daemon"
 641                 fi
 642         fi
 643         return $retval
 644 }
 645
 646 function stop_lttng_consumerd()
 647 {
 648         stop_lttng_consumerd_opt 1 "$@"
 649 }
 650
 651 function stop_lttng_consumerd_notap()
 652 {
 653         stop_lttng_consumerd_opt 0 "$@"
 654 }
 655
 656 function sigstop_lttng_consumerd_opt()
 657 {
 658         local withtap=$1
 659         local signal=SIGSTOP
 660         local kill_opt=""
 661
 662         PID_CONSUMERD=$(pgrep $CONSUMERD_MATCH)
 663
 664         kill_opt="$kill_opt -s $signal"
 665
 666         if [ $withtap -eq "1" ]; then
 667                 diag "Sending SIGSTOP to $CONSUMERD_BIN pids: $(echo $PID_CONSUMERD | tr '\n' ' ')"
 668         fi
 669         kill $kill_opt $PID_CONSUMERD 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 670         retval=$?
 671         set +x
 672
 673         if [ $? -eq 1 ]; then
 674                 if [ $withtap -eq "1" ]; then
 675                         fail "Sending SIGSTOP to consumer daemon"
 676                 fi
 677                 return 1
 678         else
 679                 out=1
 680                 while [ $out -ne 0 ]; do
 681                         pid=$(pgrep $CONSUMERD_MATCH)
 682
 683                         # Wait until state becomes stopped for all
 684                         # consumers.
 685                         out=0
 686                         for consumer_pid in $pid; do
 687                                 state=$(ps -p $consumer_pid -o state= )
 688                                 if [[ -n "$state" && "$state" != "T" ]]; then
 689                                         out=1
 690                                 fi
 691                         done
 692                         sleep 0.5
 693                 done
 694                 if [ $withtap -eq "1" ]; then
 695                         pass "Sending SIGSTOP to consumer daemon"
 696                 fi
 697         fi
 698         return $retval
 699 }
 700
 701 function sigstop_lttng_consumerd()
 702 {
 703         sigstop_lttng_consumerd_opt 1 "$@"
 704 }
 705
 706 function sigstop_lttng_consumerd_notap()
 707 {
 708         sigstop_lttng_consumerd_opt 0 "$@"
 709 }
 710
 711 function list_lttng_with_opts ()
 712 {
 713         local opts=$1
 714         $TESTDIR/../src/bin/lttng/$LTTNG_BIN list $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 715         ok $? "Lttng-tool list command with option $opts"
 716 }
 717
 718 function create_lttng_session_no_output ()
 719 {
 720         local sess_name=$1
 721
 722         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name --no-output 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 723         ok $? "Create session $sess_name in no-output mode"
 724 }
 725
 726 function create_lttng_session_uri () {
 727         local sess_name=$1
 728         local uri=$2
 729         local opts="${@:3}"
 730
 731         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -U $uri $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 732         ok $? "Create session $sess_name with uri:$uri and opts: $opts"
 733 }
 734
 735 function create_lttng_session ()
 736 {
 737         local withtap=$1
 738         local expected_to_fail=$2
 739         local sess_name=$3
 740         local trace_path=$4
 741         local opt=$5
 742
 743         $TESTDIR/../src/bin/lttng/$LTTNG_BIN create $sess_name -o $trace_path $opt > $OUTPUT_DEST
 744         ret=$?
 745         if [ $expected_to_fail -eq "1" ]; then
 746                 test "$ret" -ne "0"
 747                 ret=$?
 748                 if [ $withtap -eq "1" ]; then
 749                         ok $ret "Create session $sess_name in $trace_path failed as expected"
 750                 fi
 751         else
 752                 if [ $withtap -eq "1" ]; then
 753                         ok $ret "Create session $sess_name in $trace_path"
 754                 fi
 755         fi
 756         return $ret
 757 }
 758
 759 function create_lttng_session_ok ()
 760 {
 761         create_lttng_session 1 0 "$@"
 762 }
 763
 764 function create_lttng_session_fail ()
 765 {
 766         create_lttng_session 1 1 "$@"
 767 }
 768
 769 function create_lttng_session_notap ()
 770 {
 771         create_lttng_session 0 0 "$@"
 772 }
 773
 774
 775 function enable_ust_lttng_channel ()
 776 {
 777         local withtap=$1
 778         local expected_to_fail=$2
 779         local sess_name=$3
 780         local channel_name=$4
 781         local opt=$5
 782
 783         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -u $channel_name -s $sess_name $opt 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 784         ret=$?
 785         if [[ $expected_to_fail -eq "1" ]]; then
 786                 test "$ret" -ne "0"
 787                 ret=$?
 788                 if [ $withtap -eq "1" ]; then
 789                         ok $ret "Enable channel $channel_name for session $sess_name failed as expected"
 790                 fi
 791         else
 792                 if [ $withtap -eq "1" ]; then
 793                         ok $ret "Enable channel $channel_name for session $sess_name"
 794                 fi
 795         fi
 796         return $ret
 797 }
 798
 799 function enable_ust_lttng_channel_ok ()
 800 {
 801         enable_ust_lttng_channel 1 0 "$@"
 802 }
 803
 804 function enable_ust_lttng_channel_fail ()
 805 {
 806         enable_ust_lttng_channel 1 1 "$@"
 807 }
 808
 809 function enable_ust_lttng_channel_notap ()
 810 {
 811         enable_ust_lttng_channel 0 0 "$@"
 812 }
 813
 814 function disable_ust_lttng_channel()
 815 {
 816         local sess_name=$1
 817         local channel_name=$2
 818
 819         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-channel -u $channel_name -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 820         ok $? "Disable channel $channel_name for session $sess_name"
 821 }
 822
 823 function enable_lttng_mmap_overwrite_kernel_channel()
 824 {
 825         local sess_name=$1
 826         local channel_name=$2
 827
 828         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 829         ok $? "Enable channel $channel_name for session $sess_name"
 830 }
 831
 832 function enable_lttng_mmap_discard_small_kernel_channel()
 833 {
 834         local sess_name=$1
 835         local channel_name=$2
 836
 837         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --discard --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 838         ok $? "Enable small discard channel $channel_name for session $sess_name"
 839 }
 840
 841 function enable_lttng_mmap_overwrite_small_kernel_channel()
 842 {
 843         local sess_name=$1
 844         local channel_name=$2
 845
 846         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -k --output mmap --overwrite --subbuf-size=$(getconf PAGE_SIZE) --num-subbuf=2 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 847         ok $? "Enable small discard channel $channel_name for session $sess_name"
 848 }
 849
 850 function enable_lttng_mmap_overwrite_ust_channel()
 851 {
 852         local sess_name=$1
 853         local channel_name=$2
 854
 855         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-channel -s $sess_name $channel_name -u --output mmap --overwrite 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 856         ok $? "Enable channel $channel_name for session $sess_name"
 857 }
 858
 859 function enable_ust_lttng_event ()
 860 {
 861         local withtap=$1
 862         local expected_to_fail=$2
 863         local sess_name=$3
 864         local event_name="$4"
 865         local channel_name=$5
 866
 867         if [ -z $channel_name ]; then
 868                 # default channel if none specified
 869                 chan=""
 870         else
 871                 chan="-c $channel_name"
 872         fi
 873
 874         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 875         ret=$?
 876         if [[ $expected_to_fail -eq "1" ]]; then
 877                 test $ret -ne "0"
 878                 ret=$?
 879                 if [[ $withtap -eq "1" ]]; then
 880                         ok $ret "Enable ust event $event_name for session $session_name failed as expected"
 881                 fi
 882         else
 883                 if [[ $withtap -eq "1" ]]; then
 884                         ok $ret "Enable ust event $event_name for session $sess_name"
 885                 fi
 886         fi
 887         return $ret
 888 }
 889
 890 function enable_ust_lttng_event_ok ()
 891 {
 892         enable_ust_lttng_event 1 0 "$@"
 893 }
 894
 895 function enable_ust_lttng_event_fail ()
 896 {
 897         enable_ust_lttng_event 1 1 "$@"
 898 }
 899
 900 function enable_ust_lttng_event_notap ()
 901 {
 902         enable_ust_lttng_event 0 0 "$@"
 903 }
 904
 905 function enable_jul_lttng_event()
 906 {
 907         sess_name=$1
 908         event_name="$2"
 909         channel_name=$3
 910
 911         if [ -z $channel_name ]; then
 912                 # default channel if none specified
 913                 chan=""
 914         else
 915                 chan="-c $channel_name"
 916         fi
 917
 918         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 919         ok $? "Enable JUL event $event_name for session $sess_name"
 920 }
 921
 922 function enable_jul_lttng_event_loglevel()
 923 {
 924         local sess_name=$1
 925         local event_name="$2"
 926         local loglevel=$3
 927         local channel_name=$4
 928
 929         if [ -z $channel_name ]; then
 930                 # default channel if none specified
 931                 chan=""
 932         else
 933                 chan="-c $channel_name"
 934         fi
 935
 936         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -j 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 937         ok $? "Enable JUL event $event_name for session $sess_name with loglevel $loglevel"
 938 }
 939
 940 function enable_log4j_lttng_event()
 941 {
 942         sess_name=$1
 943         event_name="$2"
 944         channel_name=$3
 945
 946         if [ -z $channel_name ]; then
 947                 # default channel if none specified
 948                 chan=""
 949         else
 950                 chan="-c $channel_name"
 951         fi
 952
 953         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 954         ok $? "Enable LOG4J event $event_name for session $sess_name"
 955 }
 956
 957 function enable_log4j_lttng_event_loglevel()
 958 {
 959         local sess_name=$1
 960         local event_name="$2"
 961         local loglevel=$3
 962         local channel_name=$4
 963
 964         if [ -z $channel_name ]; then
 965                 # default channel if none specified
 966                 chan=""
 967         else
 968                 chan="-c $channel_name"
 969         fi
 970
 971         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -l 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 972         ok $? "Enable LOG4J event $event_name for session $sess_name with loglevel $loglevel"
 973 }
 974
 975 function enable_python_lttng_event()
 976 {
 977         sess_name=$1
 978         event_name="$2"
 979         channel_name=$3
 980
 981         if [ -z $channel_name ]; then
 982                 # default channel if none specified
 983                 chan=""
 984         else
 985                 chan="-c $channel_name"
 986         fi
 987
 988         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
 989         ok $? "Enable Python event $event_name for session $sess_name"
 990 }
 991
 992 function enable_python_lttng_event_loglevel()
 993 {
 994         local sess_name=$1
 995         local event_name="$2"
 996         local loglevel=$3
 997         local channel_name=$4
 998
 999         if [ -z $channel_name ]; then
1000                 # default channel if none specified
1001                 chan=""
1002         else
1003                 chan="-c $channel_name"
1004         fi
1005
1006         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event --loglevel $loglevel "$event_name" $chan -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1007         ok $? "Enable Python event $event_name for session $sess_name with loglevel $loglevel"
1008 }
1009
1010 function enable_ust_lttng_event_filter()
1011 {
1012         local sess_name="$1"
1013         local event_name="$2"
1014         local filter="$3"
1015
1016         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --filter "$filter" 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1017         ok $? "Enable event $event_name with filtering for session $sess_name"
1018 }
1019
1020 function enable_ust_lttng_event_loglevel()
1021 {
1022         local sess_name="$1"
1023         local event_name="$2"
1024         local loglevel="$3"
1025
1026         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1027         ok $? "Enable event $event_name with loglevel $loglevel"
1028 }
1029
1030 function enable_ust_lttng_event_loglevel_only()
1031 {
1032         local sess_name="$1"
1033         local event_name="$2"
1034         local loglevel="$3"
1035
1036         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-event "$event_name" -s $sess_name -u --loglevel-only $loglevel 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1037         ok $? "Enable event $event_name with loglevel-only $loglevel"
1038 }
1039
1040 function disable_ust_lttng_event ()
1041 {
1042         local sess_name="$1"
1043         local event_name="$2"
1044         local channel_name="$3"
1045
1046         if [ -z $channel_name ]; then
1047                 # default channel if none specified
1048                 chan=""
1049         else
1050                 chan="-c $channel_name"
1051         fi
1052
1053         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name $chan -u 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1054         ok $? "Disable event $event_name for session $sess_name"
1055 }
1056
1057 function disable_jul_lttng_event ()
1058 {
1059         local sess_name="$1"
1060         local event_name="$2"
1061
1062         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -j >/dev/null 2>&1
1063         ok $? "Disable JUL event $event_name for session $sess_name"
1064 }
1065
1066 function disable_log4j_lttng_event ()
1067 {
1068         local sess_name="$1"
1069         local event_name="$2"
1070
1071         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -l >/dev/null 2>&1
1072         ok $? "Disable LOG4J event $event_name for session $sess_name"
1073 }
1074
1075 function disable_python_lttng_event ()
1076 {
1077         local sess_name="$1"
1078         local event_name="$2"
1079
1080         $TESTDIR/../src/bin/lttng/$LTTNG_BIN disable-event "$event_name" -s $sess_name -p 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1081         ok $? "Disable Python event $event_name for session $sess_name"
1082 }
1083
1084 function start_lttng_tracing_opt ()
1085 {
1086         local withtap=$1
1087         local expected_to_fail=$2
1088         local sess_name=$3
1089
1090         $TESTDIR/../src/bin/lttng/$LTTNG_BIN start $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1091         ret=$?
1092         if [[ $expected_to_fail -eq "1" ]]; then
1093                 test "$ret" -ne "0"
1094                 ret=$?
1095                 if [ $withtap -eq "1" ]; then
1096                         ok $? "Start tracing for session $sess_name failed as expected"
1097                 fi
1098         else
1099                 if [ $withtap -eq "1" ]; then
1100                         ok $ret "Start tracing for session $sess_name"
1101                 fi
1102         fi
1103 }
1104
1105 function start_lttng_tracing_ok ()
1106 {
1107         start_lttng_tracing_opt 1 0 "$@"
1108 }
1109
1110 function start_lttng_tracing_fail ()
1111 {
1112         start_lttng_tracing_opt 1 1 "$@"
1113 }
1114
1115 function start_lttng_tracing_notap ()
1116 {
1117         start_lttng_tracing_opt 0 1 "$@"
1118 }
1119
1120 function stop_lttng_tracing_opt ()
1121 {
1122         local withtap=$1
1123         local expected_to_fail=$2
1124         local sess_name=$3
1125
1126         $TESTDIR/../src/bin/lttng/$LTTNG_BIN stop $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1127         ret=$?
1128         if [[ $expected_to_fail -eq "1" ]]; then
1129                 test "$ret" -ne "0"
1130                 ret=$?
1131                 if [ $withtap -eq "1" ]; then
1132                         ok $? "Stop lttng tracing for session $sess_name failed as expected"
1133                 fi
1134         else
1135                 if [ $withtap -eq "1" ]; then
1136                         ok $ret "Stop lttng tracing for session $sess_name"
1137                 fi
1138         fi
1139 }
1140
1141 function stop_lttng_tracing_ok ()
1142 {
1143         stop_lttng_tracing_opt 1 0 "$@"
1144 }
1145
1146 function stop_lttng_tracing_fail ()
1147 {
1148         stop_lttng_tracing_opt 1 1 "$@"
1149 }
1150
1151 function stop_lttng_tracing_notap ()
1152 {
1153         stop_lttng_tracing_opt 0 0 "$@"
1154 }
1155
1156 function destroy_lttng_session ()
1157 {
1158         local withtap=$1
1159         local expected_to_fail=$2
1160         local sess_name=$3
1161
1162         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1163         ret=$?
1164         if [[ $expected_to_fail -eq "1" ]]; then
1165                 test "$ret" -ne "0"
1166                 ret=$?
1167                 if [ $withtap -eq "1" ]; then
1168                         ok $ret "Destroy session $sess_name failed as expected"
1169                 fi
1170         else
1171                 if [ $withtap -eq "1" ]; then
1172                         ok $ret "Destroy session $sess_name"
1173                 fi
1174         fi
1175 }
1176
1177 function destroy_lttng_session_ok ()
1178 {
1179         destroy_lttng_session 1 0 "$@"
1180
1181 }
1182
1183 function destroy_lttng_session_fail ()
1184 {
1185         destroy_lttng_session 1 1 "$@"
1186 }
1187
1188 function destroy_lttng_session_notap ()
1189 {
1190         destroy_lttng_session 0 0 "$@"
1191 }
1192
1193 function destroy_lttng_sessions ()
1194 {
1195         $TESTDIR/../src/bin/lttng/$LTTNG_BIN destroy --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1196         ok $? "Destroy all lttng sessions"
1197 }
1198
1199 function lttng_snapshot_add_output ()
1200 {
1201         local expected_to_fail=$1
1202         local sess_name=$2
1203         local trace_path=$3
1204
1205         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot add-output -s $sess_name file://$trace_path 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1206         ret=$?
1207         if [[ $expected_to_fail -eq 1 ]]; then
1208                 test "$ret" -ne "0"
1209                 ok $? "Added snapshot output file://$trace_path failed as expected"
1210         else
1211                 ok $ret "Added snapshot output file://$trace_path"
1212         fi
1213 }
1214
1215 function lttng_snapshot_add_output_ok ()
1216 {
1217         lttng_snapshot_add_output 0 "$@"
1218 }
1219
1220 function lttng_snapshot_add_output_fail ()
1221 {
1222         lttng_snapshot_add_output 1 "$@"
1223 }
1224
1225 function lttng_snapshot_del_output ()
1226 {
1227         local expected_to_fail=$1
1228         local sess_name=$2
1229         local id=$3
1230
1231         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot del-output -s $sess_name $id 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1232         ret=$?
1233         if [[ $expected_to_fail -eq "1" ]]; then
1234                 test "$ret" -ne "0"
1235                 ok $? "Deleted snapshot output id $id failed as expected"
1236         else
1237                 ok $ret "Deleted snapshot output id $id"
1238         fi
1239 }
1240
1241 function lttng_snapshot_del_output_ok ()
1242 {
1243         lttng_snapshot_del_output 0 "$@"
1244 }
1245
1246 function lttng_snapshot_del_output_fail ()
1247 {
1248         lttng_snapshot_del_output 1 "$@"
1249 }
1250
1251 function lttng_snapshot_record ()
1252 {
1253         local sess_name=$1
1254         local trace_path=$2
1255
1256         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot record -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1257         ok $? "Snapshot recorded"
1258 }
1259
1260 function lttng_snapshot_list ()
1261 {
1262         local sess_name=$1
1263         $TESTDIR/../src/bin/lttng/$LTTNG_BIN snapshot list-output -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1264         ok $? "Snapshot list"
1265 }
1266
1267 function lttng_save()
1268 {
1269         local sess_name=$1
1270         local opts=$2
1271
1272         $TESTDIR/../src/bin/lttng/$LTTNG_BIN save $sess_name $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1273         ok $? "Session saved"
1274 }
1275
1276 function lttng_load()
1277 {
1278         local expected_to_fail=$1
1279         local opts=$2
1280
1281         $TESTDIR/../src/bin/lttng/$LTTNG_BIN load $opts 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1282         ret=$?
1283         if [[ $expected_to_fail -eq "1" ]]; then
1284                 test $ret -ne "0"
1285                 ok $? "Load command failed as expected with opts: $opts"
1286         else
1287                 ok $ret "Load command with opts: $opts"
1288         fi
1289 }
1290
1291 function lttng_load_ok()
1292 {
1293         lttng_load 0 "$@"
1294 }
1295
1296 function lttng_load_fail()
1297 {
1298         lttng_load 1 "$@"
1299 }
1300
1301 function lttng_track()
1302 {
1303         local expected_to_fail=$1
1304         local opts=$2
1305         $TESTDIR/../src/bin/lttng/$LTTNG_BIN track $opts >$OUTPUT_DEST
1306         ret=$?
1307         if [[ $expected_to_fail -eq "1" ]]; then
1308                 test $ret -ne "0"
1309                 ok $? "Track command failed as expected with opts: $opts"
1310         else
1311                 ok $ret "Track command with opts: $opts"
1312         fi
1313 }
1314
1315 function lttng_track_ok()
1316 {
1317         lttng_track 0 "$@"
1318 }
1319
1320 function lttng_track_fail()
1321 {
1322         lttng_track 1 "$@"
1323 }
1324
1325 function lttng_untrack()
1326 {
1327         local expected_to_fail=$1
1328         local opts=$2
1329         $TESTDIR/../src/bin/lttng/$LTTNG_BIN untrack $opts >$OUTPUT_DEST
1330         ret=$?
1331         if [[ $expected_to_fail -eq "1" ]]; then
1332                 test $ret -ne "0"
1333                 ok $? "Untrack command failed as expected with opts: $opts"
1334         else
1335                 ok $ret "Untrack command with opts: $opts"
1336         fi
1337 }
1338
1339 function lttng_untrack_ok()
1340 {
1341         lttng_untrack 0 "$@"
1342 }
1343
1344 function lttng_untrack_fail()
1345 {
1346         lttng_untrack 1 "$@"
1347 }
1348
1349 function lttng_track_pid_ok()
1350 {
1351         PID=$1
1352         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" track --kernel --pid=$PID 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1353         ok $? "Lttng track pid on the kernel domain"
1354 }
1355
1356 function lttng_untrack_kernel_all_ok()
1357 {
1358         "$TESTDIR/../src/bin/lttng/$LTTNG_BIN" untrack --kernel --pid --all 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1359         ok $? "Lttng untrack all pid on the kernel domain"
1360 }
1361
1362 function lttng_add_context_list()
1363 {
1364         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context --list 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1365         ret=$?
1366         ok $ret "Context listing"
1367 }
1368
1369 function add_context_lttng()
1370 {
1371         local expected_to_fail="$1"
1372         local domain="$2"
1373         local session_name="$3"
1374         local channel_name="$4"
1375         local type="$5"
1376
1377         $TESTDIR/../src/bin/lttng/$LTTNG_BIN add-context -s $session_name -c $channel_name -t $type $domain  1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1378         ret=$?
1379         if [[ $expected_to_fail -eq "1" ]]; then
1380                 test $ret -ne "0"
1381                 ok $? "Add context command failed as expected for type: $type"
1382         else
1383                 ok $ret "Add context command for type: $type"
1384         fi
1385 }
1386
1387 function add_context_ust_ok()
1388 {
1389         add_context_lttng 0 -u "$@"
1390 }
1391
1392 function add_context_ust_fail()
1393 {
1394         add_context_lttng 1 -u "$@"
1395 }
1396
1397 function add_context_kernel_ok()
1398 {
1399         add_context_lttng 0 -k "$@"
1400 }
1401
1402 function add_context_kernel_fail()
1403 {
1404         add_context_lttng 1 -k "$@"
1405 }
1406
1407 function validate_metadata_event ()
1408 {
1409         local event_name=$1
1410         local nr_event_id=$2
1411         local trace_path=$3
1412
1413         local metadata_file=$(find $trace_path | grep metadata)
1414         local metadata_path=$(dirname $metadata_file)
1415
1416         which $BABELTRACE_BIN >/dev/null
1417         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1418
1419         local count=$($BABELTRACE_BIN --output-format=ctf-metadata $metadata_path | grep $event_name | wc -l)
1420
1421         if [ "$count" -ne "$nr_event_id" ]; then
1422                 fail "Metadata match with the metadata of $count event(s) named $event_name"
1423                 diag "$count matching event id found in metadata"
1424         else
1425                 pass "Metadata match with the metadata of $count event(s) named $event_name"
1426         fi
1427
1428 }
1429
1430 function trace_matches ()
1431 {
1432         local event_name=$1
1433         local nr_iter=$2
1434         local trace_path=$3
1435
1436         which $BABELTRACE_BIN >/dev/null
1437         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1438
1439         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1440
1441         if [ "$count" -ne "$nr_iter" ]; then
1442                 fail "Trace match"
1443                 diag "$count matching events found in trace"
1444         else
1445                 pass "Trace match"
1446         fi
1447 }
1448
1449 function trace_match_only()
1450 {
1451         local event_name=$1
1452         local nr_iter=$2
1453         local trace_path=$3
1454
1455         which $BABELTRACE_BIN >/dev/null
1456         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1457
1458         local count=$($BABELTRACE_BIN $trace_path | grep $event_name | wc -l)
1459         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1460
1461         if [ "$nr_iter" -eq "$count" ] && [ "$total" -eq "$nr_iter" ]; then
1462                 pass "Trace match with $total event $event_name"
1463         else
1464                 fail "Trace match"
1465                 diag "$total event(s) found, expecting $nr_iter of event $event_name and only found $count"
1466         fi
1467 }
1468
1469 function validate_trace
1470 {
1471         local event_name=$1
1472         local trace_path=$2
1473
1474         which $BABELTRACE_BIN >/dev/null
1475         if [ $? -ne 0 ]; then
1476             skip 0 "Babeltrace binary not found. Skipping trace validation"
1477         fi
1478
1479         OLDIFS=$IFS
1480         IFS=","
1481         for i in $event_name; do
1482                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1483                 if [ "$traced" -ne 0 ]; then
1484                         pass "Validate trace for event $i, $traced events"
1485                 else
1486                         fail "Validate trace for event $i"
1487                         diag "Found $traced occurences of $i"
1488                 fi
1489         done
1490         ret=$?
1491         IFS=$OLDIFS
1492         return $ret
1493 }
1494
1495 function validate_trace_count
1496 {
1497         local event_name=$1
1498         local trace_path=$2
1499         local expected_count=$3
1500
1501         which $BABELTRACE_BIN >/dev/null
1502         if [ $? -ne 0 ]; then
1503             skip 0 "Babeltrace binary not found. Skipping trace validation"
1504         fi
1505
1506         cnt=0
1507         OLDIFS=$IFS
1508         IFS=","
1509         for i in $event_name; do
1510                 traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep $i | wc -l)
1511                 if [ "$traced" -ne 0 ]; then
1512                         pass "Validate trace for event $i, $traced events"
1513                 else
1514                         fail "Validate trace for event $i"
1515                         diag "Found $traced occurences of $i"
1516                 fi
1517                 cnt=$(($cnt + $traced))
1518         done
1519         IFS=$OLDIFS
1520         test $cnt -eq $expected_count
1521         ok $? "Read a total of $cnt events, expected $expected_count"
1522 }
1523
1524 function trace_first_line
1525 {
1526         local trace_path=$1
1527
1528         which $BABELTRACE_BIN >/dev/null
1529         if [ $? -ne 0 ]; then
1530             skip 0 "Babeltrace binary not found. Skipping trace validation"
1531         fi
1532
1533         $BABELTRACE_BIN $trace_path 2>/dev/null | head -n 1
1534 }
1535
1536 function validate_trace_exp()
1537 {
1538         local event_exp=$1
1539         local trace_path=$2
1540
1541         which $BABELTRACE_BIN >/dev/null
1542         skip $? -ne 0 "Babeltrace binary not found. Skipping trace validation"
1543
1544         traced=$($BABELTRACE_BIN $trace_path 2>/dev/null | grep --extended-regexp ${event_exp} | wc -l)
1545         if [ "$traced" -ne 0 ]; then
1546                 pass "Validate trace for expression '${event_exp}', $traced events"
1547         else
1548                 fail "Validate trace for expression '${event_exp}'"
1549                 diag "Found $traced occurences of '${event_exp}'"
1550         fi
1551         ret=$?
1552         return $ret
1553 }
1554
1555 function validate_trace_only_exp()
1556 {
1557         local event_exp=$1
1558         local trace_path=$2
1559
1560         which $BABELTRACE_BIN >/dev/null
1561         skip $? -ne 0 "Babeltrace binary not found. Skipping trace matches"
1562
1563         local count=$($BABELTRACE_BIN $trace_path | grep --extended-regexp ${event_exp} | wc -l)
1564         local total=$($BABELTRACE_BIN $trace_path | wc -l)
1565
1566         if [ "$count" -ne 0 ] && [ "$total" -eq "$count" ]; then
1567                 pass "Trace match with $total for expression '${event_exp}'"
1568         else
1569                 fail "Trace match"
1570                 diag "$total syscall event(s) found, only syscalls matching expression '${event_exp}' ($count occurrences) are expected"
1571         fi
1572         ret=$?
1573         return $ret
1574 }
1575
1576 function validate_trace_empty()
1577 {
1578         local trace_path=$1
1579
1580         which $BABELTRACE_BIN >/dev/null
1581         if [ $? -ne 0 ]; then
1582             skip 0 "Babeltrace binary not found. Skipping trace validation"
1583         fi
1584
1585         events=$($BABELTRACE_BIN $trace_path 2>/dev/null)
1586         ret=$?
1587         if [ $ret -ne 0 ]; then
1588                 fail "Failed to parse trace"
1589                 return $ret
1590         fi
1591
1592         traced=$(echo -n "$events" | wc -l)
1593         if [ "$traced" -eq 0 ]; then
1594                 pass "Validate empty trace"
1595         else
1596                 fail "Validate empty trace"
1597                 diag "Found $traced events in trace"
1598         fi
1599         ret=$?
1600         return $ret
1601 }
1602
1603 function regenerate_metadata ()
1604 {
1605         local expected_to_fail=$1
1606         local sess_name=$2
1607
1608         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate metadata -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1609         ret=$?
1610         if [[ $expected_to_fail -eq "1" ]]; then
1611                 test "$ret" -ne "0"
1612                 ok $? "Expected fail on regenerate metadata $sess_name"
1613         else
1614                 ok $ret "Metadata regenerate $sess_name"
1615         fi
1616 }
1617
1618 function regenerate_metadata_ok ()
1619 {
1620         regenerate_metadata 0 "$@"
1621 }
1622
1623 function regenerate_metadata_fail ()
1624 {
1625         regenerate_metadata 1 "$@"
1626 }
1627
1628 function regenerate_statedump ()
1629 {
1630         local expected_to_fail=$1
1631         local sess_name=$2
1632
1633         $TESTDIR/../src/bin/lttng/$LTTNG_BIN regenerate statedump -s $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1634         ret=$?
1635         if [[ $expected_to_fail -eq "1" ]]; then
1636                 test "$ret" -ne "0"
1637                 ok $? "Expected fail on regenerate statedump $sess_name"
1638         else
1639                 ok $ret "Metadata regenerate $sess_name"
1640         fi
1641 }
1642
1643 function regenerate_statedump_ok ()
1644 {
1645         regenerate_statedump 0 "$@"
1646 }
1647
1648 function regenerate_statedump_fail ()
1649 {
1650         regenerate_statedump 1 "$@"
1651 }
1652
1653 function rotate_session ()
1654 {
1655         local expected_to_fail=$1
1656         local sess_name=$2
1657
1658         $TESTDIR/../src/bin/lttng/$LTTNG_BIN rotate $sess_name 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1659         ret=$?
1660         if [[ $expected_to_fail -eq "1" ]]; then
1661                 test "$ret" -ne "0"
1662                 ok $? "Expected fail on rotate session $sess_name"
1663         else
1664                 ok $ret "Rotate session $sess_name"
1665         fi
1666 }
1667
1668 function rotate_session_ok ()
1669 {
1670         rotate_session 0 "$@"
1671 }
1672
1673 function rotate_session_fail ()
1674 {
1675         rotate_session 1 "$@"
1676 }
1677
1678 function destructive_tests_enabled ()
1679 {
1680         if [ ${LTTNG_ENABLE_DESTRUCTIVE_TESTS} = "will-break-my-system" ]; then
1681                 return 0
1682         else
1683                 return 1
1684         fi
1685 }
1686
1687 function lttng_enable_rotation_timer ()
1688 {
1689         local expected_to_fail=$1
1690         local sess_name=$2
1691         local period=$3
1692
1693         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --timer $period 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1694         ret=$?
1695         if [[ $expected_to_fail -eq "1" ]]; then
1696                 test "$ret" -ne "0"
1697                 ok $? "Expected fail on rotate session $sess_name"
1698         else
1699                 ok $ret "Rotate session $sess_name"
1700         fi
1701 }
1702
1703 function lttng_enable_rotation_timer_ok ()
1704 {
1705         lttng_enable_rotation_timer 0 $@
1706 }
1707
1708 function lttng_enable_rotation_timer_fail ()
1709 {
1710         lttng_enable_rotation_timer 1 $@
1711 }
1712
1713 function lttng_enable_rotation_size ()
1714 {
1715         local expected_to_fail=$1
1716         local sess_name=$2
1717         local size=$3
1718
1719         $TESTDIR/../src/bin/lttng/$LTTNG_BIN enable-rotation -s $sess_name --size $size 1> $OUTPUT_DEST 2> $ERROR_OUTPUT_DEST
1720         ret=$?
1721         if [[ $expected_to_fail -eq "1" ]]; then
1722                 test "$ret" -ne "0"
1723                 ok $? "Expected fail on rotate session $sess_name"
1724         else
1725                 ok $ret "Rotate session $sess_name"
1726         fi
1727 }
1728
1729 function lttng_enable_rotation_size_ok ()
1730 {
1731         lttng_enable_rotation_size 0 $@
1732 }
1733
1734 function lttng_enable_rotation_size_fail ()
1735 {
1736         lttng_enable_rotation_size 1 $@
1737 }