[lttng-tools.git] / tests / regression / tools / notification / test_notification_multi_app

#!/bin/bash
#
# Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
#
# SPDX-License-Identifier: LGPL-2.1-only

TEST_DESC="Notification"

CURDIR=$(dirname $0)/
TESTDIR=$CURDIR/../../../

TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)

TESTAPP_PATH="$TESTDIR/utils/testapp"
TESTAPP_NAME="gen-ust-events"
TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
TESTAPP_STATE_FILE="$(mktemp -u)"

NR_ITER=1000
NR_USEC_WAIT=5

SESSION_NAME="my_session"
CHANNEL_NAME="my_channel"


DIR=$(readlink -f $TESTDIR)

PAGE_SIZE=$(getconf PAGE_SIZE)

NUM_TEST_UST=50
NUM_TEST_KERNEL=50
NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))

source $TESTDIR/utils/utils.sh

consumerd_pipe=()
file_sync_after_first_event=$(mktemp -u)

# MUST set TESTDIR before calling those functions
plan_tests $NUM_TESTS

print_test_banner "$TEST_DESC"

app_pids=()

function kernel_event_generator_toggle_state
{
	kernel_event_generator_suspended=$((kernel_event_generator_suspended==0))

}
function kernel_event_generator
{
	state_file=$1
	kernel_event_generator_suspended=0
	trap kernel_event_generator_toggle_state SIGUSR1

	while (true); do
		if [[ $kernel_event_generator_suspended -eq "1" ]]; then
			touch $state_file
			sleep 0.5
		else
			if [[ -f $state_file ]]; then
				rm $state_file 2> /dev/null
			fi
			echo -n "1000" > /proc/lttng-test-filter-event
		fi
	done
}

function ust_event_generator_toggle_state
{
	ust_event_generator_suspended=$((ust_event_generator_suspended==0))

}
function ust_event_generator
{
	state_file=$1
	ust_event_generator_suspended=0
	trap ust_event_generator_toggle_state SIGUSR1
	trap "exit" SIGTERM SIGINT
	while (true); do
		if [[ $ust_event_generator_suspended -eq "1" ]]; then
			touch $state_file
			sleep 0.5
		else
			if [[ -f $state_file ]]; then
				rm $state_file 2> /dev/null
			fi
			taskset  -c 0 $TESTAPP_BIN -i $NR_ITER -w $NR_USEC_WAIT > /dev/null 2>&1
		fi
	done
}

function start_client {
	local pid=-1
	local output_file=$1
	local session_name=$2
	local channel_name=$3
	local domain_type=$4
	local buffer_usage_type=$5
	local buffer_usage_threshold_type=$6
	local buffer_usage_threshold_value=$7
	local nr_expected_notification=$8

	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	pid=$!

	app_pids+=("$pid")
}

function wait_for_message ()
{
	local directory=$1
	local file_pattern=$2
	local message=$3

	for file in $directory/${file_pattern}*; do
		while(true); do
			# Check for "error" message
			grep -q "error:" ${file}
			app_error=$?
			if [ $app_error -eq "0" ] ; then
				# An error occurred
				fail "Waiting message: error logged see file content: ${message}, ${file}"
				return 1
			fi

			grep -q "${message}" ${file}
			if [[ "$?" -ne "0" ]]; then
				# Lookup failed restart loop
				diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
				sleep 0.25
				continue
			fi
			break
		done
	done
	pass "Message received: ${message}"
	return 0
}

function print_errors ()
{
	local directory=$1
	local file_pattern=$2

	for file in $directory/${file_pattern}*; do
		# Check for "error" message
		error_message=$(grep "error:" ${file})
		if [[ "${error_message}x" != "x" ]]; then
			diag "Errors for application ${file}:"
			diag "${error_message}"
		fi
	done
}

function comm_consumerd ()
{
	local message=$1
	local pipe=$2
	echo -ne "${message}" > "${pipe}"
}

function stop_consumerd ()
{
	local pipe=$1
	comm_consumerd "1" "$pipe"
}

function resume_consumerd ()
{
	local pipe=$1
	comm_consumerd "\0" "$pipe"
}

function test_multi_app ()
{
	local domain_type=$1
	local event_generator_pid=$2

	local app_pids=()
	local low_output_file_pattern="low_app_output_file_"
	local high_output_file_pattern="high_app_output_file_"
	local output_dir=$(mktemp -d)

	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")

	local nr_notification_expected=5
	local nr_client_app=50
	local domain_string=""
	local event_name=""

	case $domain_type in
		ust)
			domain_string=LTTNG_DOMAIN_UST
			event_name="tp:tptest"
			;;
		kernel)
			domain_string=LTTNG_DOMAIN_KERNEL
			event_name="lttng_test_filter_event"
			;;
		*)
			fail "Invalid domain type"
			exit 1
			;;
	esac

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	consumerd_pipe=()
	for f in "$testpoint_pipe_path"*; do
		consumerd_pipe+=("$f")
	done

	for (( i = 0; i < $nr_client_app; i++ )); do
		low_app_output_file=$output_dir/${low_output_file_pattern}${i}
		high_app_output_file=$output_dir/${high_output_file_pattern}${i}
		start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected
		start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected
	done

	wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
	wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"

	# Test notification reception
	for (( i = 0; i < $nr_notification_expected; i++ )); do

		# Stop consumerd consumption to force high notification
		start_lttng_tracing_ok $SESSION_NAME
		for pipe in "${consumerd_pipe[@]}"; do
			stop_consumerd "${pipe}"
		done

		wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"

		# Put application in suspend mode to prevent double low
		# notification and synchronize on state file.
		kill -s SIGUSR1 $event_generator_pid
		while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
			sleep 0.5
		done

		# Resume consumerd
		for pipe in "${consumerd_pipe[@]}"; do
			resume_consumerd "${pipe}"
		done
		# Stop tracing forcing full buffer consumption
		stop_lttng_tracing_ok $SESSION_NAME

		# Check for notifications reception
		wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
		ret=$?
		ok $ret "Notifications $i received"
		if [[ $ret -ne "0" ]]; then
			# Error occurred bail out
			break;
		fi

		# Put application in active mode and synchronize on state file.
		kill -s SIGUSR1 $event_generator_pid
		while [ -f "${TESTAPP_STATE_FILE}" ]; do
			sleep 0.5
		done
	done

	wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for low notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		print_errors $output_dir "${low_output_file_pattern}"
	fi

	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		print_errors $output_dir "${high_output_file_pattern}"
	fi


	destroy_lttng_session_ok $SESSION_NAME
	stop_lttng_sessiond

	for pipe in "${consumerd_pipe[@]}"; do
		rm -rf "${pipe}"
	done

	rm -rf $output_dir
}

function test_multi_app_ust ()
{
	diag "Multi client app UST notification"
	ust_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_multi_app ust $generator_pid

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
}

function test_multi_app_kernel ()
{
	diag "Multi client app kernel notification"
	modprobe lttng-test

	kernel_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_multi_app kernel $generator_pid


	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

	rmmod lttng-test
}

function test_on_register_evaluation_ust ()
{
	diag "On register notification UST"

	# Start app in infinite loop
	ust_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_on_register_evaluation ust $generator_pid

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

}

function test_on_register_evaluation_kernel()
{
	diag "On register notification kernel"

	modprobe lttng-test

	kernel_event_generator $TESTAPP_STATE_FILE &
	local generator_pid=$!

	test_on_register_evaluation kernel $generator_pid


	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null
	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null

	rmmod lttng-test
}

function test_on_register_evaluation ()
{
	local domain_type=$1
	local event_generator_pid=$2

	local app_pids=()
	local high_output_file_pattern="high_app_output_file_on_register_evaluation"

	local output_dir=$(mktemp -d)
	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	local domain_string=""
	local event_name=""

	case $domain_type in
		ust)
			domain_string=LTTNG_DOMAIN_UST
			event_name="tp:tptest"
			;;
		kernel)
			domain_string=LTTNG_DOMAIN_KERNEL
			event_name="lttng_test_filter_event"
			;;
		*)
			fail "Invalid domain type"
			exit 1
			;;
	esac

	# Setup
	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	start_lttng_sessiond

	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME

	# Fetch consumerd testpoint pipe information
	# This is needed since the testpoint create a pipe with the consumer type suffixed
	consumerd_pipe=()
	for f in "$testpoint_pipe_path"*; do
		consumerd_pipe+=("$f")
	done

	high_app_output_file=${high_output_file_pattern}.first_receiver
	high_app_output_path=$output_dir/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1

	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"

	# Stop consumerd consumption to force high notification
	start_lttng_tracing_ok $SESSION_NAME

	for pipe in "${consumerd_pipe[@]}"; do
		stop_consumerd "${pipe}"
	done

	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"

	# Start a second receiver, the receiver should receive a high
	# notification on subscription
	high_app_output_file=${high_output_file_pattern}.second_receiver
	high_app_output_path=$output_dir/${high_app_output_file}
	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"

	# Resume consumerd
	for pipe in "${consumerd_pipe[@]}"; do
		resume_consumerd "${pipe}"
	done

	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	ret=$?
	ok $ret "Application for high notification terminated normally"
	if [[ $ret -ne "0" ]]; then
		# Keep the file
		print_errors "${high_output_file_pattern}"
	fi


	destroy_lttng_session_ok $SESSION_NAME
	stop_lttng_sessiond

	kill -s SIGTERM $generator_pid 2> /dev/null
	wait $generator_pid 2> /dev/null

	for pipe in "${consumerd_pipe[@]}"; do
		rm -rf "${pipe}"
	done

	rm -rf "$output_dir"
}


TESTS=(
	test_multi_app_ust
	test_on_register_evaluation_ust
)

if [ "$(id -u)" == "0" ]; then
	validate_lttng_modules_present
	TESTS+=(
	test_multi_app_kernel
	test_on_register_evaluation_kernel
)
else
        skip 0 "Root access is needed. Skipping all kernel multi-app notification tests." $NUM_TEST_KERNEL
fi


for fct_test in ${TESTS[@]};
do
	TRACE_PATH=$(mktemp -d)

	${fct_test}
	if [ $? -ne 0 ]; then
		break;
	fi

	# Only delete if successful
	rm -rf $TRACE_PATH
done
Commit	Line	Data
	1	#!/bin/bash
	2	#
	3	# Copyright (C) 2017 Jonathan Rajotte <jonathan.rajotte-julien@efficiso.com>>
	4	#
	5	# SPDX-License-Identifier: LGPL-2.1-only
	6
	7	TEST_DESC="Notification"
	8
	9	CURDIR=$(dirname $0)/
	10	TESTDIR=$CURDIR/../../../
	11
	12	TESTPOINT=$(readlink -f ${CURDIR}/.libs/libpause_consumer.so)
	13
	14	TESTAPP_PATH="$TESTDIR/utils/testapp"
	15	TESTAPP_NAME="gen-ust-events"
	16	TESTAPP_BIN="$TESTAPP_PATH/$TESTAPP_NAME/$TESTAPP_NAME"
	17	TESTAPP_STATE_FILE="$(mktemp -u)"
	18
	19	NR_ITER=1000
	20	NR_USEC_WAIT=5
	21
	22	SESSION_NAME="my_session"
	23	CHANNEL_NAME="my_channel"
	24
	25
	26	DIR=$(readlink -f $TESTDIR)
	27
	28	PAGE_SIZE=$(getconf PAGE_SIZE)
	29
	30	NUM_TEST_UST=50
	31	NUM_TEST_KERNEL=50
	32	NUM_TESTS=$(($NUM_TEST_UST + $NUM_TEST_KERNEL))
	33
	34	source $TESTDIR/utils/utils.sh
	35
	36	consumerd_pipe=()
	37	file_sync_after_first_event=$(mktemp -u)
	38
	39	# MUST set TESTDIR before calling those functions
	40	plan_tests $NUM_TESTS
	41
	42	print_test_banner "$TEST_DESC"
	43
	44	app_pids=()
	45
	46	function kernel_event_generator_toggle_state
	47	{
	48	kernel_event_generator_suspended=$((kernel_event_generator_suspended==0))
	49
	50	}
	51	function kernel_event_generator
	52	{
	53	state_file=$1
	54	kernel_event_generator_suspended=0
	55	trap kernel_event_generator_toggle_state SIGUSR1
	56
	57	while (true); do
	58	if [[ $kernel_event_generator_suspended -eq "1" ]]; then
	59	touch $state_file
	60	sleep 0.5
	61	else
	62	if [[ -f $state_file ]]; then
	63	rm $state_file 2> /dev/null
	64	fi
	65	echo -n "1000" > /proc/lttng-test-filter-event
	66	fi
	67	done
	68	}
	69
	70	function ust_event_generator_toggle_state
	71	{
	72	ust_event_generator_suspended=$((ust_event_generator_suspended==0))
	73
	74	}
	75	function ust_event_generator
	76	{
	77	state_file=$1
	78	ust_event_generator_suspended=0
	79	trap ust_event_generator_toggle_state SIGUSR1
	80	trap "exit" SIGTERM SIGINT
	81	while (true); do
	82	if [[ $ust_event_generator_suspended -eq "1" ]]; then
	83	touch $state_file
	84	sleep 0.5
	85	else
	86	if [[ -f $state_file ]]; then
	87	rm $state_file 2> /dev/null
	88	fi
	89	taskset -c 0 $TESTAPP_BIN -i $NR_ITER -w $NR_USEC_WAIT > /dev/null 2>&1
	90	fi
	91	done
	92	}
	93
	94	function start_client {
	95	local pid=-1
	96	local output_file=$1
	97	local session_name=$2
	98	local channel_name=$3
	99	local domain_type=$4
	100	local buffer_usage_type=$5
	101	local buffer_usage_threshold_type=$6
	102	local buffer_usage_threshold_value=$7
	103	local nr_expected_notification=$8
	104
	105	${CURDIR}/base_client ${session_name} ${channel_name} ${domain_type} ${buffer_usage_type} ${buffer_usage_threshold_type} ${buffer_usage_threshold_value} ${nr_expected_notification} > ${output_file} &
	106	pid=$!
	107
	108	app_pids+=("$pid")
	109	}
	110
	111	function wait_for_message ()
	112	{
	113	local directory=$1
	114	local file_pattern=$2
	115	local message=$3
	116
	117	for file in $directory/${file_pattern}*; do
	118	while(true); do
	119	# Check for "error" message
	120	grep -q "error:" ${file}
	121	app_error=$?
	122	if [ $app_error -eq "0" ] ; then
	123	# An error occurred
	124	fail "Waiting message: error logged see file content: ${message}, ${file}"
	125	return 1
	126	fi
	127
	128	grep -q "${message}" ${file}
	129	if [[ "$?" -ne "0" ]]; then
	130	# Lookup failed restart loop
	131	diag "Lookup failed sleep and retry grep for: ${message}, ${file}"
	132	sleep 0.25
	133	continue
	134	fi
	135	break
	136	done
	137	done
	138	pass "Message received: ${message}"
	139	return 0
	140	}
	141
	142	function print_errors ()
	143	{
	144	local directory=$1
	145	local file_pattern=$2
	146
	147	for file in $directory/${file_pattern}*; do
	148	# Check for "error" message
	149	error_message=$(grep "error:" ${file})
	150	if [[ "${error_message}x" != "x" ]]; then
	151	diag "Errors for application ${file}:"
	152	diag "${error_message}"
	153	fi
	154	done
	155	}
	156
	157	function comm_consumerd ()
	158	{
	159	local message=$1
	160	local pipe=$2
	161	echo -ne "${message}" > "${pipe}"
	162	}
	163
	164	function stop_consumerd ()
	165	{
	166	local pipe=$1
	167	comm_consumerd "1" "$pipe"
	168	}
	169
	170	function resume_consumerd ()
	171	{
	172	local pipe=$1
	173	comm_consumerd "\0" "$pipe"
	174	}
	175
	176	function test_multi_app ()
	177	{
	178	local domain_type=$1
	179	local event_generator_pid=$2
	180
	181	local app_pids=()
	182	local low_output_file_pattern="low_app_output_file_"
	183	local high_output_file_pattern="high_app_output_file_"
	184	local output_dir=$(mktemp -d)
	185
	186	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_multi_app")
	187	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	188
	189	local nr_notification_expected=5
	190	local nr_client_app=50
	191	local domain_string=""
	192	local event_name=""
	193
	194	case $domain_type in
	195	ust)
	196	domain_string=LTTNG_DOMAIN_UST
	197	event_name="tp:tptest"
	198	;;
	199	kernel)
	200	domain_string=LTTNG_DOMAIN_KERNEL
	201	event_name="lttng_test_filter_event"
	202	;;
	203	*)
	204	fail "Invalid domain type"
	205	exit 1
	206	;;
	207	esac
	208
	209	# Setup
	210	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	211	start_lttng_sessiond
	212
	213	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	214	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	215	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
	216
	217	# Fetch consumerd testpoint pipe information
	218	# This is needed since the testpoint create a pipe with the consumer type suffixed
	219	consumerd_pipe=()
	220	for f in "$testpoint_pipe_path"*; do
	221	consumerd_pipe+=("$f")
	222	done
	223
	224	for (( i = 0; i < $nr_client_app; i++ )); do
	225	low_app_output_file=$output_dir/${low_output_file_pattern}${i}
	226	high_app_output_file=$output_dir/${high_output_file_pattern}${i}
	227	start_client $low_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string LOW RATIO 0.0 $nr_notification_expected
	228	start_client $high_app_output_file $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 $nr_notification_expected
	229	done
	230
	231	wait_for_message $output_dir "${low_output_file_pattern}" "sync: ready"
	232	wait_for_message $output_dir "${high_output_file_pattern}" "sync: ready"
	233
	234	# Test notification reception
	235	for (( i = 0; i < $nr_notification_expected; i++ )); do
	236
	237	# Stop consumerd consumption to force high notification
	238	start_lttng_tracing_ok $SESSION_NAME
	239	for pipe in "${consumerd_pipe[@]}"; do
	240	stop_consumerd "${pipe}"
	241	done
	242
	243	wait_for_message $output_dir "${high_output_file_pattern}" "notification: high $i"
	244
	245	# Put application in suspend mode to prevent double low
	246	# notification and synchronize on state file.
	247	kill -s SIGUSR1 $event_generator_pid
	248	while [ ! -f "${TESTAPP_STATE_FILE}" ]; do
	249	sleep 0.5
	250	done
	251
	252	# Resume consumerd
	253	for pipe in "${consumerd_pipe[@]}"; do
	254	resume_consumerd "${pipe}"
	255	done
	256	# Stop tracing forcing full buffer consumption
	257	stop_lttng_tracing_ok $SESSION_NAME
	258
	259	# Check for notifications reception
	260	wait_for_message $output_dir "${low_output_file_pattern}" "notification: low $i"
	261	ret=$?
	262	ok $ret "Notifications $i received"
	263	if [[ $ret -ne "0" ]]; then
	264	# Error occurred bail out
	265	break;
	266	fi
	267
	268	# Put application in active mode and synchronize on state file.
	269	kill -s SIGUSR1 $event_generator_pid
	270	while [ -f "${TESTAPP_STATE_FILE}" ]; do
	271	sleep 0.5
	272	done
	273	done
	274
	275	wait_for_message $output_dir "${low_output_file_pattern}" "exit: 0"
	276	ret=$?
	277	ok $ret "Application for low notification terminated normally"
	278	if [[ $ret -ne "0" ]]; then
	279	print_errors $output_dir "${low_output_file_pattern}"
	280	fi
	281
	282	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	283	ret=$?
	284	ok $ret "Application for high notification terminated normally"
	285	if [[ $ret -ne "0" ]]; then
	286	print_errors $output_dir "${high_output_file_pattern}"
	287	fi
	288
	289
	290	destroy_lttng_session_ok $SESSION_NAME
	291	stop_lttng_sessiond
	292
	293	for pipe in "${consumerd_pipe[@]}"; do
	294	rm -rf "${pipe}"
	295	done
	296
	297	rm -rf $output_dir
	298	}
	299
	300	function test_multi_app_ust ()
	301	{
	302	diag "Multi client app UST notification"
	303	ust_event_generator $TESTAPP_STATE_FILE &
	304	local generator_pid=$!
	305
	306	test_multi_app ust $generator_pid
	307
	308	kill -s SIGTERM $generator_pid 2> /dev/null
	309	wait $generator_pid 2> /dev/null
	310	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	311	}
	312
	313	function test_multi_app_kernel ()
	314	{
	315	diag "Multi client app kernel notification"
	316	modprobe lttng-test
	317
	318	kernel_event_generator $TESTAPP_STATE_FILE &
	319	local generator_pid=$!
	320
	321	test_multi_app kernel $generator_pid
	322
	323
	324	kill -s SIGTERM $generator_pid 2> /dev/null
	325	wait $generator_pid 2> /dev/null
	326	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	327
	328	rmmod lttng-test
	329	}
	330
	331	function test_on_register_evaluation_ust ()
	332	{
	333	diag "On register notification UST"
	334
	335	# Start app in infinite loop
	336	ust_event_generator $TESTAPP_STATE_FILE &
	337	local generator_pid=$!
	338
	339	test_on_register_evaluation ust $generator_pid
	340
	341	kill -s SIGTERM $generator_pid 2> /dev/null
	342	wait $generator_pid 2> /dev/null
	343	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	344
	345	}
	346
	347	function test_on_register_evaluation_kernel()
	348	{
	349	diag "On register notification kernel"
	350
	351	modprobe lttng-test
	352
	353	kernel_event_generator $TESTAPP_STATE_FILE &
	354	local generator_pid=$!
	355
	356	test_on_register_evaluation kernel $generator_pid
	357
	358
	359	kill -s SIGTERM $generator_pid 2> /dev/null
	360	wait $generator_pid 2> /dev/null
	361	rm -rf ${TESTAPP_STATE_FILE} 2> /dev/null
	362
	363	rmmod lttng-test
	364	}
	365
	366	function test_on_register_evaluation ()
	367	{
	368	local domain_type=$1
	369	local event_generator_pid=$2
	370
	371	local app_pids=()
	372	local high_output_file_pattern="high_app_output_file_on_register_evaluation"
	373
	374	local output_dir=$(mktemp -d)
	375	local testpoint_base_path=$(readlink -f "$output_dir/lttng.t_p_n_register_evaluation")
	376	local testpoint_pipe_path=$(mktemp -u "${testpoint_base_path}.XXXXXX")
	377	local domain_string=""
	378	local event_name=""
	379
	380	case $domain_type in
	381	ust)
	382	domain_string=LTTNG_DOMAIN_UST
	383	event_name="tp:tptest"
	384	;;
	385	kernel)
	386	domain_string=LTTNG_DOMAIN_KERNEL
	387	event_name="lttng_test_filter_event"
	388	;;
	389	*)
	390	fail "Invalid domain type"
	391	exit 1
	392	;;
	393	esac
	394
	395	# Setup
	396	LTTNG_SESSIOND_ENV_VARS="LTTNG_TESTPOINT_ENABLE=1 CONSUMER_PAUSE_PIPE_PATH=${testpoint_pipe_path} LD_PRELOAD=${TESTPOINT}"
	397	start_lttng_sessiond
	398
	399	create_lttng_session_ok $SESSION_NAME $TRACE_PATH
	400	enable_${domain_type}_lttng_channel_ok $SESSION_NAME $CHANNEL_NAME --subbuf-size=$PAGE_SIZE
	401	enable_${domain_type}_lttng_event_ok $SESSION_NAME $event_name $CHANNEL_NAME
	402
	403	# Fetch consumerd testpoint pipe information
	404	# This is needed since the testpoint create a pipe with the consumer type suffixed
	405	consumerd_pipe=()
	406	for f in "$testpoint_pipe_path"*; do
	407	consumerd_pipe+=("$f")
	408	done
	409
	410	high_app_output_file=${high_output_file_pattern}.first_receiver
	411	high_app_output_path=$output_dir/${high_app_output_file}
	412	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	413
	414	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	415
	416	# Stop consumerd consumption to force high notification
	417	start_lttng_tracing_ok $SESSION_NAME
	418
	419	for pipe in "${consumerd_pipe[@]}"; do
	420	stop_consumerd "${pipe}"
	421	done
	422
	423	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
	424
	425	# Start a second receiver, the receiver should receive a high
	426	# notification on subscription
	427	high_app_output_file=${high_output_file_pattern}.second_receiver
	428	high_app_output_path=$output_dir/${high_app_output_file}
	429	start_client $high_app_output_path $SESSION_NAME $CHANNEL_NAME $domain_string HIGH RATIO 0.420 1
	430	wait_for_message $output_dir "${high_app_output_file}" "sync: ready"
	431	wait_for_message $output_dir "${high_app_output_file}" "notification: high 0"
	432
	433	# Resume consumerd
	434	for pipe in "${consumerd_pipe[@]}"; do
	435	resume_consumerd "${pipe}"
	436	done
	437
	438	wait_for_message $output_dir "${high_output_file_pattern}" "exit: 0"
	439	ret=$?
	440	ok $ret "Application for high notification terminated normally"
	441	if [[ $ret -ne "0" ]]; then
	442	# Keep the file
	443	print_errors "${high_output_file_pattern}"
	444	fi
	445
	446
	447	destroy_lttng_session_ok $SESSION_NAME
	448	stop_lttng_sessiond
	449
	450	kill -s SIGTERM $generator_pid 2> /dev/null
	451	wait $generator_pid 2> /dev/null
	452
	453	for pipe in "${consumerd_pipe[@]}"; do
	454	rm -rf "${pipe}"
	455	done
	456
	457	rm -rf "$output_dir"
	458	}
	459
	460
	461	TESTS=(
	462	test_multi_app_ust
	463	test_on_register_evaluation_ust
	464	)
	465
	466	if [ "$(id -u)" == "0" ]; then
	467	validate_lttng_modules_present
	468	TESTS+=(
	469	test_multi_app_kernel
	470	test_on_register_evaluation_kernel
	471	)
	472	else
	473	skip 0 "Root access is needed. Skipping all kernel multi-app notification tests." $NUM_TEST_KERNEL
	474	fi
	475
	476
	477	for fct_test in ${TESTS[@]};
	478	do
	479	TRACE_PATH=$(mktemp -d)
	480
	481	${fct_test}
	482	if [ $? -ne 0 ]; then
	483	break;
	484	fi
	485
	486	# Only delete if successful
	487	rm -rf $TRACE_PATH
	488	done