From 58db335ee64dfdedc0e351fd8e1524e7ca381896 Mon Sep 17 00:00:00 2001 From: Francis Deslauriers Date: Fri, 5 Jul 2019 12:02:47 -0400 Subject: [PATCH] tests: bt_diff_cli: compare expected `stderr` too Useful to confirm that a trace is read without warnings but could also be used to confirm that an expected warning is indeed present. Signed-off-by: Francis Deslauriers Change-Id: I0c48c68a0aac832729da2d9c5b849fb58ec578e4 Reviewed-on: https://review.lttng.org/c/babeltrace/+/1638 Tested-by: jenkins Reviewed-by: Philippe Proulx --- tests/plugins/flt.utils.trimmer/test_trimming | 73 ++++++++++--------- tests/utils/utils.sh | 61 +++++++++++----- 2 files changed, 80 insertions(+), 54 deletions(-) diff --git a/tests/plugins/flt.utils.trimmer/test_trimming b/tests/plugins/flt.utils.trimmer/test_trimming index 3ec05a41..5f84bf1b 100755 --- a/tests/plugins/flt.utils.trimmer/test_trimming +++ b/tests/plugins/flt.utils.trimmer/test_trimming @@ -31,7 +31,8 @@ fi source "$UTILSSH" data_dir="$BT_TESTS_DATADIR/plugins/flt.utils.trimmer" -temp_expected=$(mktemp) +temp_stdout_expected=$(mktemp) +temp_stderr_expected="/dev/null" plan_tests 32 @@ -63,7 +64,7 @@ function run_test # with_stream_msgs_cs is set to "true" or "false" by the tests. - bt_diff_cli "$temp_expected" "${local_args[@]}" + bt_diff_cli "$temp_stdout_expected" "$temp_stderr_expected" "${local_args[@]}" ok $? "$test_name" } @@ -71,7 +72,7 @@ function test_with_stream_msg_cs { with_stream_msgs_cs="true" # Baseline (without trimming) - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -82,7 +83,7 @@ function test_with_stream_msg_cs { run_test "" "" # Trim begin at a time before what the clock class can represent - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -93,7 +94,7 @@ function test_with_stream_msg_cs { run_test 50 "" # Trim begin before stream beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -104,7 +105,7 @@ function test_with_stream_msg_cs { run_test 10050 "" # Trim begin before packet beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [150 10,150,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -115,7 +116,7 @@ function test_with_stream_msg_cs { run_test 10150 "" # Trim begin before first event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [250 10,250,000,000,000] {0 0 0} Stream beginning [250 10,250,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -126,7 +127,7 @@ function test_with_stream_msg_cs { run_test 10250 "" # Trim begin before second event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [350 10,350,000,000,000] {0 0 0} Stream beginning [350 10,350,000,000,000] {0 0 0} Packet beginning [400 10,400,000,000,000] {0 0 0} Event `event 2` (1) @@ -137,7 +138,7 @@ function test_with_stream_msg_cs { run_test 10350 "" # Trim begin before packet end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [850 10,850,000,000,000] {0 0 0} Stream beginning [850 10,850,000,000,000] {0 0 0} Packet beginning [900 10,900,000,000,000] {0 0 0} Packet end @@ -147,13 +148,13 @@ function test_with_stream_msg_cs { run_test 10850 "" # Trim begin after everything - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" END run_test 11050 "" # Trim end after stream end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -165,7 +166,7 @@ function test_with_stream_msg_cs { run_test "" 11050 # Trim end after packet end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -177,7 +178,7 @@ function test_with_stream_msg_cs { run_test "" 10950 # Trim end after second event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -189,7 +190,7 @@ function test_with_stream_msg_cs { run_test "" 10450 # Trim end after first event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -200,7 +201,7 @@ function test_with_stream_msg_cs { run_test "" 10350 # Trim end after packet beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [250 10,250,000,000,000] {0 0 0} Packet end @@ -210,7 +211,7 @@ function test_with_stream_msg_cs { run_test "" 10250 # Trim end after stream beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [100 10,100,000,000,000] {0 0 0} Stream beginning [150 10,150,000,000,000] {0 0 0} Stream end END @@ -218,13 +219,13 @@ function test_with_stream_msg_cs { run_test "" 10150 # Trim end before everything - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" END run_test "" 10050 # Trim end at a time before what the clock class can represent - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" END run_test "" 50 @@ -234,7 +235,7 @@ function test_without_stream_msg_cs { with_stream_msgs_cs="false" # Baseline (without trimming) - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -245,7 +246,7 @@ function test_without_stream_msg_cs { run_test "" "" # Trim begin at a time before what the clock class can represent - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -256,7 +257,7 @@ function test_without_stream_msg_cs { run_test 50 "" # Trim begin before stream beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -267,7 +268,7 @@ function test_without_stream_msg_cs { run_test 10050 "" # Trim begin before packet beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -278,7 +279,7 @@ function test_without_stream_msg_cs { run_test 10150 "" # Trim begin before first event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [250 10,250,000,000,000] {0 0 0} Stream beginning [250 10,250,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -289,7 +290,7 @@ function test_without_stream_msg_cs { run_test 10250 "" # Trim begin before second event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [350 10,350,000,000,000] {0 0 0} Stream beginning [350 10,350,000,000,000] {0 0 0} Packet beginning [400 10,400,000,000,000] {0 0 0} Event `event 2` (1) @@ -300,7 +301,7 @@ function test_without_stream_msg_cs { run_test 10350 "" # Trim begin before packet end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [850 10,850,000,000,000] {0 0 0} Stream beginning [850 10,850,000,000,000] {0 0 0} Packet beginning [900 10,900,000,000,000] {0 0 0} Packet end @@ -310,13 +311,13 @@ function test_without_stream_msg_cs { run_test 10850 "" # Trim begin after everything - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" END run_test 11050 "" # Trim end after stream end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -328,7 +329,7 @@ function test_without_stream_msg_cs { run_test "" 11050 # Trim end after packet end - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -340,7 +341,7 @@ function test_without_stream_msg_cs { run_test "" 10950 # Trim end after second event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -352,7 +353,7 @@ function test_without_stream_msg_cs { run_test "" 10450 # Trim end after first event - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [300 10,300,000,000,000] {0 0 0} Event `event 1` (0) @@ -363,7 +364,7 @@ function test_without_stream_msg_cs { run_test "" 10350 # Trim end after packet beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [200 10,200,000,000,000] {0 0 0} Packet beginning [250 10,250,000,000,000] {0 0 0} Packet end @@ -373,7 +374,7 @@ function test_without_stream_msg_cs { run_test "" 10250 # Trim end after stream beginning - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [Unknown] {0 0 0} Stream end END @@ -381,7 +382,7 @@ function test_without_stream_msg_cs { run_test "" 10150 # Trim end before everything - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [Unknown] {0 0 0} Stream end END @@ -389,7 +390,7 @@ function test_without_stream_msg_cs { run_test "" 10050 # Trim end at a time before what the clock class can represent - cat <<- 'END' > "$temp_expected" + cat <<- 'END' > "$temp_stdout_expected" [Unknown] {0 0 0} Stream beginning [Unknown] {0 0 0} Stream end END @@ -400,4 +401,6 @@ function test_without_stream_msg_cs { test_with_stream_msg_cs test_without_stream_msg_cs -rm -f "$temp_expected" +# Do not `rm` $temp_stderr_expected because it's set to `/dev/null` right now +# and that would print an error. +rm -f "$temp_stdout_expected" diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh index 6463164d..27a5f3ee 100644 --- a/tests/utils/utils.sh +++ b/tests/utils/utils.sh @@ -134,37 +134,59 @@ BT_DEBUG_INFO_PATH="${BT_TESTS_DATADIR}/debug-info" ### Diff Functions ### -# Checks the difference between the content of the file with path "$1" -# and the output of the CLI when called with the rest of arguments -# to this function. +# Checks the difference between: +# +# 1. What the CLI outputs on its standard output when given the arguments +# "$@" (excluding the first two arguments). +# 2. The file with path "$1". +# +# And the difference between: +# +# 1. What the CLI outputs on its standard error when given the arguments +# "$@" (excluding the first two arguments). +# 2. The file with path "$2". # # Returns 0 if there's no difference, and 1 if there is, also printing # said difference to the standard error. bt_diff_cli() { - local expected_file="$1" - shift 1 + local expected_stdout_file="$1" + local expected_stderr_file="$2" + shift 2 local args=("$@") - local temp_output_file + local temp_stdout_output_file + local temp_stderr_output_file local temp_diff local ret=0 - temp_output_file="$(mktemp)" + temp_stdout_output_file="$(mktemp)" + temp_stderr_output_file="$(mktemp)" temp_diff="$(mktemp)" - # Run the CLI to get a detailed file. Strip any \r present due to - # Windows (\n -> \r\n). "diff --string-trailing-cr" is not used since it - # is not present on Solaris. - run_python_bt2 "$BT_TESTS_BT2_BIN" "${args[@]}" | tr -d "\r" > "$temp_output_file" + # Run the CLI to get a detailed file. + run_python_bt2 "$BT_TESTS_BT2_BIN" "${args[@]}" 1>"$temp_stdout_output_file" 2>"$temp_stderr_output_file" + + # Strip any \r present due to Windows (\n -> \r\n). + # "diff --string-trailing-cr" is not used since it is not present on + # Solaris. + "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stdout_output_file" + "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stderr_output_file" + + # Compare stdout output with expected stdout output + if ! diff -u "$temp_stdout_output_file" "$expected_stdout_file" 2>/dev/null >"$temp_diff"; then + echo "ERROR: for '${args[*]}': actual standard output and expected output differ:" >&2 + cat "$temp_diff" >&2 + ret=1 + fi - # Compare output with expected output - if ! diff -u "$temp_output_file" "$expected_file" 2>/dev/null >"$temp_diff"; then - echo "ERROR: for '${args[*]}': actual and expected outputs differ:" >&2 + # Compare stderr output with expected stderr output + if ! diff -u "$temp_stderr_output_file" "$expected_stderr_file" 2>/dev/null >"$temp_diff"; then + echo "ERROR: for '${args[*]}': actual standard error and expected error differ:" >&2 cat "$temp_diff" >&2 ret=1 fi - rm -f "$temp_output_file" "$temp_diff" + rm -f "$temp_stdout_output_file" "$temp_stderr_output_file" "$temp_diff" return $ret } @@ -177,13 +199,14 @@ bt_diff_cli() { # Returns 0 if there's no difference, and 1 if there is, also printing # said difference to the standard error. bt_diff_details_ctf_single() { - local expected_file="$1" + local expected_stdout_file="$1" local trace_dir="$2" shift 2 local extra_details_args=("$@") + expected_stderr_file="/dev/null" # Compare using the CLI with `sink.text.details` - bt_diff_cli "$expected_file" "$trace_dir" "-c" "sink.text.details" "${extra_details_args[@]}" + bt_diff_cli "$expected_stdout_file" "$expected_stderr_file" "$trace_dir" "-c" "sink.text.details" "${extra_details_args[@]}" } # Calls bt_diff_details_ctf_single(), except that "$1" is the path to a @@ -192,7 +215,7 @@ bt_diff_details_ctf_single() { # CTF trace as its first argument. bt_diff_details_ctf_gen_single() { local ctf_gen_prog_path="$1" - local expected_file="$2" + local expected_stdout_file="$2" shift 2 local extra_details_args=("$@") @@ -209,7 +232,7 @@ bt_diff_details_ctf_gen_single() { fi # Compare using the CLI with `sink.text.details` - bt_diff_details_ctf_single "$expected_file" "$temp_trace_dir" "${extra_details_args[@]}" + bt_diff_details_ctf_single "$expected_stdout_file" "$temp_trace_dir" "${extra_details_args[@]}" ret=$? rm -rf "$temp_trace_dir" return $ret -- 2.34.1