tests: bt_diff_cli: compare expected `stderr` too

Useful to confirm that a trace is read without warnings but could also
be used to confirm that an expected warning is indeed present.

Signed-off-by: Francis Deslauriers <francis.deslauriers@efficios.com>
Change-Id: I0c48c68a0aac832729da2d9c5b849fb58ec578e4
Reviewed-on: https://review.lttng.org/c/babeltrace/+/1638
Tested-by: jenkins <jenkins@lttng.org>
Reviewed-by: Philippe Proulx <eeppeliteloop@gmail.com>

diff --git a/tests/plugins/flt.utils.trimmer/test_trimming b/tests/plugins/flt.utils.trimmer/test_trimming
index 3ec05a4115ecbb976201259c56c39a3fa45970ba..5f84bf1b7fa38dead45223d167c25a636dda2dea 100755 (executable)
--- a/tests/plugins/flt.utils.trimmer/test_trimming
+++ b/tests/plugins/flt.utils.trimmer/test_trimming
@@ -31,7 +31,8 @@ fi
 source "$UTILSSH"
 
 data_dir="$BT_TESTS_DATADIR/plugins/flt.utils.trimmer"
-temp_expected=$(mktemp)
+temp_stdout_expected=$(mktemp)
+temp_stderr_expected="/dev/null"
 
 plan_tests 32
 
@@ -63,7 +64,7 @@ function run_test
 
        # with_stream_msgs_cs is set to "true" or "false" by the tests.
 
-       bt_diff_cli "$temp_expected" "${local_args[@]}"
+       bt_diff_cli "$temp_stdout_expected" "$temp_stderr_expected" "${local_args[@]}"
        ok $? "$test_name"
 }
 
@@ -71,7 +72,7 @@ function test_with_stream_msg_cs {
        with_stream_msgs_cs="true"
 
        # Baseline (without trimming)
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -82,7 +83,7 @@ function test_with_stream_msg_cs {
        run_test "" ""
 
        # Trim begin at a time before what the clock class can represent
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -93,7 +94,7 @@ function test_with_stream_msg_cs {
        run_test 50 ""
 
        # Trim begin before stream beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -104,7 +105,7 @@ function test_with_stream_msg_cs {
        run_test 10050 ""
 
        # Trim begin before packet beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [150 10,150,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -115,7 +116,7 @@ function test_with_stream_msg_cs {
        run_test 10150 ""
 
        # Trim begin before first event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [250 10,250,000,000,000] {0 0 0} Stream beginning
        [250 10,250,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -126,7 +127,7 @@ function test_with_stream_msg_cs {
        run_test 10250 ""
 
        # Trim begin before second event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [350 10,350,000,000,000] {0 0 0} Stream beginning
        [350 10,350,000,000,000] {0 0 0} Packet beginning
        [400 10,400,000,000,000] {0 0 0} Event `event 2` (1)
@@ -137,7 +138,7 @@ function test_with_stream_msg_cs {
        run_test 10350 ""
 
        # Trim begin before packet end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [850 10,850,000,000,000] {0 0 0} Stream beginning
        [850 10,850,000,000,000] {0 0 0} Packet beginning
        [900 10,900,000,000,000] {0 0 0} Packet end
@@ -147,13 +148,13 @@ function test_with_stream_msg_cs {
        run_test 10850 ""
 
        # Trim begin after everything
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        END
 
        run_test 11050 ""
 
        # Trim end after stream end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -165,7 +166,7 @@ function test_with_stream_msg_cs {
        run_test "" 11050
 
        # Trim end after packet end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -177,7 +178,7 @@ function test_with_stream_msg_cs {
        run_test "" 10950
 
        # Trim end after second event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -189,7 +190,7 @@ function test_with_stream_msg_cs {
        run_test "" 10450
 
        # Trim end after first event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -200,7 +201,7 @@ function test_with_stream_msg_cs {
        run_test "" 10350
 
        # Trim end after packet beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [250 10,250,000,000,000] {0 0 0} Packet end
@@ -210,7 +211,7 @@ function test_with_stream_msg_cs {
        run_test "" 10250
 
        # Trim end after stream beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [100 10,100,000,000,000] {0 0 0} Stream beginning
        [150 10,150,000,000,000] {0 0 0} Stream end
        END
@@ -218,13 +219,13 @@ function test_with_stream_msg_cs {
        run_test "" 10150
 
        # Trim end before everything
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        END
 
        run_test "" 10050
 
        # Trim end at a time before what the clock class can represent
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        END
 
        run_test "" 50
@@ -234,7 +235,7 @@ function test_without_stream_msg_cs {
        with_stream_msgs_cs="false"
 
        # Baseline (without trimming)
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -245,7 +246,7 @@ function test_without_stream_msg_cs {
        run_test "" ""
 
        # Trim begin at a time before what the clock class can represent
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -256,7 +257,7 @@ function test_without_stream_msg_cs {
        run_test 50 ""
 
        # Trim begin before stream beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -267,7 +268,7 @@ function test_without_stream_msg_cs {
        run_test 10050 ""
 
        # Trim begin before packet beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -278,7 +279,7 @@ function test_without_stream_msg_cs {
        run_test 10150 ""
 
        # Trim begin before first event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [250 10,250,000,000,000] {0 0 0} Stream beginning
        [250 10,250,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -289,7 +290,7 @@ function test_without_stream_msg_cs {
        run_test 10250 ""
 
        # Trim begin before second event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [350 10,350,000,000,000] {0 0 0} Stream beginning
        [350 10,350,000,000,000] {0 0 0} Packet beginning
        [400 10,400,000,000,000] {0 0 0} Event `event 2` (1)
@@ -300,7 +301,7 @@ function test_without_stream_msg_cs {
        run_test 10350 ""
 
        # Trim begin before packet end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [850 10,850,000,000,000] {0 0 0} Stream beginning
        [850 10,850,000,000,000] {0 0 0} Packet beginning
        [900 10,900,000,000,000] {0 0 0} Packet end
@@ -310,13 +311,13 @@ function test_without_stream_msg_cs {
        run_test 10850 ""
 
        # Trim begin after everything
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        END
 
        run_test 11050 ""
 
        # Trim end after stream end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -328,7 +329,7 @@ function test_without_stream_msg_cs {
        run_test "" 11050
 
        # Trim end after packet end
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -340,7 +341,7 @@ function test_without_stream_msg_cs {
        run_test "" 10950
 
        # Trim end after second event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -352,7 +353,7 @@ function test_without_stream_msg_cs {
        run_test "" 10450
 
        # Trim end after first event
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [300 10,300,000,000,000] {0 0 0} Event `event 1` (0)
@@ -363,7 +364,7 @@ function test_without_stream_msg_cs {
        run_test "" 10350
 
        # Trim end after packet beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [200 10,200,000,000,000] {0 0 0} Packet beginning
        [250 10,250,000,000,000] {0 0 0} Packet end
@@ -373,7 +374,7 @@ function test_without_stream_msg_cs {
        run_test "" 10250
 
        # Trim end after stream beginning
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [Unknown] {0 0 0} Stream end
        END
@@ -381,7 +382,7 @@ function test_without_stream_msg_cs {
        run_test "" 10150
 
        # Trim end before everything
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [Unknown] {0 0 0} Stream end
        END
@@ -389,7 +390,7 @@ function test_without_stream_msg_cs {
        run_test "" 10050
 
        # Trim end at a time before what the clock class can represent
-       cat <<- 'END' > "$temp_expected"
+       cat <<- 'END' > "$temp_stdout_expected"
        [Unknown] {0 0 0} Stream beginning
        [Unknown] {0 0 0} Stream end
        END
@@ -400,4 +401,6 @@ function test_without_stream_msg_cs {
 test_with_stream_msg_cs
 test_without_stream_msg_cs
 
-rm -f "$temp_expected"
+# Do not `rm` $temp_stderr_expected because it's set to `/dev/null` right now
+# and that would print an error.
+rm -f "$temp_stdout_expected"

diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh
index 6463164d1730424bbd05d8021a12240407f48c50..27a5f3ee903e4fd85d165a151fcf3955ab58383c 100644 (file)
--- a/tests/utils/utils.sh
+++ b/tests/utils/utils.sh
@@ -134,37 +134,59 @@ BT_DEBUG_INFO_PATH="${BT_TESTS_DATADIR}/debug-info"
 
 ### Diff Functions ###
 
-# Checks the difference between the content of the file with path "$1"
-# and the output of the CLI when called with the rest of arguments
-# to this function.
+# Checks the difference between:
+#
+#   1. What the CLI outputs on its standard output when given the arguments
+#   "$@" (excluding the first two arguments).
+#   2. The file with path "$1".
+#
+# And the difference between:
+#
+#   1. What the CLI outputs on its standard error when given the arguments
+#   "$@" (excluding the first two arguments).
+#   2. The file with path "$2".
 #
 # Returns 0 if there's no difference, and 1 if there is, also printing
 # said difference to the standard error.
 bt_diff_cli() {
-       local expected_file="$1"
-       shift 1
+       local expected_stdout_file="$1"
+       local expected_stderr_file="$2"
+       shift 2
        local args=("$@")
 
-       local temp_output_file
+       local temp_stdout_output_file
+       local temp_stderr_output_file
        local temp_diff
        local ret=0
 
-       temp_output_file="$(mktemp)"
+       temp_stdout_output_file="$(mktemp)"
+       temp_stderr_output_file="$(mktemp)"
        temp_diff="$(mktemp)"
 
-       # Run the CLI to get a detailed file. Strip any \r present due to
-       # Windows (\n -> \r\n). "diff --string-trailing-cr" is not used since it
-       # is not present on Solaris.
-       run_python_bt2 "$BT_TESTS_BT2_BIN" "${args[@]}" | tr -d "\r" > "$temp_output_file"
+       # Run the CLI to get a detailed file.
+       run_python_bt2 "$BT_TESTS_BT2_BIN" "${args[@]}" 1>"$temp_stdout_output_file" 2>"$temp_stderr_output_file"
+
+       # Strip any \r present due to Windows (\n -> \r\n).
+       # "diff --string-trailing-cr" is not used since it is not present on
+       # Solaris.
+       "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stdout_output_file"
+       "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stderr_output_file"
+
+       # Compare stdout output with expected stdout output
+       if ! diff -u "$temp_stdout_output_file" "$expected_stdout_file" 2>/dev/null >"$temp_diff"; then
+               echo "ERROR: for '${args[*]}': actual standard output and expected output differ:" >&2
+               cat "$temp_diff" >&2
+               ret=1
+       fi
 
-       # Compare output with expected output
-       if ! diff -u "$temp_output_file" "$expected_file" 2>/dev/null >"$temp_diff"; then
-               echo "ERROR: for '${args[*]}': actual and expected outputs differ:" >&2
+       # Compare stderr output with expected stderr output
+       if ! diff -u "$temp_stderr_output_file" "$expected_stderr_file" 2>/dev/null >"$temp_diff"; then
+               echo "ERROR: for '${args[*]}': actual standard error and expected error differ:" >&2
                cat "$temp_diff" >&2
                ret=1
        fi
 
-       rm -f "$temp_output_file" "$temp_diff"
+       rm -f "$temp_stdout_output_file" "$temp_stderr_output_file" "$temp_diff"
 
        return $ret
 }
@@ -177,13 +199,14 @@ bt_diff_cli() {
 # Returns 0 if there's no difference, and 1 if there is, also printing
 # said difference to the standard error.
 bt_diff_details_ctf_single() {
-       local expected_file="$1"
+       local expected_stdout_file="$1"
        local trace_dir="$2"
        shift 2
        local extra_details_args=("$@")
+       expected_stderr_file="/dev/null"
 
        # Compare using the CLI with `sink.text.details`
-       bt_diff_cli "$expected_file" "$trace_dir" "-c" "sink.text.details" "${extra_details_args[@]}"
+       bt_diff_cli "$expected_stdout_file" "$expected_stderr_file" "$trace_dir" "-c" "sink.text.details" "${extra_details_args[@]}"
 }
 
 # Calls bt_diff_details_ctf_single(), except that "$1" is the path to a
@@ -192,7 +215,7 @@ bt_diff_details_ctf_single() {
 # CTF trace as its first argument.
 bt_diff_details_ctf_gen_single() {
        local ctf_gen_prog_path="$1"
-       local expected_file="$2"
+       local expected_stdout_file="$2"
        shift 2
        local extra_details_args=("$@")
 
@@ -209,7 +232,7 @@ bt_diff_details_ctf_gen_single() {
        fi
 
        # Compare using the CLI with `sink.text.details`
-       bt_diff_details_ctf_single "$expected_file" "$temp_trace_dir" "${extra_details_args[@]}"
+       bt_diff_details_ctf_single "$expected_stdout_file" "$temp_trace_dir" "${extra_details_args[@]}"
        ret=$?
        rm -rf "$temp_trace_dir"
        return $ret