tests: bt_diff_cli: compare expected `stderr` too
[babeltrace.git] / tests / utils / utils.sh
CommitLineData
644e0364
MJ
1#!/bin/bash
2
3# Copyright (c) 2019 Michael Jeanson <mjeanson@efficios.com>
4# Copyright (C) 2019 Philippe Proulx <pproulx@efficios.com>
5#
6# This program is free software; you can redistribute it and/or modify
7# it under the terms of the GNU General Public License as published by
8# the Free Software Foundation; under version 2 of the License.
9#
10# This program is distributed in the hope that it will be useful,
11# but WITHOUT ANY WARRANTY; without even the implied warranty of
12# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13# GNU General Public License for more details.
14#
15# You should have received a copy of the GNU General Public License along
16# with this program; if not, write to the Free Software Foundation, Inc.,
17# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
18
19# This file is meant to be sourced at the start of shell script-based tests.
20
21
22# Error out when encountering an undefined variable
23set -u
24
25scriptdir="$(dirname "${BASH_SOURCE[0]}")"
26
5058d31b
JR
27# The OS on which we are running. See [1] for possible values of 'uname -s'.
28# We do a bit of translation to ease our life down the road for comparison.
29# Export it so that called executables can use it.
30# [1] https://en.wikipedia.org/wiki/Uname#Examples
31if [ "x${BT_OS_TYPE:-}" = "x" ]; then
32 BT_OS_TYPE="$(uname -s)"
33 case "$BT_OS_TYPE" in
34 MINGW*)
35 BT_OS_TYPE="mingw"
36 ;;
37 Darwin)
38 BT_OS_TYPE="darwin"
39 ;;
40 Linux)
41 BT_OS_TYPE="linux"
42 ;;
df9db467
JR
43 CYGWIN*)
44 BT_OS_TYPE="cygwin"
45 ;;
5058d31b
JR
46 *)
47 BT_OS_TYPE="unsupported"
48 ;;
49 esac
50fi
51export BT_OS_TYPE
52
644e0364
MJ
53# Allow overriding the source and build directories
54if [ "x${BT_TESTS_SRCDIR:-}" = "x" ]; then
4881a20e 55 BT_TESTS_SRCDIR="$scriptdir/.."
644e0364
MJ
56fi
57export BT_TESTS_SRCDIR
58
59if [ "x${BT_TESTS_BUILDDIR:-}" = "x" ]; then
4881a20e 60 BT_TESTS_BUILDDIR="$scriptdir/.."
644e0364
MJ
61fi
62export BT_TESTS_BUILDDIR
63
64# By default, it will not source tap.sh. If you to tap output directly from
65# the test script, define the 'SH_TAP' variable to '1' before sourcing this
66# script.
67if [ "x${SH_TAP:-}" = x1 ]; then
1d970e04 68 # shellcheck source=./tap/tap.sh
644e0364
MJ
69 . "${BT_TESTS_SRCDIR}/utils/tap/tap.sh"
70fi
71
72# Allow overriding the babeltrace2 executables
73if [ "x${BT_TESTS_BT2_BIN:-}" = "x" ]; then
4881a20e 74 BT_TESTS_BT2_BIN="$BT_TESTS_BUILDDIR/../src/cli/babeltrace2"
5058d31b 75 if [ "$BT_OS_TYPE" = "mingw" ]; then
27e2c58d
JR
76 BT_TESTS_BT2_BIN="${BT_TESTS_BT2_BIN}.exe"
77 fi
644e0364
MJ
78fi
79export BT_TESTS_BT2_BIN
80
81if [ "x${BT_TESTS_BT2LOG_BIN:-}" = "x" ]; then
4881a20e 82 BT_TESTS_BT2LOG_BIN="$BT_TESTS_BUILDDIR/../src/cli/babeltrace2-log"
5058d31b 83 if [ "$BT_OS_TYPE" = "mingw" ]; then
27e2c58d
JR
84 BT_TESTS_BT2LOG_BIN="${BT_TESTS_BT2LOG_BIN}.exe"
85 fi
644e0364
MJ
86fi
87export BT_TESTS_BT2LOG_BIN
88
89# TODO: Remove when bindings/python/bt2/test_plugin.py is fixed
90BT_PLUGINS_PATH="${BT_TESTS_BUILDDIR}/../src/plugins"
91
92# Allow overriding the babeltrace2 plugin path
93if [ "x${BT_TESTS_BABELTRACE_PLUGIN_PATH:-}" = "x" ]; then
94 BT_TESTS_BABELTRACE_PLUGIN_PATH="${BT_PLUGINS_PATH}/ctf:${BT_PLUGINS_PATH}/utils:${BT_PLUGINS_PATH}/text"
95fi
96
b14c7bf1
MJ
97if [ "x${BT_TESTS_PROVIDER_DIR:-}" = "x" ]; then
98 BT_TESTS_PROVIDER_DIR="${BT_TESTS_BUILDDIR}/../src/python-plugin-provider/.libs"
99fi
100
644e0364
MJ
101# Allow overriding the babeltrace2 executables
102if [ "x${BT_TESTS_PYTHONPATH:-}" = "x" ]; then
103 BT_TESTS_PYTHONPATH="${BT_TESTS_BUILDDIR}/../src/bindings/python/bt2/build/build_lib"
104fi
105
106
107### External Tools ###
108if [ "x${BT_TESTS_AWK_BIN:-}" = "x" ]; then
109 BT_TESTS_AWK_BIN="awk"
110fi
111export BT_TESTS_AWK_BIN
112
113if [ "x${BT_TESTS_GREP_BIN:-}" = "x" ]; then
114 BT_TESTS_GREP_BIN="grep"
115fi
116export BT_TESTS_GREP_BIN
117
118if [ "x${BT_TESTS_PYTHON_BIN:-}" = "x" ]; then
119 BT_TESTS_PYTHON_BIN="python3"
120fi
121export BT_TESTS_PYTHON_BIN
122
123if [ "x${BT_TESTS_SED_BIN:-}" = "x" ]; then
124 BT_TESTS_SED_BIN="sed"
125fi
126export BT_TESTS_SED_BIN
127
128
129# Data files path
130BT_TESTS_DATADIR="${BT_TESTS_SRCDIR}/data"
131BT_CTF_TRACES_PATH="${BT_TESTS_DATADIR}/ctf-traces"
132BT_DEBUG_INFO_PATH="${BT_TESTS_DATADIR}/debug-info"
133
134
135### Diff Functions ###
136
58db335e
FD
137# Checks the difference between:
138#
139# 1. What the CLI outputs on its standard output when given the arguments
140# "$@" (excluding the first two arguments).
141# 2. The file with path "$1".
142#
143# And the difference between:
144#
145# 1. What the CLI outputs on its standard error when given the arguments
146# "$@" (excluding the first two arguments).
147# 2. The file with path "$2".
644e0364
MJ
148#
149# Returns 0 if there's no difference, and 1 if there is, also printing
150# said difference to the standard error.
151bt_diff_cli() {
58db335e
FD
152 local expected_stdout_file="$1"
153 local expected_stderr_file="$2"
154 shift 2
53cc240b
MJ
155 local args=("$@")
156
58db335e
FD
157 local temp_stdout_output_file
158 local temp_stderr_output_file
644e0364
MJ
159 local temp_diff
160 local ret=0
161
58db335e
FD
162 temp_stdout_output_file="$(mktemp)"
163 temp_stderr_output_file="$(mktemp)"
644e0364
MJ
164 temp_diff="$(mktemp)"
165
58db335e
FD
166 # Run the CLI to get a detailed file.
167 run_python_bt2 "$BT_TESTS_BT2_BIN" "${args[@]}" 1>"$temp_stdout_output_file" 2>"$temp_stderr_output_file"
168
169 # Strip any \r present due to Windows (\n -> \r\n).
170 # "diff --string-trailing-cr" is not used since it is not present on
171 # Solaris.
172 "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stdout_output_file"
173 "$BT_TESTS_SED_BIN" -i 's/\r//g' "$temp_stderr_output_file"
174
175 # Compare stdout output with expected stdout output
176 if ! diff -u "$temp_stdout_output_file" "$expected_stdout_file" 2>/dev/null >"$temp_diff"; then
177 echo "ERROR: for '${args[*]}': actual standard output and expected output differ:" >&2
178 cat "$temp_diff" >&2
179 ret=1
180 fi
644e0364 181
58db335e
FD
182 # Compare stderr output with expected stderr output
183 if ! diff -u "$temp_stderr_output_file" "$expected_stderr_file" 2>/dev/null >"$temp_diff"; then
184 echo "ERROR: for '${args[*]}': actual standard error and expected error differ:" >&2
644e0364
MJ
185 cat "$temp_diff" >&2
186 ret=1
187 fi
188
58db335e 189 rm -f "$temp_stdout_output_file" "$temp_stderr_output_file" "$temp_diff"
644e0364
MJ
190
191 return $ret
192}
193
53cc240b
MJ
194# Checks the difference between the content of the file with path "$1"
195# and the output of the CLI when called on the directory path "$2" with
196# the arguments '-c sink.text.details' and the rest of the arguments to
197# this function.
644e0364
MJ
198#
199# Returns 0 if there's no difference, and 1 if there is, also printing
200# said difference to the standard error.
201bt_diff_details_ctf_single() {
58db335e 202 local expected_stdout_file="$1"
53cc240b
MJ
203 local trace_dir="$2"
204 shift 2
205 local extra_details_args=("$@")
58db335e 206 expected_stderr_file="/dev/null"
644e0364
MJ
207
208 # Compare using the CLI with `sink.text.details`
58db335e 209 bt_diff_cli "$expected_stdout_file" "$expected_stderr_file" "$trace_dir" "-c" "sink.text.details" "${extra_details_args[@]}"
644e0364
MJ
210}
211
212# Calls bt_diff_details_ctf_single(), except that "$1" is the path to a
213# program which generates the CTF trace to compare to. The program "$1"
214# receives the path to a temporary, empty directory where to write the
215# CTF trace as its first argument.
216bt_diff_details_ctf_gen_single() {
217 local ctf_gen_prog_path="$1"
58db335e 218 local expected_stdout_file="$2"
53cc240b
MJ
219 shift 2
220 local extra_details_args=("$@")
644e0364
MJ
221
222 local temp_trace_dir
223 local ret
224
225 temp_trace_dir="$(mktemp -d)"
226
227 # Run the CTF trace generator program to get a CTF trace
228 if ! "$ctf_gen_prog_path" "$temp_trace_dir" 2>/dev/null; then
229 echo "ERROR: \"$ctf_gen_prog_path\" \"$temp_trace_dir\" failed" >&2
230 rm -rf "$temp_trace_dir"
231 return 1
232 fi
233
234 # Compare using the CLI with `sink.text.details`
58db335e 235 bt_diff_details_ctf_single "$expected_stdout_file" "$temp_trace_dir" "${extra_details_args[@]}"
644e0364
MJ
236 ret=$?
237 rm -rf "$temp_trace_dir"
238 return $ret
239}
240
241
242### Functions ###
243
244check_coverage() {
245 coverage run "$@"
246}
247
248# Execute a shell command in the appropriate environment to have access to the
249# bt2 Python bindings.
250run_python_bt2() {
251 local lib_search_var
252 local lib_search_path
253
0408934a
SM
254 local main_lib_path="${BT_TESTS_BUILDDIR}/../src/lib/.libs"
255
644e0364 256 # Set the library search path so the python interpreter can load libbabeltrace2
df9db467 257 if [ "$BT_OS_TYPE" = "mingw" ] || [ "$BT_OS_TYPE" = "cygwin" ]; then
644e0364 258 lib_search_var="PATH"
b14c7bf1 259 lib_search_path="${main_lib_path}:${PATH:-}"
5058d31b
JR
260 elif [ "$BT_OS_TYPE" = "darwin" ]; then
261 lib_search_var="DYLD_LIBRARY_PATH"
b14c7bf1 262 lib_search_path="${main_lib_path}:${DYLD_LIBRARY_PATH:-}"
644e0364
MJ
263 else
264 lib_search_var="LD_LIBRARY_PATH"
b14c7bf1 265 lib_search_path="${main_lib_path}:${LD_LIBRARY_PATH:-}"
644e0364
MJ
266 fi
267
268 env \
269 BABELTRACE_PYTHON_BT2_NO_TRACEBACK=1 \
270 BABELTRACE_PLUGIN_PATH="${BT_TESTS_BABELTRACE_PLUGIN_PATH}" \
b14c7bf1 271 LIBBABELTRACE2_PLUGIN_PROVIDER_DIR=${BT_TESTS_PROVIDER_DIR} \
644e0364
MJ
272 BT_CTF_TRACES_PATH="${BT_CTF_TRACES_PATH}" \
273 BT_PLUGINS_PATH="${BT_PLUGINS_PATH}" \
274 PYTHONPATH="${BT_TESTS_PYTHONPATH}:${BT_TESTS_SRCDIR}/utils/python" \
275 "${lib_search_var}"="${lib_search_path}" \
276 "$@"
277}
278
279# Set the environment and run python tests in the directory.
280#
281# $1 : The directory containing the python test scripts
282# $2 : The pattern to match python test script names (optional)
644e0364
MJ
283run_python_bt2_test() {
284 local test_dir="$1"
a818a617 285 local test_pattern="${2:-'*'}" # optional, if none default to "*"
644e0364
MJ
286
287 local ret
288 local test_runner_args=()
289
290 test_runner_args+=("$test_dir")
291 if [ "x${test_pattern}" != "x" ]; then
292 test_runner_args+=("${test_pattern}")
293 fi
294
295 if test "x${BT_TESTS_COVERAGE:-}" = "x1"; then
296 python_exec="check_coverage"
297 else
298 python_exec="${BT_TESTS_PYTHON_BIN}"
299 fi
300
301 run_python_bt2 \
302 "${python_exec}" \
303 "${BT_TESTS_SRCDIR}/utils/python/testrunner.py" \
a818a617
FD
304 --pattern "$test_pattern" \
305 "$test_dir" \
306
644e0364
MJ
307 ret=$?
308
309 if test "x${BT_TESTS_COVERAGE_REPORT:-}" = "x1"; then
310 coverage report -m
311 fi
312
313 if test "x${BT_TESTS_COVERAGE_HTML:-}" = "x1"; then
314 coverage html
315 fi
316
317 return $ret
318}
This page took 0.039197 seconds and 4 git commands to generate.