From f3c9a159782f70dbd0e5dedb37e4a1ef8a6d304e Mon Sep 17 00:00:00 2001 From: Simon Marchi Date: Mon, 5 Aug 2019 14:45:38 -0400 Subject: [PATCH] bt2: add auto source discovery support to TraceCollectionMessageIterator This patch makes TraceCollectionMessageIterator capable of automatically detecting source components from given input strings, similar to how the command line interface does. A new type AutoSourceComponentSpec is introduced. The user can instantiate this type, passing a mandatory input string (equivalent to a non-option argument on the CLI). The user can then pass instances of AutoSourceComponentSpec in the source_component_specs list when creating a TraceCollectionMessageIterator. In TraceCollectionMessageIterator.__init_, we run the automatic source algorithm on all inputs passed through AutoSourceComponentSpec objects. We create ComponentSpec from the results of the auto discovery and append them to the source component spec list. The rest of the TraceCollectionMessageIterator then works as before. The user can attach params and log levels to AutoSourceComponentSpec inputs. The semantic for those is the same as for the CLI. There is also an `obj` parameter, allowing to pass an arbitrary Python object. The semantic for this one is the same as the log level: if an auto-discovered component comes from multiple inputs, it will receive the obj of the last input that contributed to getting it instantiated that has a non-None `obj`. For convenience, some shorthands when creating a TraceCollectionMessageIterator are equivalent to using an AutoSourceComponentSpec. For example, these three forms are equivalent: - TraceCollectionMessageIterator('foo') - TraceCollectionMessageIterator(AutoSourceComponentSpec('foo')) - TraceCollectionMessageIterator([AutoSourceComponentSpec('foo')]) Implementation details ---------------------- New test cases are added to test the new feature. They are based on the corresponding CLI tests, in that they test equivalent scenarios (same inputs, expecting the same results). In order for TraceCollectionMessageIterator to find the test plugin (both bt_plugin_test.py files), the only way I found was to amend BABELTRACE_PLUGIN_PATH for the duration of the test. Change-Id: I0e1100d850c920723861609cfa14707f8c669892 Signed-off-by: Simon Marchi Reviewed-on: https://review.lttng.org/c/babeltrace/+/1826 Reviewed-by: Philippe Proulx Tested-by: jenkins --- src/bindings/python/bt2/Makefile.am | 3 + src/bindings/python/bt2/bt2/__init__.py | 1 + src/bindings/python/bt2/bt2/native_bt.i | 1 + .../python/bt2/bt2/native_bt_autodisc.i | 30 ++ .../python/bt2/bt2/native_bt_autodisc.i.h | 209 ++++++++++ .../bt2/trace_collection_message_iterator.py | 195 ++++++++-- src/bindings/python/bt2/setup.py.in | 1 + tests/bindings/python/bt2/test_package.py | 3 + .../test_trace_collection_message_iterator.py | 366 ++++++++++++++++++ .../params-log-level/bt_plugin_test.py | 12 +- tests/utils/utils.sh | 1 + 11 files changed, 795 insertions(+), 27 deletions(-) create mode 100644 src/bindings/python/bt2/bt2/native_bt_autodisc.i create mode 100644 src/bindings/python/bt2/bt2/native_bt_autodisc.i.h diff --git a/src/bindings/python/bt2/Makefile.am b/src/bindings/python/bt2/Makefile.am index 66696340..a0eb73a9 100644 --- a/src/bindings/python/bt2/Makefile.am +++ b/src/bindings/python/bt2/Makefile.am @@ -7,6 +7,8 @@ INSTALLED_FILES=$(builddir)/installed_files.txt SWIG_INTERFACE_FILES = \ bt2/native_bt.i \ + bt2/native_bt_autodisc.i \ + bt2/native_bt_autodisc.i.h \ bt2/native_bt_bt2_objects.h \ bt2/native_bt_clock_class.i \ bt2/native_bt_clock_snapshot.i \ @@ -85,6 +87,7 @@ STATIC_BINDINGS_DEPS = \ # Convenience static libraries on which the Python bindings library depends. # These are listed in the setup.py(.in) file. STATIC_LIBRARIES_DEPS = \ + $(top_builddir)/src/autodisc/libbabeltrace2-autodisc.la \ $(top_builddir)/src/logging/libbabeltrace2-logging.la \ $(top_builddir)/src/common/libbabeltrace2-common.la \ $(top_builddir)/src/py-common/libbabeltrace2-py-common.la diff --git a/src/bindings/python/bt2/bt2/__init__.py b/src/bindings/python/bt2/bt2/__init__.py index d6e3ab2b..0bead489 100644 --- a/src/bindings/python/bt2/bt2/__init__.py +++ b/src/bindings/python/bt2/bt2/__init__.py @@ -99,6 +99,7 @@ from bt2.plugin import find_plugin from bt2.py_plugin import plugin_component_class from bt2.py_plugin import register_plugin from bt2.query_executor import QueryExecutor +from bt2.trace_collection_message_iterator import AutoSourceComponentSpec from bt2.trace_collection_message_iterator import ComponentSpec from bt2.trace_collection_message_iterator import TraceCollectionMessageIterator from bt2.value import create_value diff --git a/src/bindings/python/bt2/bt2/native_bt.i b/src/bindings/python/bt2/bt2/native_bt.i index 6d9a3696..de9ff3c3 100644 --- a/src/bindings/python/bt2/bt2/native_bt.i +++ b/src/bindings/python/bt2/bt2/native_bt.i @@ -204,6 +204,7 @@ void bt_bt2_exit_handler(void); %include /* Per-module interface files */ +%include "native_bt_autodisc.i" %include "native_bt_clock_class.i" %include "native_bt_clock_snapshot.i" %include "native_bt_component.i" diff --git a/src/bindings/python/bt2/bt2/native_bt_autodisc.i b/src/bindings/python/bt2/bt2/native_bt_autodisc.i new file mode 100644 index 00000000..5c54cc39 --- /dev/null +++ b/src/bindings/python/bt2/bt2/native_bt_autodisc.i @@ -0,0 +1,30 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2017 Philippe Proulx + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +bt_value *bt_bt2_auto_discover_source_components(const bt_value *inputs, + const bt_plugin_set *plugin_set); + +%{ +#include "native_bt_autodisc.i.h" +%} diff --git a/src/bindings/python/bt2/bt2/native_bt_autodisc.i.h b/src/bindings/python/bt2/bt2/native_bt_autodisc.i.h new file mode 100644 index 00000000..e09f54e7 --- /dev/null +++ b/src/bindings/python/bt2/bt2/native_bt_autodisc.i.h @@ -0,0 +1,209 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2016 Philippe Proulx + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include + +/* + * Interface between the Python bindings and the auto source discovery system: + * input strings go in, specs for components to instantiate go out. + * + * `inputs` must be an array of strings, the list of inputs in which to look + * for traces. `plugin_set` is the set of plugins to query. + * + * Returns a map with the following entries: + * + * - status: signed integer, return status of this function + * - results: array, each element is an array describing one auto + * source discovery result (see `struct + * auto_source_discovery_result` for more details about these): + * + * - 0: plugin name, string + * - 1: class name, string + * - 2: inputs, array of strings + * - 3: original input indices, array of unsigned integers + * + * This function can also return None, if it failed to allocate memory + * for the return value and status code. + */ +bt_value *bt_bt2_auto_discover_source_components(const bt_value *inputs, + const bt_plugin_set *plugin_set) +{ + uint64_t i; + int status = 0; + static const char * const module_name = "Automatic source discovery"; + const bt_plugin **plugins = NULL; + const uint64_t plugin_count = bt_plugin_set_get_plugin_count(plugin_set); + struct auto_source_discovery auto_disc = { 0 }; + bt_value *result = NULL; + bt_value *components_list = NULL; + bt_value *component_info = NULL; + bt_value_map_insert_entry_status insert_entry_status; + + BT_ASSERT(bt_value_get_type(inputs) == BT_VALUE_TYPE_ARRAY); + for (i = 0; i < bt_value_array_get_size(inputs); i++) { + const bt_value *elem = bt_value_array_borrow_element_by_index_const(inputs, i); + BT_ASSERT(bt_value_get_type(elem) == BT_VALUE_TYPE_STRING); + } + + result = bt_value_map_create(); + if (!result) { + static const char * const err = "Failed to create a map value."; + BT_LOGE_STR(err); + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, err); + PyErr_NoMemory(); + goto end; + } + + status = auto_source_discovery_init(&auto_disc); + if (status != 0) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to initialize auto source discovery structure."); + goto error; + } + + /* Create array of bt_plugin pointers from plugin set. */ + plugins = g_new(const bt_plugin *, plugin_count); + if (!plugins) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to allocate plugins array."); + status = __BT_FUNC_STATUS_MEMORY_ERROR; + goto error; + } + + for (i = 0; i < plugin_count; i++) { + plugins[i] = bt_plugin_set_borrow_plugin_by_index_const(plugin_set, i); + } + + status = auto_discover_source_components( + inputs, + plugins, + plugin_count, + NULL, + bt_python_bindings_bt2_log_level, + &auto_disc); + if (status != 0) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to auto discover sources."); + goto error; + } + + components_list = bt_value_array_create(); + if (!components_list) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to allocate one array value."); + status = __BT_FUNC_STATUS_MEMORY_ERROR; + goto error; + } + + insert_entry_status = bt_value_map_insert_entry(result, "results", components_list); + if (insert_entry_status != BT_VALUE_MAP_INSERT_ENTRY_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to insert a map entry."); + status = (int) insert_entry_status; + goto error; + } + + for (i = 0; i < auto_disc.results->len; i++) { + struct auto_source_discovery_result *result = + g_ptr_array_index(auto_disc.results, i); + bt_value_array_append_element_status append_element_status; + + component_info = bt_value_array_create(); + if (!component_info) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to allocate one array value."); + status = __BT_FUNC_STATUS_MEMORY_ERROR; + goto error; + } + + append_element_status = bt_value_array_append_string_element( + component_info, result->plugin_name); + if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to append one array element."); + status = (int) append_element_status; + goto error; + } + + append_element_status = bt_value_array_append_string_element( + component_info, result->source_cc_name); + if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to append one array element."); + status = (int) append_element_status; + goto error; + } + + append_element_status = bt_value_array_append_element( + component_info, result->inputs); + if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to append one array element."); + status = (int) append_element_status; + goto error; + } + + append_element_status = bt_value_array_append_element( + component_info, result->original_input_indices); + if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to append one array element."); + status = (int) append_element_status; + goto error; + } + + append_element_status = bt_value_array_append_element(components_list, + component_info); + if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) { + BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, + "Failed to append one array element."); + status = (int) append_element_status; + goto error; + } + + BT_VALUE_PUT_REF_AND_RESET(component_info); + } + + status = 0; + goto end; +error: + BT_ASSERT(status != 0); + +end: + if (result) { + insert_entry_status = bt_value_map_insert_signed_integer_entry(result, "status", status); + if (insert_entry_status != BT_VALUE_MAP_INSERT_ENTRY_STATUS_OK) { + BT_VALUE_PUT_REF_AND_RESET(result); + PyErr_NoMemory(); + } + } + + auto_source_discovery_fini(&auto_disc); + g_free(plugins); + bt_value_put_ref(components_list); + bt_value_put_ref(component_info); + + return result; +} diff --git a/src/bindings/python/bt2/bt2/trace_collection_message_iterator.py b/src/bindings/python/bt2/bt2/trace_collection_message_iterator.py index 9c931fe2..56b48da9 100644 --- a/src/bindings/python/bt2/bt2/trace_collection_message_iterator.py +++ b/src/bindings/python/bt2/bt2/trace_collection_message_iterator.py @@ -20,13 +20,15 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. -from bt2 import utils +from bt2 import utils, native_bt import bt2 import itertools from bt2 import message_iterator as bt2_message_iterator from bt2 import logging as bt2_logging from bt2 import port as bt2_port from bt2 import component as bt2_component +from bt2 import value as bt2_value +from bt2 import plugin as bt2_plugin import datetime from collections import namedtuple import numbers @@ -36,7 +38,29 @@ import numbers _ComponentAndSpec = namedtuple('_ComponentAndSpec', ['comp', 'spec']) -class ComponentSpec: +class _BaseComponentSpec: + def __init__(self, params, obj, logging_level): + if logging_level is not None: + utils._check_log_level(logging_level) + + self._params = bt2.create_value(params) + self._obj = obj + self._logging_level = logging_level + + @property + def params(self): + return self._params + + @property + def obj(self): + return self._obj + + @property + def logging_level(self): + return self._logging_level + + +class ComponentSpec(_BaseComponentSpec): def __init__( self, plugin_name, @@ -45,18 +69,16 @@ class ComponentSpec: obj=None, logging_level=bt2_logging.LoggingLevel.NONE, ): + if type(params) is str: + params = {'inputs': [params]} + + super().__init__(params, obj, logging_level) + utils._check_str(plugin_name) utils._check_str(class_name) - utils._check_log_level(logging_level) + self._plugin_name = plugin_name self._class_name = class_name - self._logging_level = logging_level - self._obj = obj - - if type(params) is str: - self._params = bt2.create_value({'inputs': [params]}) - else: - self._params = bt2.create_value(params) @property def plugin_name(self): @@ -66,17 +88,102 @@ class ComponentSpec: def class_name(self): return self._class_name - @property - def logging_level(self): - return self._logging_level - @property - def params(self): - return self._params +class AutoSourceComponentSpec(_BaseComponentSpec): + _no_obj = object() + + def __init__(self, input, params=None, obj=_no_obj, logging_level=None): + super().__init__(params, obj, logging_level) + self._input = input @property - def obj(self): - return self._obj + def input(self): + return self._input + + +def _auto_discover_source_component_specs(auto_source_comp_specs, plugin_set): + # Transform a list of `AutoSourceComponentSpec` in a list of `ComponentSpec` + # using the automatic source discovery mechanism. + inputs = bt2.ArrayValue([spec.input for spec in auto_source_comp_specs]) + + if plugin_set is None: + plugin_set = bt2.find_plugins() + else: + utils._check_type(plugin_set, bt2_plugin._PluginSet) + + res_ptr = native_bt.bt2_auto_discover_source_components( + inputs._ptr, plugin_set._ptr + ) + + if res_ptr is None: + raise bt2._MemoryError('cannot auto discover source components') + + res = bt2_value._create_from_ptr(res_ptr) + + assert type(res) == bt2.MapValue + assert 'status' in res + + status = res['status'] + utils._handle_func_status(status, 'cannot auto-discover source components') + + comp_specs = [] + comp_specs_raw = res['results'] + assert type(comp_specs_raw) == bt2.ArrayValue + + for comp_spec_raw in comp_specs_raw: + assert type(comp_spec_raw) == bt2.ArrayValue + assert len(comp_spec_raw) == 4 + + plugin_name = comp_spec_raw[0] + assert type(plugin_name) == bt2.StringValue + plugin_name = str(plugin_name) + + class_name = comp_spec_raw[1] + assert type(class_name) == bt2.StringValue + class_name = str(class_name) + + comp_inputs = comp_spec_raw[2] + assert type(comp_inputs) == bt2.ArrayValue + + comp_orig_indices = comp_spec_raw[3] + assert type(comp_orig_indices) + + params = bt2.MapValue() + logging_level = bt2.LoggingLevel.NONE + obj = None + + # Compute `params` for this component by piling up params given to all + # AutoSourceComponentSpec objects that contributed in the instantiation + # of this component. + # + # The effective log level for a component is the last one specified + # across the AutoSourceComponentSpec that contributed in its + # instantiation. + for idx in comp_orig_indices: + orig_spec = auto_source_comp_specs[idx] + + if orig_spec.params is not None: + params.update(orig_spec.params) + + if orig_spec.logging_level is not None: + logging_level = orig_spec.logging_level + + if orig_spec.obj is not AutoSourceComponentSpec._no_obj: + obj = orig_spec.obj + + params['inputs'] = comp_inputs + + comp_specs.append( + ComponentSpec( + plugin_name, + class_name, + params=params, + obj=obj, + logging_level=logging_level, + ) + ) + + return comp_specs # datetime.datetime or integral to nanoseconds @@ -127,6 +234,7 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator): stream_intersection_mode=False, begin=None, end=None, + plugin_set=None, ): utils._check_bool(stream_intersection_mode) self._stream_intersection_mode = stream_intersection_mode @@ -134,15 +242,46 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator): self._end_ns = _get_ns(end) self._msg_list = [None] - if type(source_component_specs) is ComponentSpec: + # If a single item is provided, convert to a list. + if type(source_component_specs) in ( + ComponentSpec, + AutoSourceComponentSpec, + str, + ): source_component_specs = [source_component_specs] + # Convert any string to an AutoSourceComponentSpec. + def str_to_auto(item): + if type(item) is str: + item = AutoSourceComponentSpec(item) + + return item + + source_component_specs = [str_to_auto(s) for s in source_component_specs] + if type(filter_component_specs) is ComponentSpec: filter_component_specs = [filter_component_specs] elif filter_component_specs is None: filter_component_specs = [] - self._src_comp_specs = source_component_specs + self._validate_source_component_specs(source_component_specs) + self._validate_filter_component_specs(filter_component_specs) + + # Pass any `ComponentSpec` instance as-is. + self._src_comp_specs = [ + spec for spec in source_component_specs if type(spec) is ComponentSpec + ] + + # Convert any `AutoSourceComponentSpec` in concrete `ComponentSpec` instances. + auto_src_comp_specs = [ + spec + for spec in source_component_specs + if type(spec) is AutoSourceComponentSpec + ] + self._src_comp_specs += _auto_discover_source_component_specs( + auto_src_comp_specs, plugin_set + ) + self._flt_comp_specs = filter_component_specs self._next_suffix = 1 self._connect_ports = False @@ -151,11 +290,21 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator): self._src_comps_and_specs = [] self._flt_comps_and_specs = [] - self._validate_component_specs(source_component_specs) - self._validate_component_specs(filter_component_specs) self._build_graph() - def _validate_component_specs(self, comp_specs): + def _validate_source_component_specs(self, comp_specs): + for comp_spec in comp_specs: + if ( + type(comp_spec) is not ComponentSpec + and type(comp_spec) is not AutoSourceComponentSpec + ): + raise TypeError( + '"{}" object is not a ComponentSpec or AutoSourceComponentSpec'.format( + type(comp_spec) + ) + ) + + def _validate_filter_component_specs(self, comp_specs): for comp_spec in comp_specs: if type(comp_spec) is not ComponentSpec: raise TypeError( diff --git a/src/bindings/python/bt2/setup.py.in b/src/bindings/python/bt2/setup.py.in index 8172839c..0a94d1f3 100644 --- a/src/bindings/python/bt2/setup.py.in +++ b/src/bindings/python/bt2/setup.py.in @@ -41,6 +41,7 @@ def main(): sources=['bt2/native_bt.c', '@srcdir@/bt2/logging.c'], libraries=['babeltrace2', 'glib-2.0'], extra_objects=[ + '@top_builddir@/src/autodisc/.libs/libbabeltrace2-autodisc.a', '@top_builddir@/src/logging/.libs/libbabeltrace2-logging.a', '@top_builddir@/src/common/.libs/libbabeltrace2-common.a', '@top_builddir@/src/py-common/.libs/libbabeltrace2-py-common.a', diff --git a/tests/bindings/python/bt2/test_package.py b/tests/bindings/python/bt2/test_package.py index 051423f6..32c4791f 100644 --- a/tests/bindings/python/bt2/test_package.py +++ b/tests/bindings/python/bt2/test_package.py @@ -258,6 +258,9 @@ class PackageTestCase(unittest.TestCase): def test_has_QueryExecutor(self): self._assert_in_bt2('QueryExecutor') + def test_has_AutoSourceComponentSpec(self): + self._assert_in_bt2('AutoSourceComponentSpec') + def test_has_ComponentSpec(self): self._assert_in_bt2('ComponentSpec') diff --git a/tests/bindings/python/bt2/test_trace_collection_message_iterator.py b/tests/bindings/python/bt2/test_trace_collection_message_iterator.py index 22de3f36..947c37b2 100644 --- a/tests/bindings/python/bt2/test_trace_collection_message_iterator.py +++ b/tests/bindings/python/bt2/test_trace_collection_message_iterator.py @@ -23,10 +23,21 @@ import os import os.path +_BT_TESTS_DATADIR = os.environ['BT_TESTS_DATADIR'] _BT_CTF_TRACES_PATH = os.environ['BT_CTF_TRACES_PATH'] _3EVENTS_INTERSECT_TRACE_PATH = os.path.join( _BT_CTF_TRACES_PATH, 'intersection', '3eventsintersect' ) +_NOINTERSECT_TRACE_PATH = os.path.join( + _BT_CTF_TRACES_PATH, 'intersection', 'nointersect' +) +_SEQUENCE_TRACE_PATH = os.path.join(_BT_CTF_TRACES_PATH, 'succeed', 'sequence') +_AUTO_SOURCE_DISCOVERY_GROUPING_PATH = os.path.join( + _BT_TESTS_DATADIR, 'auto-source-discovery', 'grouping' +) +_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH = os.path.join( + _BT_TESTS_DATADIR, 'auto-source-discovery', 'params-log-level' +) class ComponentSpecTestCase(unittest.TestCase): @@ -168,3 +179,358 @@ class TraceCollectionMessageIteratorTestCase(unittest.TestCase): msg_iter = bt2.TraceCollectionMessageIterator(specs, end=13515309.000000075) hist = _count_msgs_by_type(msg_iter) self.assertEqual(hist[bt2._EventMessage], 5) + + def test_iter_auto_source_component_spec(self): + specs = [bt2.AutoSourceComponentSpec(_3EVENTS_INTERSECT_TRACE_PATH)] + msg_iter = bt2.TraceCollectionMessageIterator(specs) + msgs = list(msg_iter) + self.assertEqual(len(msgs), 28) + hist = _count_msgs_by_type(msgs) + self.assertEqual(hist[bt2._EventMessage], 8) + + def test_iter_auto_source_component_spec_list_of_strings(self): + msg_iter = bt2.TraceCollectionMessageIterator([_3EVENTS_INTERSECT_TRACE_PATH]) + msgs = list(msg_iter) + self.assertEqual(len(msgs), 28) + hist = _count_msgs_by_type(msgs) + self.assertEqual(hist[bt2._EventMessage], 8) + + def test_iter_auto_source_component_spec_string(self): + msg_iter = bt2.TraceCollectionMessageIterator(_3EVENTS_INTERSECT_TRACE_PATH) + msgs = list(msg_iter) + self.assertEqual(len(msgs), 28) + hist = _count_msgs_by_type(msgs) + self.assertEqual(hist[bt2._EventMessage], 8) + + def test_iter_mixed_inputs(self): + msg_iter = bt2.TraceCollectionMessageIterator( + [ + _3EVENTS_INTERSECT_TRACE_PATH, + bt2.AutoSourceComponentSpec(_SEQUENCE_TRACE_PATH), + bt2.ComponentSpec('ctf', 'fs', _NOINTERSECT_TRACE_PATH), + ] + ) + msgs = list(msg_iter) + self.assertEqual(len(msgs), 76) + hist = _count_msgs_by_type(msgs) + self.assertEqual(hist[bt2._EventMessage], 24) + + +class _TestAutoDiscoverSourceComponentSpecs(unittest.TestCase): + def setUp(self): + self._saved_babeltrace_plugin_path = os.environ['BABELTRACE_PLUGIN_PATH'] + os.environ['BABELTRACE_PLUGIN_PATH'] += ':' + self._plugin_path + + def tearDown(self): + os.environ['BABELTRACE_PLUGIN_PATH'] = self._saved_babeltrace_plugin_path + + +class TestAutoDiscoverSourceComponentSpecsGrouping( + _TestAutoDiscoverSourceComponentSpecs +): + _plugin_path = _AUTO_SOURCE_DISCOVERY_GROUPING_PATH + + def test_grouping(self): + specs = [ + bt2.AutoSourceComponentSpec('ABCDE'), + bt2.AutoSourceComponentSpec(_AUTO_SOURCE_DISCOVERY_GROUPING_PATH), + bt2.AutoSourceComponentSpec('does-not-exist'), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 8) + + self.assertEqual(msgs[0].stream.name, 'TestSourceABCDE: ABCDE') + self.assertEqual(msgs[1].stream.name, 'TestSourceExt: aaa1, aaa2, aaa3') + self.assertEqual(msgs[2].stream.name, 'TestSourceExt: bbb1, bbb2') + self.assertEqual(msgs[3].stream.name, 'TestSourceExt: ccc1') + self.assertEqual(msgs[4].stream.name, 'TestSourceExt: ccc2') + self.assertEqual(msgs[5].stream.name, 'TestSourceExt: ccc3') + self.assertEqual(msgs[6].stream.name, 'TestSourceExt: ccc4') + self.assertEqual(msgs[7].stream.name, 'TestSourceSomeDir: some-dir') + + +class TestAutoDiscoverSourceComponentSpecsParamsObjLogLevel( + _TestAutoDiscoverSourceComponentSpecs +): + _plugin_path = _AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH + + _dir_a = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-a') + _dir_b = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-b') + _dir_ab = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-ab') + + def _test_two_comps_from_one_spec(self, params, obj=None, logging_level=None): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_ab, params=params, obj=obj, logging_level=logging_level + ) + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + + return msgs + + def test_params_two_comps_from_one_spec(self): + msgs = self._test_two_comps_from_one_spec( + params={'test-allo': 'madame', 'what': 'test-params'} + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: ('test-allo', 'madame')") + self.assertEqual(msgs[1].stream.name, "TestSourceB: ('test-allo', 'madame')") + + def test_obj_two_comps_from_one_spec(self): + msgs = self._test_two_comps_from_one_spec( + params={'what': 'python-obj'}, obj='deore' + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: deore") + self.assertEqual(msgs[1].stream.name, "TestSourceB: deore") + + def test_log_level_two_comps_from_one_spec(self): + msgs = self._test_two_comps_from_one_spec( + params={'what': 'log-level'}, logging_level=bt2.LoggingLevel.DEBUG + ) + + self.assertEqual( + msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.DEBUG) + ) + self.assertEqual( + msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.DEBUG) + ) + + def _test_two_comps_from_two_specs( + self, + params_a=None, + params_b=None, + obj_a=None, + obj_b=None, + logging_level_a=None, + logging_level_b=None, + ): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a + ), + bt2.AutoSourceComponentSpec( + self._dir_b, params=params_b, obj=obj_b, logging_level=logging_level_b + ), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + + return msgs + + def test_params_two_comps_from_two_specs(self): + msgs = self._test_two_comps_from_two_specs( + params_a={'test-allo': 'madame', 'what': 'test-params'}, + params_b={'test-bonjour': 'monsieur', 'what': 'test-params'}, + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: ('test-allo', 'madame')") + self.assertEqual( + msgs[1].stream.name, "TestSourceB: ('test-bonjour', 'monsieur')" + ) + + def test_obj_two_comps_from_two_specs(self): + msgs = self._test_two_comps_from_two_specs( + params_a={'what': 'python-obj'}, + params_b={'what': 'python-obj'}, + obj_a='deore', + obj_b='alivio', + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: deore") + self.assertEqual(msgs[1].stream.name, "TestSourceB: alivio") + + def test_log_level_two_comps_from_two_specs(self): + msgs = self._test_two_comps_from_two_specs( + params_a={'what': 'log-level'}, + params_b={'what': 'log-level'}, + logging_level_a=bt2.LoggingLevel.DEBUG, + logging_level_b=bt2.LoggingLevel.TRACE, + ) + + self.assertEqual( + msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.DEBUG) + ) + self.assertEqual( + msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.TRACE) + ) + + def _test_one_comp_from_one_spec_one_comp_from_both_1( + self, + params_a=None, + params_ab=None, + obj_a=None, + obj_ab=None, + logging_level_a=None, + logging_level_ab=None, + ): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a + ), + bt2.AutoSourceComponentSpec( + self._dir_ab, + params=params_ab, + obj=obj_ab, + logging_level=logging_level_ab, + ), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + + return msgs + + def test_params_one_comp_from_one_spec_one_comp_from_both_1(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1( + params_a={'test-allo': 'madame', 'what': 'test-params'}, + params_ab={'test-bonjour': 'monsieur', 'what': 'test-params'}, + ) + + self.assertEqual( + msgs[0].stream.name, + "TestSourceA: ('test-allo', 'madame'), ('test-bonjour', 'monsieur')", + ) + self.assertEqual( + msgs[1].stream.name, "TestSourceB: ('test-bonjour', 'monsieur')" + ) + + def test_obj_one_comp_from_one_spec_one_comp_from_both_1(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1( + params_a={'what': 'python-obj'}, + params_ab={'what': 'python-obj'}, + obj_a='deore', + obj_ab='alivio', + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: alivio") + self.assertEqual(msgs[1].stream.name, "TestSourceB: alivio") + + def test_log_level_one_comp_from_one_spec_one_comp_from_both_1(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1( + params_a={'what': 'log-level'}, + params_ab={'what': 'log-level'}, + logging_level_a=bt2.LoggingLevel.DEBUG, + logging_level_ab=bt2.LoggingLevel.TRACE, + ) + + self.assertEqual( + msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.TRACE) + ) + self.assertEqual( + msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.TRACE) + ) + + def _test_one_comp_from_one_spec_one_comp_from_both_2( + self, + params_ab=None, + params_a=None, + obj_ab=None, + obj_a=None, + logging_level_ab=None, + logging_level_a=None, + ): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_ab, + params=params_ab, + obj=obj_ab, + logging_level=logging_level_ab, + ), + bt2.AutoSourceComponentSpec( + self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a + ), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + + return msgs + + def test_params_one_comp_from_one_spec_one_comp_from_both_2(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2( + params_ab={ + 'test-bonjour': 'madame', + 'test-salut': 'les amis', + 'what': 'test-params', + }, + params_a={'test-bonjour': 'monsieur', 'what': 'test-params'}, + ) + + self.assertEqual( + msgs[0].stream.name, + "TestSourceA: ('test-bonjour', 'monsieur'), ('test-salut', 'les amis')", + ) + self.assertEqual( + msgs[1].stream.name, + "TestSourceB: ('test-bonjour', 'madame'), ('test-salut', 'les amis')", + ) + + def test_obj_one_comp_from_one_spec_one_comp_from_both_2(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2( + params_ab={'what': 'python-obj'}, + params_a={'what': 'python-obj'}, + obj_ab='deore', + obj_a='alivio', + ) + + self.assertEqual(msgs[0].stream.name, "TestSourceA: alivio") + self.assertEqual(msgs[1].stream.name, "TestSourceB: deore") + + def test_log_level_one_comp_from_one_spec_one_comp_from_both_2(self): + msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2( + params_ab={'what': 'log-level'}, + params_a={'what': 'log-level'}, + logging_level_ab=bt2.LoggingLevel.DEBUG, + logging_level_a=bt2.LoggingLevel.TRACE, + ) + + self.assertEqual( + msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.TRACE) + ) + self.assertEqual( + msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.DEBUG) + ) + + def test_obj_override_with_none(self): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_ab, params={'what': 'python-obj'}, obj='deore' + ), + bt2.AutoSourceComponentSpec( + self._dir_a, params={'what': 'python-obj'}, obj=None + ), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + self.assertEqual(msgs[0].stream.name, "TestSourceA: None") + self.assertEqual(msgs[1].stream.name, "TestSourceB: deore") + + def test_obj_no_override_with_no_obj(self): + specs = [ + bt2.AutoSourceComponentSpec( + self._dir_ab, params={'what': 'python-obj'}, obj='deore' + ), + bt2.AutoSourceComponentSpec(self._dir_a, params={'what': 'python-obj'}), + ] + it = bt2.TraceCollectionMessageIterator(specs) + msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage] + + self.assertEqual(len(msgs), 2) + self.assertEqual(msgs[0].stream.name, "TestSourceA: deore") + self.assertEqual(msgs[1].stream.name, "TestSourceB: deore") + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/data/auto-source-discovery/params-log-level/bt_plugin_test.py b/tests/data/auto-source-discovery/params-log-level/bt_plugin_test.py index a1ac6994..336f49c2 100644 --- a/tests/data/auto-source-discovery/params-log-level/bt_plugin_test.py +++ b/tests/data/auto-source-discovery/params-log-level/bt_plugin_test.py @@ -14,6 +14,7 @@ import os class TestIter(bt2._UserMessageIterator): def __init__(self, output_port): params = output_port.user_data['params'] + obj = output_port.user_data['obj'] comp_cls_name = self._component.__class__.__name__ @@ -23,6 +24,9 @@ class TestIter(bt2._UserMessageIterator): elif params['what'] == 'log-level': log_level = self._component.logging_level stream_name = '{}: {}'.format(comp_cls_name, log_level) + elif params['what'] == 'python-obj': + assert type(obj) == str or obj is None + stream_name = '{}: {}'.format(comp_cls_name, obj) else: assert False @@ -44,17 +48,17 @@ class TestIter(bt2._UserMessageIterator): class Base: - def __init__(self, params): + def __init__(self, params, obj): tc = self._create_trace_class() sc = tc.create_stream_class() - self._add_output_port('out', {'params': params, 'sc': sc}) + self._add_output_port('out', {'params': params, 'obj': obj, 'sc': sc}) @bt2.plugin_component_class class TestSourceA(Base, bt2._UserSourceComponent, message_iterator_class=TestIter): def __init__(self, params, obj): - super().__init__(params) + super().__init__(params, obj) @staticmethod def _user_query(priv_query_exec, obj, params, method_obj): @@ -77,7 +81,7 @@ class TestSourceA(Base, bt2._UserSourceComponent, message_iterator_class=TestIte @bt2.plugin_component_class class TestSourceB(Base, bt2._UserSourceComponent, message_iterator_class=TestIter): def __init__(self, params, obj): - super().__init__(params) + super().__init__(params, obj) @staticmethod def _user_query(priv_query_exec, obj, params, method_obj): diff --git a/tests/utils/utils.sh b/tests/utils/utils.sh index 4b545d09..83926362 100644 --- a/tests/utils/utils.sh +++ b/tests/utils/utils.sh @@ -339,6 +339,7 @@ run_python_bt2() { "BABELTRACE_PYTHON_BT2_NO_TRACEBACK=1" \ "BABELTRACE_PLUGIN_PATH=${BT_TESTS_BABELTRACE_PLUGIN_PATH}" \ "LIBBABELTRACE2_PLUGIN_PROVIDER_DIR=${BT_TESTS_PROVIDER_DIR}" \ + "BT_TESTS_DATADIR=${BT_TESTS_DATADIR}" \ "BT_CTF_TRACES_PATH=${BT_CTF_TRACES_PATH}" \ "BT_PLUGINS_PATH=${BT_PLUGINS_PATH}" \ "PYTHONPATH=${BT_TESTS_PYTHONPATH}:${BT_TESTS_SRCDIR}/utils/python" -- 2.34.1