bt2: add auto source discovery support to TraceCollectionMessageIterator
authorSimon Marchi <simon.marchi@efficios.com>
Mon, 5 Aug 2019 18:45:38 +0000 (14:45 -0400)
committerPhilippe Proulx <eeppeliteloop@gmail.com>
Mon, 12 Aug 2019 02:19:51 +0000 (22:19 -0400)
This patch makes TraceCollectionMessageIterator capable of automatically
detecting source components from given input strings, similar to how the
command line interface does.

A new type AutoSourceComponentSpec is introduced.  The user can
instantiate this type, passing a mandatory input string (equivalent to a
non-option argument on the CLI).  The user can then pass instances of
AutoSourceComponentSpec in the source_component_specs list when creating
a TraceCollectionMessageIterator.  In
TraceCollectionMessageIterator.__init_, we run the automatic source
algorithm on all inputs passed through AutoSourceComponentSpec
objects.  We create ComponentSpec from the results of the auto discovery
and append them to the source component spec list.  The rest of the
TraceCollectionMessageIterator then works as before.

The user can attach params and log levels to AutoSourceComponentSpec
inputs.  The semantic for those is the same as for the CLI.  There is
also an `obj` parameter, allowing to pass an arbitrary Python object.
The semantic for this one is the same as the log level: if an
auto-discovered component comes from multiple inputs, it will receive
the obj of the last input that contributed to getting it instantiated
that has a non-None `obj`.

For convenience, some shorthands when creating a
TraceCollectionMessageIterator are equivalent to using an
AutoSourceComponentSpec.  For example, these three forms are equivalent:

 - TraceCollectionMessageIterator('foo')
 - TraceCollectionMessageIterator(AutoSourceComponentSpec('foo'))
 - TraceCollectionMessageIterator([AutoSourceComponentSpec('foo')])

Implementation details
----------------------

New test cases are added to test the new feature.  They are based on
the corresponding CLI tests, in that they test equivalent scenarios
(same inputs, expecting the same results).

In order for TraceCollectionMessageIterator to find the test plugin
(both bt_plugin_test.py files), the only way I found was to amend
BABELTRACE_PLUGIN_PATH for the duration of the test.

Change-Id: I0e1100d850c920723861609cfa14707f8c669892
Signed-off-by: Simon Marchi <simon.marchi@efficios.com>
Reviewed-on: https://review.lttng.org/c/babeltrace/+/1826
Reviewed-by: Philippe Proulx <eeppeliteloop@gmail.com>
Tested-by: jenkins <jenkins@lttng.org>
src/bindings/python/bt2/Makefile.am
src/bindings/python/bt2/bt2/__init__.py
src/bindings/python/bt2/bt2/native_bt.i
src/bindings/python/bt2/bt2/native_bt_autodisc.i [new file with mode: 0644]
src/bindings/python/bt2/bt2/native_bt_autodisc.i.h [new file with mode: 0644]
src/bindings/python/bt2/bt2/trace_collection_message_iterator.py
src/bindings/python/bt2/setup.py.in
tests/bindings/python/bt2/test_package.py
tests/bindings/python/bt2/test_trace_collection_message_iterator.py
tests/data/auto-source-discovery/params-log-level/bt_plugin_test.py
tests/utils/utils.sh

index 666963404fa305a9d50f4839421418ffc18908f0..a0eb73a93c92c52cc69285fc31068e4767bb8440 100644 (file)
@@ -7,6 +7,8 @@ INSTALLED_FILES=$(builddir)/installed_files.txt
 
 SWIG_INTERFACE_FILES =                                 \
        bt2/native_bt.i                                 \
+       bt2/native_bt_autodisc.i                        \
+       bt2/native_bt_autodisc.i.h                      \
        bt2/native_bt_bt2_objects.h                     \
        bt2/native_bt_clock_class.i                     \
        bt2/native_bt_clock_snapshot.i                  \
@@ -85,6 +87,7 @@ STATIC_BINDINGS_DEPS =                                        \
 # Convenience static libraries on which the Python bindings library depends.
 # These are listed in the setup.py(.in) file.
 STATIC_LIBRARIES_DEPS = \
+       $(top_builddir)/src/autodisc/libbabeltrace2-autodisc.la \
        $(top_builddir)/src/logging/libbabeltrace2-logging.la \
        $(top_builddir)/src/common/libbabeltrace2-common.la \
        $(top_builddir)/src/py-common/libbabeltrace2-py-common.la
index d6e3ab2bb5c7741a1d5c9003dfc2f5ea11d68faf..0bead489b47ea3c78af3fbee0661df4609daf00b 100644 (file)
@@ -99,6 +99,7 @@ from bt2.plugin import find_plugin
 from bt2.py_plugin import plugin_component_class
 from bt2.py_plugin import register_plugin
 from bt2.query_executor import QueryExecutor
+from bt2.trace_collection_message_iterator import AutoSourceComponentSpec
 from bt2.trace_collection_message_iterator import ComponentSpec
 from bt2.trace_collection_message_iterator import TraceCollectionMessageIterator
 from bt2.value import create_value
index 6d9a3696c8045b86b75c1ea16e9d18b4f2869166..de9ff3c32bb5269ce36382ea5918e217fcfec395 100644 (file)
@@ -204,6 +204,7 @@ void bt_bt2_exit_handler(void);
 %include <babeltrace2/func-status.h>
 
 /* Per-module interface files */
+%include "native_bt_autodisc.i"
 %include "native_bt_clock_class.i"
 %include "native_bt_clock_snapshot.i"
 %include "native_bt_component.i"
diff --git a/src/bindings/python/bt2/bt2/native_bt_autodisc.i b/src/bindings/python/bt2/bt2/native_bt_autodisc.i
new file mode 100644 (file)
index 0000000..5c54cc3
--- /dev/null
@@ -0,0 +1,30 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2017 Philippe Proulx <pproulx@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+bt_value *bt_bt2_auto_discover_source_components(const bt_value *inputs,
+               const bt_plugin_set *plugin_set);
+
+%{
+#include "native_bt_autodisc.i.h"
+%}
diff --git a/src/bindings/python/bt2/bt2/native_bt_autodisc.i.h b/src/bindings/python/bt2/bt2/native_bt_autodisc.i.h
new file mode 100644 (file)
index 0000000..e09f54e
--- /dev/null
@@ -0,0 +1,209 @@
+/*
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2016 Philippe Proulx <pproulx@efficios.com>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <autodisc/autodisc.h>
+#include <common/common.h>
+
+/*
+ * Interface between the Python bindings and the auto source discovery system:
+ * input strings go in, specs for components to instantiate go out.
+ *
+ * `inputs` must be an array of strings, the list of inputs in which to look
+ * for traces.  `plugin_set` is the set of plugins to query.
+ *
+ * Returns a map with the following entries:
+ *
+ *   - status: signed integer, return status of this function
+ *   - results: array, each element is an array describing one auto
+ *              source discovery result (see `struct
+ *              auto_source_discovery_result` for more details about these):
+ *
+ *     - 0: plugin name, string
+ *     - 1: class name, string
+ *     - 2: inputs, array of strings
+ *     - 3: original input indices, array of unsigned integers
+ *
+ * This function can also return None, if it failed to allocate memory
+ * for the return value and status code.
+ */
+bt_value *bt_bt2_auto_discover_source_components(const bt_value *inputs,
+               const bt_plugin_set *plugin_set)
+{
+       uint64_t i;
+       int status = 0;
+       static const char * const module_name = "Automatic source discovery";
+       const bt_plugin **plugins = NULL;
+       const uint64_t plugin_count = bt_plugin_set_get_plugin_count(plugin_set);
+       struct auto_source_discovery auto_disc = { 0 };
+       bt_value *result = NULL;
+       bt_value *components_list = NULL;
+       bt_value *component_info = NULL;
+       bt_value_map_insert_entry_status insert_entry_status;
+
+       BT_ASSERT(bt_value_get_type(inputs) == BT_VALUE_TYPE_ARRAY);
+       for (i = 0; i < bt_value_array_get_size(inputs); i++) {
+               const bt_value *elem = bt_value_array_borrow_element_by_index_const(inputs, i);
+               BT_ASSERT(bt_value_get_type(elem) == BT_VALUE_TYPE_STRING);
+       }
+
+       result = bt_value_map_create();
+       if (!result) {
+               static const char * const err = "Failed to create a map value.";
+               BT_LOGE_STR(err);
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name, err);
+               PyErr_NoMemory();
+               goto end;
+       }
+
+       status = auto_source_discovery_init(&auto_disc);
+       if (status != 0) {
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                       "Failed to initialize auto source discovery structure.");
+                goto error;
+       }
+
+       /* Create array of bt_plugin pointers from plugin set. */
+       plugins = g_new(const bt_plugin *, plugin_count);
+       if (!plugins) {
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                       "Failed to allocate plugins array.");
+               status = __BT_FUNC_STATUS_MEMORY_ERROR;
+                goto error;
+       }
+
+       for (i = 0; i < plugin_count; i++) {
+               plugins[i] = bt_plugin_set_borrow_plugin_by_index_const(plugin_set, i);
+       }
+
+       status = auto_discover_source_components(
+               inputs,
+               plugins,
+               plugin_count,
+               NULL,
+               bt_python_bindings_bt2_log_level,
+               &auto_disc);
+       if (status != 0) {
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                       "Failed to auto discover sources.");
+               goto error;
+       }
+
+       components_list = bt_value_array_create();
+       if (!components_list) {
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                       "Failed to allocate one array value.");
+               status = __BT_FUNC_STATUS_MEMORY_ERROR;
+               goto error;
+       }
+
+       insert_entry_status = bt_value_map_insert_entry(result, "results", components_list);
+       if (insert_entry_status != BT_VALUE_MAP_INSERT_ENTRY_STATUS_OK) {
+               BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                       "Failed to insert a map entry.");
+               status = (int) insert_entry_status;
+               goto error;
+       }
+
+       for (i = 0; i < auto_disc.results->len; i++) {
+               struct auto_source_discovery_result *result =
+                       g_ptr_array_index(auto_disc.results, i);
+               bt_value_array_append_element_status append_element_status;
+
+               component_info = bt_value_array_create();
+               if (!component_info) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to allocate one array value.");
+                       status = __BT_FUNC_STATUS_MEMORY_ERROR;
+                       goto error;
+               }
+
+               append_element_status = bt_value_array_append_string_element(
+                       component_info, result->plugin_name);
+               if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to append one array element.");
+                       status = (int) append_element_status;
+                       goto error;
+               }
+
+               append_element_status = bt_value_array_append_string_element(
+                       component_info, result->source_cc_name);
+               if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to append one array element.");
+                       status = (int) append_element_status;
+                       goto error;
+               }
+
+               append_element_status = bt_value_array_append_element(
+                       component_info, result->inputs);
+               if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to append one array element.");
+                       status = (int) append_element_status;
+                       goto error;
+               }
+
+               append_element_status = bt_value_array_append_element(
+                       component_info, result->original_input_indices);
+               if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to append one array element.");
+                       status = (int) append_element_status;
+                       goto error;
+               }
+
+               append_element_status = bt_value_array_append_element(components_list,
+                       component_info);
+               if (append_element_status != BT_VALUE_ARRAY_APPEND_ELEMENT_STATUS_OK) {
+                       BT_CURRENT_THREAD_ERROR_APPEND_CAUSE_FROM_UNKNOWN(module_name,
+                               "Failed to append one array element.");
+                       status = (int) append_element_status;
+                       goto error;
+               }
+
+               BT_VALUE_PUT_REF_AND_RESET(component_info);
+       }
+
+       status = 0;
+       goto end;
+error:
+       BT_ASSERT(status != 0);
+
+end:
+       if (result) {
+               insert_entry_status = bt_value_map_insert_signed_integer_entry(result, "status", status);
+               if (insert_entry_status != BT_VALUE_MAP_INSERT_ENTRY_STATUS_OK) {
+                       BT_VALUE_PUT_REF_AND_RESET(result);
+                       PyErr_NoMemory();
+               }
+       }
+
+       auto_source_discovery_fini(&auto_disc);
+       g_free(plugins);
+       bt_value_put_ref(components_list);
+       bt_value_put_ref(component_info);
+
+       return result;
+}
index 9c931fe2d4be568a5362cf1c020be87f8e76e00b..56b48da9ff8061fe4a96e70870495ec711c313a2 100644 (file)
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-from bt2 import utils
+from bt2 import utils, native_bt
 import bt2
 import itertools
 from bt2 import message_iterator as bt2_message_iterator
 from bt2 import logging as bt2_logging
 from bt2 import port as bt2_port
 from bt2 import component as bt2_component
+from bt2 import value as bt2_value
+from bt2 import plugin as bt2_plugin
 import datetime
 from collections import namedtuple
 import numbers
@@ -36,7 +38,29 @@ import numbers
 _ComponentAndSpec = namedtuple('_ComponentAndSpec', ['comp', 'spec'])
 
 
-class ComponentSpec:
+class _BaseComponentSpec:
+    def __init__(self, params, obj, logging_level):
+        if logging_level is not None:
+            utils._check_log_level(logging_level)
+
+        self._params = bt2.create_value(params)
+        self._obj = obj
+        self._logging_level = logging_level
+
+    @property
+    def params(self):
+        return self._params
+
+    @property
+    def obj(self):
+        return self._obj
+
+    @property
+    def logging_level(self):
+        return self._logging_level
+
+
+class ComponentSpec(_BaseComponentSpec):
     def __init__(
         self,
         plugin_name,
@@ -45,18 +69,16 @@ class ComponentSpec:
         obj=None,
         logging_level=bt2_logging.LoggingLevel.NONE,
     ):
+        if type(params) is str:
+            params = {'inputs': [params]}
+
+        super().__init__(params, obj, logging_level)
+
         utils._check_str(plugin_name)
         utils._check_str(class_name)
-        utils._check_log_level(logging_level)
+
         self._plugin_name = plugin_name
         self._class_name = class_name
-        self._logging_level = logging_level
-        self._obj = obj
-
-        if type(params) is str:
-            self._params = bt2.create_value({'inputs': [params]})
-        else:
-            self._params = bt2.create_value(params)
 
     @property
     def plugin_name(self):
@@ -66,17 +88,102 @@ class ComponentSpec:
     def class_name(self):
         return self._class_name
 
-    @property
-    def logging_level(self):
-        return self._logging_level
 
-    @property
-    def params(self):
-        return self._params
+class AutoSourceComponentSpec(_BaseComponentSpec):
+    _no_obj = object()
+
+    def __init__(self, input, params=None, obj=_no_obj, logging_level=None):
+        super().__init__(params, obj, logging_level)
+        self._input = input
 
     @property
-    def obj(self):
-        return self._obj
+    def input(self):
+        return self._input
+
+
+def _auto_discover_source_component_specs(auto_source_comp_specs, plugin_set):
+    # Transform a list of `AutoSourceComponentSpec` in a list of `ComponentSpec`
+    # using the automatic source discovery mechanism.
+    inputs = bt2.ArrayValue([spec.input for spec in auto_source_comp_specs])
+
+    if plugin_set is None:
+        plugin_set = bt2.find_plugins()
+    else:
+        utils._check_type(plugin_set, bt2_plugin._PluginSet)
+
+    res_ptr = native_bt.bt2_auto_discover_source_components(
+        inputs._ptr, plugin_set._ptr
+    )
+
+    if res_ptr is None:
+        raise bt2._MemoryError('cannot auto discover source components')
+
+    res = bt2_value._create_from_ptr(res_ptr)
+
+    assert type(res) == bt2.MapValue
+    assert 'status' in res
+
+    status = res['status']
+    utils._handle_func_status(status, 'cannot auto-discover source components')
+
+    comp_specs = []
+    comp_specs_raw = res['results']
+    assert type(comp_specs_raw) == bt2.ArrayValue
+
+    for comp_spec_raw in comp_specs_raw:
+        assert type(comp_spec_raw) == bt2.ArrayValue
+        assert len(comp_spec_raw) == 4
+
+        plugin_name = comp_spec_raw[0]
+        assert type(plugin_name) == bt2.StringValue
+        plugin_name = str(plugin_name)
+
+        class_name = comp_spec_raw[1]
+        assert type(class_name) == bt2.StringValue
+        class_name = str(class_name)
+
+        comp_inputs = comp_spec_raw[2]
+        assert type(comp_inputs) == bt2.ArrayValue
+
+        comp_orig_indices = comp_spec_raw[3]
+        assert type(comp_orig_indices)
+
+        params = bt2.MapValue()
+        logging_level = bt2.LoggingLevel.NONE
+        obj = None
+
+        # Compute `params` for this component by piling up params given to all
+        # AutoSourceComponentSpec objects that contributed in the instantiation
+        # of this component.
+        #
+        # The effective log level for a component is the last one specified
+        # across the AutoSourceComponentSpec that contributed in its
+        # instantiation.
+        for idx in comp_orig_indices:
+            orig_spec = auto_source_comp_specs[idx]
+
+            if orig_spec.params is not None:
+                params.update(orig_spec.params)
+
+            if orig_spec.logging_level is not None:
+                logging_level = orig_spec.logging_level
+
+            if orig_spec.obj is not AutoSourceComponentSpec._no_obj:
+                obj = orig_spec.obj
+
+        params['inputs'] = comp_inputs
+
+        comp_specs.append(
+            ComponentSpec(
+                plugin_name,
+                class_name,
+                params=params,
+                obj=obj,
+                logging_level=logging_level,
+            )
+        )
+
+    return comp_specs
 
 
 # datetime.datetime or integral to nanoseconds
@@ -127,6 +234,7 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator):
         stream_intersection_mode=False,
         begin=None,
         end=None,
+        plugin_set=None,
     ):
         utils._check_bool(stream_intersection_mode)
         self._stream_intersection_mode = stream_intersection_mode
@@ -134,15 +242,46 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator):
         self._end_ns = _get_ns(end)
         self._msg_list = [None]
 
-        if type(source_component_specs) is ComponentSpec:
+        # If a single item is provided, convert to a list.
+        if type(source_component_specs) in (
+            ComponentSpec,
+            AutoSourceComponentSpec,
+            str,
+        ):
             source_component_specs = [source_component_specs]
 
+        # Convert any string to an AutoSourceComponentSpec.
+        def str_to_auto(item):
+            if type(item) is str:
+                item = AutoSourceComponentSpec(item)
+
+            return item
+
+        source_component_specs = [str_to_auto(s) for s in source_component_specs]
+
         if type(filter_component_specs) is ComponentSpec:
             filter_component_specs = [filter_component_specs]
         elif filter_component_specs is None:
             filter_component_specs = []
 
-        self._src_comp_specs = source_component_specs
+        self._validate_source_component_specs(source_component_specs)
+        self._validate_filter_component_specs(filter_component_specs)
+
+        # Pass any `ComponentSpec` instance as-is.
+        self._src_comp_specs = [
+            spec for spec in source_component_specs if type(spec) is ComponentSpec
+        ]
+
+        # Convert any `AutoSourceComponentSpec` in concrete `ComponentSpec` instances.
+        auto_src_comp_specs = [
+            spec
+            for spec in source_component_specs
+            if type(spec) is AutoSourceComponentSpec
+        ]
+        self._src_comp_specs += _auto_discover_source_component_specs(
+            auto_src_comp_specs, plugin_set
+        )
+
         self._flt_comp_specs = filter_component_specs
         self._next_suffix = 1
         self._connect_ports = False
@@ -151,11 +290,21 @@ class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator):
         self._src_comps_and_specs = []
         self._flt_comps_and_specs = []
 
-        self._validate_component_specs(source_component_specs)
-        self._validate_component_specs(filter_component_specs)
         self._build_graph()
 
-    def _validate_component_specs(self, comp_specs):
+    def _validate_source_component_specs(self, comp_specs):
+        for comp_spec in comp_specs:
+            if (
+                type(comp_spec) is not ComponentSpec
+                and type(comp_spec) is not AutoSourceComponentSpec
+            ):
+                raise TypeError(
+                    '"{}" object is not a ComponentSpec or AutoSourceComponentSpec'.format(
+                        type(comp_spec)
+                    )
+                )
+
+    def _validate_filter_component_specs(self, comp_specs):
         for comp_spec in comp_specs:
             if type(comp_spec) is not ComponentSpec:
                 raise TypeError(
index 8172839cc1a5be99829e21465aa8c867bcbc33d6..0a94d1f3bb9424efb8fb6cbb13cabf35db341134 100644 (file)
@@ -41,6 +41,7 @@ def main():
         sources=['bt2/native_bt.c', '@srcdir@/bt2/logging.c'],
         libraries=['babeltrace2', 'glib-2.0'],
         extra_objects=[
+            '@top_builddir@/src/autodisc/.libs/libbabeltrace2-autodisc.a',
             '@top_builddir@/src/logging/.libs/libbabeltrace2-logging.a',
             '@top_builddir@/src/common/.libs/libbabeltrace2-common.a',
             '@top_builddir@/src/py-common/.libs/libbabeltrace2-py-common.a',
index 051423f660abab584821d2c1e267091a814bdb40..32c4791f4084ad38cbe5f2a46adc4d5d0543e68b 100644 (file)
@@ -258,6 +258,9 @@ class PackageTestCase(unittest.TestCase):
     def test_has_QueryExecutor(self):
         self._assert_in_bt2('QueryExecutor')
 
+    def test_has_AutoSourceComponentSpec(self):
+        self._assert_in_bt2('AutoSourceComponentSpec')
+
     def test_has_ComponentSpec(self):
         self._assert_in_bt2('ComponentSpec')
 
index 22de3f3641e57c8f45568459f853d7eb0802d826..947c37b21acd412e58ed8465801c24f501942175 100644 (file)
@@ -23,10 +23,21 @@ import os
 import os.path
 
 
+_BT_TESTS_DATADIR = os.environ['BT_TESTS_DATADIR']
 _BT_CTF_TRACES_PATH = os.environ['BT_CTF_TRACES_PATH']
 _3EVENTS_INTERSECT_TRACE_PATH = os.path.join(
     _BT_CTF_TRACES_PATH, 'intersection', '3eventsintersect'
 )
+_NOINTERSECT_TRACE_PATH = os.path.join(
+    _BT_CTF_TRACES_PATH, 'intersection', 'nointersect'
+)
+_SEQUENCE_TRACE_PATH = os.path.join(_BT_CTF_TRACES_PATH, 'succeed', 'sequence')
+_AUTO_SOURCE_DISCOVERY_GROUPING_PATH = os.path.join(
+    _BT_TESTS_DATADIR, 'auto-source-discovery', 'grouping'
+)
+_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH = os.path.join(
+    _BT_TESTS_DATADIR, 'auto-source-discovery', 'params-log-level'
+)
 
 
 class ComponentSpecTestCase(unittest.TestCase):
@@ -168,3 +179,358 @@ class TraceCollectionMessageIteratorTestCase(unittest.TestCase):
         msg_iter = bt2.TraceCollectionMessageIterator(specs, end=13515309.000000075)
         hist = _count_msgs_by_type(msg_iter)
         self.assertEqual(hist[bt2._EventMessage], 5)
+
+    def test_iter_auto_source_component_spec(self):
+        specs = [bt2.AutoSourceComponentSpec(_3EVENTS_INTERSECT_TRACE_PATH)]
+        msg_iter = bt2.TraceCollectionMessageIterator(specs)
+        msgs = list(msg_iter)
+        self.assertEqual(len(msgs), 28)
+        hist = _count_msgs_by_type(msgs)
+        self.assertEqual(hist[bt2._EventMessage], 8)
+
+    def test_iter_auto_source_component_spec_list_of_strings(self):
+        msg_iter = bt2.TraceCollectionMessageIterator([_3EVENTS_INTERSECT_TRACE_PATH])
+        msgs = list(msg_iter)
+        self.assertEqual(len(msgs), 28)
+        hist = _count_msgs_by_type(msgs)
+        self.assertEqual(hist[bt2._EventMessage], 8)
+
+    def test_iter_auto_source_component_spec_string(self):
+        msg_iter = bt2.TraceCollectionMessageIterator(_3EVENTS_INTERSECT_TRACE_PATH)
+        msgs = list(msg_iter)
+        self.assertEqual(len(msgs), 28)
+        hist = _count_msgs_by_type(msgs)
+        self.assertEqual(hist[bt2._EventMessage], 8)
+
+    def test_iter_mixed_inputs(self):
+        msg_iter = bt2.TraceCollectionMessageIterator(
+            [
+                _3EVENTS_INTERSECT_TRACE_PATH,
+                bt2.AutoSourceComponentSpec(_SEQUENCE_TRACE_PATH),
+                bt2.ComponentSpec('ctf', 'fs', _NOINTERSECT_TRACE_PATH),
+            ]
+        )
+        msgs = list(msg_iter)
+        self.assertEqual(len(msgs), 76)
+        hist = _count_msgs_by_type(msgs)
+        self.assertEqual(hist[bt2._EventMessage], 24)
+
+
+class _TestAutoDiscoverSourceComponentSpecs(unittest.TestCase):
+    def setUp(self):
+        self._saved_babeltrace_plugin_path = os.environ['BABELTRACE_PLUGIN_PATH']
+        os.environ['BABELTRACE_PLUGIN_PATH'] += ':' + self._plugin_path
+
+    def tearDown(self):
+        os.environ['BABELTRACE_PLUGIN_PATH'] = self._saved_babeltrace_plugin_path
+
+
+class TestAutoDiscoverSourceComponentSpecsGrouping(
+    _TestAutoDiscoverSourceComponentSpecs
+):
+    _plugin_path = _AUTO_SOURCE_DISCOVERY_GROUPING_PATH
+
+    def test_grouping(self):
+        specs = [
+            bt2.AutoSourceComponentSpec('ABCDE'),
+            bt2.AutoSourceComponentSpec(_AUTO_SOURCE_DISCOVERY_GROUPING_PATH),
+            bt2.AutoSourceComponentSpec('does-not-exist'),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 8)
+
+        self.assertEqual(msgs[0].stream.name, 'TestSourceABCDE: ABCDE')
+        self.assertEqual(msgs[1].stream.name, 'TestSourceExt: aaa1, aaa2, aaa3')
+        self.assertEqual(msgs[2].stream.name, 'TestSourceExt: bbb1, bbb2')
+        self.assertEqual(msgs[3].stream.name, 'TestSourceExt: ccc1')
+        self.assertEqual(msgs[4].stream.name, 'TestSourceExt: ccc2')
+        self.assertEqual(msgs[5].stream.name, 'TestSourceExt: ccc3')
+        self.assertEqual(msgs[6].stream.name, 'TestSourceExt: ccc4')
+        self.assertEqual(msgs[7].stream.name, 'TestSourceSomeDir: some-dir')
+
+
+class TestAutoDiscoverSourceComponentSpecsParamsObjLogLevel(
+    _TestAutoDiscoverSourceComponentSpecs
+):
+    _plugin_path = _AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH
+
+    _dir_a = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-a')
+    _dir_b = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-b')
+    _dir_ab = os.path.join(_AUTO_SOURCE_DISCOVERY_PARAMS_LOG_LEVEL_PATH, 'dir-ab')
+
+    def _test_two_comps_from_one_spec(self, params, obj=None, logging_level=None):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_ab, params=params, obj=obj, logging_level=logging_level
+            )
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+
+        return msgs
+
+    def test_params_two_comps_from_one_spec(self):
+        msgs = self._test_two_comps_from_one_spec(
+            params={'test-allo': 'madame', 'what': 'test-params'}
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: ('test-allo', 'madame')")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: ('test-allo', 'madame')")
+
+    def test_obj_two_comps_from_one_spec(self):
+        msgs = self._test_two_comps_from_one_spec(
+            params={'what': 'python-obj'}, obj='deore'
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: deore")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: deore")
+
+    def test_log_level_two_comps_from_one_spec(self):
+        msgs = self._test_two_comps_from_one_spec(
+            params={'what': 'log-level'}, logging_level=bt2.LoggingLevel.DEBUG
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.DEBUG)
+        )
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.DEBUG)
+        )
+
+    def _test_two_comps_from_two_specs(
+        self,
+        params_a=None,
+        params_b=None,
+        obj_a=None,
+        obj_b=None,
+        logging_level_a=None,
+        logging_level_b=None,
+    ):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a
+            ),
+            bt2.AutoSourceComponentSpec(
+                self._dir_b, params=params_b, obj=obj_b, logging_level=logging_level_b
+            ),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+
+        return msgs
+
+    def test_params_two_comps_from_two_specs(self):
+        msgs = self._test_two_comps_from_two_specs(
+            params_a={'test-allo': 'madame', 'what': 'test-params'},
+            params_b={'test-bonjour': 'monsieur', 'what': 'test-params'},
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: ('test-allo', 'madame')")
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: ('test-bonjour', 'monsieur')"
+        )
+
+    def test_obj_two_comps_from_two_specs(self):
+        msgs = self._test_two_comps_from_two_specs(
+            params_a={'what': 'python-obj'},
+            params_b={'what': 'python-obj'},
+            obj_a='deore',
+            obj_b='alivio',
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: deore")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: alivio")
+
+    def test_log_level_two_comps_from_two_specs(self):
+        msgs = self._test_two_comps_from_two_specs(
+            params_a={'what': 'log-level'},
+            params_b={'what': 'log-level'},
+            logging_level_a=bt2.LoggingLevel.DEBUG,
+            logging_level_b=bt2.LoggingLevel.TRACE,
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.DEBUG)
+        )
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.TRACE)
+        )
+
+    def _test_one_comp_from_one_spec_one_comp_from_both_1(
+        self,
+        params_a=None,
+        params_ab=None,
+        obj_a=None,
+        obj_ab=None,
+        logging_level_a=None,
+        logging_level_ab=None,
+    ):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a
+            ),
+            bt2.AutoSourceComponentSpec(
+                self._dir_ab,
+                params=params_ab,
+                obj=obj_ab,
+                logging_level=logging_level_ab,
+            ),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+
+        return msgs
+
+    def test_params_one_comp_from_one_spec_one_comp_from_both_1(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1(
+            params_a={'test-allo': 'madame', 'what': 'test-params'},
+            params_ab={'test-bonjour': 'monsieur', 'what': 'test-params'},
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name,
+            "TestSourceA: ('test-allo', 'madame'), ('test-bonjour', 'monsieur')",
+        )
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: ('test-bonjour', 'monsieur')"
+        )
+
+    def test_obj_one_comp_from_one_spec_one_comp_from_both_1(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1(
+            params_a={'what': 'python-obj'},
+            params_ab={'what': 'python-obj'},
+            obj_a='deore',
+            obj_ab='alivio',
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: alivio")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: alivio")
+
+    def test_log_level_one_comp_from_one_spec_one_comp_from_both_1(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_1(
+            params_a={'what': 'log-level'},
+            params_ab={'what': 'log-level'},
+            logging_level_a=bt2.LoggingLevel.DEBUG,
+            logging_level_ab=bt2.LoggingLevel.TRACE,
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.TRACE)
+        )
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.TRACE)
+        )
+
+    def _test_one_comp_from_one_spec_one_comp_from_both_2(
+        self,
+        params_ab=None,
+        params_a=None,
+        obj_ab=None,
+        obj_a=None,
+        logging_level_ab=None,
+        logging_level_a=None,
+    ):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_ab,
+                params=params_ab,
+                obj=obj_ab,
+                logging_level=logging_level_ab,
+            ),
+            bt2.AutoSourceComponentSpec(
+                self._dir_a, params=params_a, obj=obj_a, logging_level=logging_level_a
+            ),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+
+        return msgs
+
+    def test_params_one_comp_from_one_spec_one_comp_from_both_2(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2(
+            params_ab={
+                'test-bonjour': 'madame',
+                'test-salut': 'les amis',
+                'what': 'test-params',
+            },
+            params_a={'test-bonjour': 'monsieur', 'what': 'test-params'},
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name,
+            "TestSourceA: ('test-bonjour', 'monsieur'), ('test-salut', 'les amis')",
+        )
+        self.assertEqual(
+            msgs[1].stream.name,
+            "TestSourceB: ('test-bonjour', 'madame'), ('test-salut', 'les amis')",
+        )
+
+    def test_obj_one_comp_from_one_spec_one_comp_from_both_2(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2(
+            params_ab={'what': 'python-obj'},
+            params_a={'what': 'python-obj'},
+            obj_ab='deore',
+            obj_a='alivio',
+        )
+
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: alivio")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: deore")
+
+    def test_log_level_one_comp_from_one_spec_one_comp_from_both_2(self):
+        msgs = self._test_one_comp_from_one_spec_one_comp_from_both_2(
+            params_ab={'what': 'log-level'},
+            params_a={'what': 'log-level'},
+            logging_level_ab=bt2.LoggingLevel.DEBUG,
+            logging_level_a=bt2.LoggingLevel.TRACE,
+        )
+
+        self.assertEqual(
+            msgs[0].stream.name, "TestSourceA: {}".format(bt2.LoggingLevel.TRACE)
+        )
+        self.assertEqual(
+            msgs[1].stream.name, "TestSourceB: {}".format(bt2.LoggingLevel.DEBUG)
+        )
+
+    def test_obj_override_with_none(self):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_ab, params={'what': 'python-obj'}, obj='deore'
+            ),
+            bt2.AutoSourceComponentSpec(
+                self._dir_a, params={'what': 'python-obj'}, obj=None
+            ),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: None")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: deore")
+
+    def test_obj_no_override_with_no_obj(self):
+        specs = [
+            bt2.AutoSourceComponentSpec(
+                self._dir_ab, params={'what': 'python-obj'}, obj='deore'
+            ),
+            bt2.AutoSourceComponentSpec(self._dir_a, params={'what': 'python-obj'}),
+        ]
+        it = bt2.TraceCollectionMessageIterator(specs)
+        msgs = [x for x in it if type(x) is bt2._StreamBeginningMessage]
+
+        self.assertEqual(len(msgs), 2)
+        self.assertEqual(msgs[0].stream.name, "TestSourceA: deore")
+        self.assertEqual(msgs[1].stream.name, "TestSourceB: deore")
+
+
+if __name__ == '__main__':
+    unittest.main()
index a1ac699454bef32ec8937e49a789e1a381e0c942..336f49c2c904e1fef8007247984794e55530941f 100644 (file)
@@ -14,6 +14,7 @@ import os
 class TestIter(bt2._UserMessageIterator):
     def __init__(self, output_port):
         params = output_port.user_data['params']
+        obj = output_port.user_data['obj']
 
         comp_cls_name = self._component.__class__.__name__
 
@@ -23,6 +24,9 @@ class TestIter(bt2._UserMessageIterator):
         elif params['what'] == 'log-level':
             log_level = self._component.logging_level
             stream_name = '{}: {}'.format(comp_cls_name, log_level)
+        elif params['what'] == 'python-obj':
+            assert type(obj) == str or obj is None
+            stream_name = '{}: {}'.format(comp_cls_name, obj)
         else:
             assert False
 
@@ -44,17 +48,17 @@ class TestIter(bt2._UserMessageIterator):
 
 
 class Base:
-    def __init__(self, params):
+    def __init__(self, params, obj):
         tc = self._create_trace_class()
         sc = tc.create_stream_class()
 
-        self._add_output_port('out', {'params': params, 'sc': sc})
+        self._add_output_port('out', {'params': params, 'obj': obj, 'sc': sc})
 
 
 @bt2.plugin_component_class
 class TestSourceA(Base, bt2._UserSourceComponent, message_iterator_class=TestIter):
     def __init__(self, params, obj):
-        super().__init__(params)
+        super().__init__(params, obj)
 
     @staticmethod
     def _user_query(priv_query_exec, obj, params, method_obj):
@@ -77,7 +81,7 @@ class TestSourceA(Base, bt2._UserSourceComponent, message_iterator_class=TestIte
 @bt2.plugin_component_class
 class TestSourceB(Base, bt2._UserSourceComponent, message_iterator_class=TestIter):
     def __init__(self, params, obj):
-        super().__init__(params)
+        super().__init__(params, obj)
 
     @staticmethod
     def _user_query(priv_query_exec, obj, params, method_obj):
index 4b545d09c602ce28675ab9218c47dce9eb4f3bad..839263626fbc0e223848632a8e91fc245bf08745 100644 (file)
@@ -339,6 +339,7 @@ run_python_bt2() {
                "BABELTRACE_PYTHON_BT2_NO_TRACEBACK=1" \
                "BABELTRACE_PLUGIN_PATH=${BT_TESTS_BABELTRACE_PLUGIN_PATH}" \
                "LIBBABELTRACE2_PLUGIN_PROVIDER_DIR=${BT_TESTS_PROVIDER_DIR}" \
+               "BT_TESTS_DATADIR=${BT_TESTS_DATADIR}" \
                "BT_CTF_TRACES_PATH=${BT_CTF_TRACES_PATH}" \
                "BT_PLUGINS_PATH=${BT_PLUGINS_PATH}" \
                "PYTHONPATH=${BT_TESTS_PYTHONPATH}:${BT_TESTS_SRCDIR}/utils/python"
This page took 0.037018 seconds and 4 git commands to generate.