src/bindings/python/bt2/bt2/trace_collection_message_iterator.py

   1 # SPDX-License-Identifier: MIT
   2 #
   3 # Copyright (c) 2017 Philippe Proulx <pproulx@efficios.com>
   4
   5 from bt2 import utils, native_bt
   6 import bt2
   7 import itertools
   8 from bt2 import message_iterator as bt2_message_iterator
   9 from bt2 import port as bt2_port
  10 from bt2 import component as bt2_component
  11 from bt2 import value as bt2_value
  12 from bt2 import plugin as bt2_plugin
  13 import datetime
  14 from collections import namedtuple
  15 import numbers
  16
  17
  18 # a pair of component and ComponentSpec
  19 _ComponentAndSpec = namedtuple('_ComponentAndSpec', ['comp', 'spec'])
  20
  21
  22 class _BaseComponentSpec:
  23     # Base for any component spec that can be passed to
  24     # TraceCollectionMessageIterator.
  25     def __init__(self, params, obj, logging_level):
  26         if logging_level is not None:
  27             utils._check_log_level(logging_level)
  28
  29         self._params = bt2.create_value(params)
  30         self._obj = obj
  31         self._logging_level = logging_level
  32
  33     @property
  34     def params(self):
  35         return self._params
  36
  37     @property
  38     def obj(self):
  39         return self._obj
  40
  41     @property
  42     def logging_level(self):
  43         return self._logging_level
  44
  45
  46 class ComponentSpec(_BaseComponentSpec):
  47     # A component spec with a specific component class.
  48     def __init__(
  49         self,
  50         component_class,
  51         params=None,
  52         obj=None,
  53         logging_level=bt2.LoggingLevel.NONE,
  54     ):
  55         if type(params) is str:
  56             params = {'inputs': [params]}
  57
  58         super().__init__(params, obj, logging_level)
  59
  60         is_cc_object = isinstance(
  61             component_class,
  62             (bt2._SourceComponentClassConst, bt2._FilterComponentClassConst),
  63         )
  64         is_user_cc_type = isinstance(
  65             component_class, bt2_component._UserComponentType
  66         ) and issubclass(
  67             component_class, (bt2._UserSourceComponent, bt2._UserFilterComponent)
  68         )
  69
  70         if not is_cc_object and not is_user_cc_type:
  71             raise TypeError(
  72                 "'{}' is not a source or filter component class".format(
  73                     component_class.__class__.__name__
  74                 )
  75             )
  76
  77         self._component_class = component_class
  78
  79     @property
  80     def component_class(self):
  81         return self._component_class
  82
  83     @classmethod
  84     def from_named_plugin_and_component_class(
  85         cls,
  86         plugin_name,
  87         component_class_name,
  88         params=None,
  89         obj=None,
  90         logging_level=bt2.LoggingLevel.NONE,
  91     ):
  92         plugin = bt2.find_plugin(plugin_name)
  93
  94         if plugin is None:
  95             raise ValueError('no such plugin: {}'.format(plugin_name))
  96
  97         if component_class_name in plugin.source_component_classes:
  98             comp_class = plugin.source_component_classes[component_class_name]
  99         elif component_class_name in plugin.filter_component_classes:
 100             comp_class = plugin.filter_component_classes[component_class_name]
 101         else:
 102             raise KeyError(
 103                 'source or filter component class `{}` not found in plugin `{}`'.format(
 104                     component_class_name, plugin_name
 105                 )
 106             )
 107
 108         return cls(comp_class, params, obj, logging_level)
 109
 110
 111 class AutoSourceComponentSpec(_BaseComponentSpec):
 112     # A component spec that does automatic source discovery.
 113     _no_obj = object()
 114
 115     def __init__(self, input, params=None, obj=_no_obj, logging_level=None):
 116         super().__init__(params, obj, logging_level)
 117         self._input = input
 118
 119     @property
 120     def input(self):
 121         return self._input
 122
 123
 124 def _auto_discover_source_component_specs(auto_source_comp_specs, plugin_set):
 125     # Transform a list of `AutoSourceComponentSpec` in a list of `ComponentSpec`
 126     # using the automatic source discovery mechanism.
 127     inputs = bt2.ArrayValue([spec.input for spec in auto_source_comp_specs])
 128
 129     if plugin_set is None:
 130         plugin_set = bt2.find_plugins()
 131     else:
 132         utils._check_type(plugin_set, bt2_plugin._PluginSet)
 133
 134     res_ptr = native_bt.bt2_auto_discover_source_components(
 135         inputs._ptr, plugin_set._ptr
 136     )
 137
 138     if res_ptr is None:
 139         raise bt2._MemoryError('cannot auto discover source components')
 140
 141     res = bt2_value._create_from_ptr(res_ptr)
 142
 143     assert type(res) == bt2.MapValue
 144     assert 'status' in res
 145
 146     status = res['status']
 147     utils._handle_func_status(status, 'cannot auto-discover source components')
 148
 149     comp_specs = []
 150     comp_specs_raw = res['results']
 151     assert type(comp_specs_raw) == bt2.ArrayValue
 152
 153     used_input_indices = set()
 154
 155     for comp_spec_raw in comp_specs_raw:
 156         assert type(comp_spec_raw) == bt2.ArrayValue
 157         assert len(comp_spec_raw) == 4
 158
 159         plugin_name = comp_spec_raw[0]
 160         assert type(plugin_name) == bt2.StringValue
 161         plugin_name = str(plugin_name)
 162
 163         class_name = comp_spec_raw[1]
 164         assert type(class_name) == bt2.StringValue
 165         class_name = str(class_name)
 166
 167         comp_inputs = comp_spec_raw[2]
 168         assert type(comp_inputs) == bt2.ArrayValue
 169
 170         comp_orig_indices = comp_spec_raw[3]
 171         assert type(comp_orig_indices)
 172
 173         params = bt2.MapValue()
 174         logging_level = bt2.LoggingLevel.NONE
 175         obj = None
 176
 177         # Compute `params` for this component by piling up params given to all
 178         # AutoSourceComponentSpec objects that contributed in the instantiation
 179         # of this component.
 180         #
 181         # The effective log level for a component is the last one specified
 182         # across the AutoSourceComponentSpec that contributed in its
 183         # instantiation.
 184         for idx in comp_orig_indices:
 185             orig_spec = auto_source_comp_specs[idx]
 186
 187             if orig_spec.params is not None:
 188                 params.update(orig_spec.params)
 189
 190             if orig_spec.logging_level is not None:
 191                 logging_level = orig_spec.logging_level
 192
 193             if orig_spec.obj is not AutoSourceComponentSpec._no_obj:
 194                 obj = orig_spec.obj
 195
 196             used_input_indices.add(int(idx))
 197
 198         params['inputs'] = comp_inputs
 199
 200         comp_specs.append(
 201             ComponentSpec.from_named_plugin_and_component_class(
 202                 plugin_name,
 203                 class_name,
 204                 params=params,
 205                 obj=obj,
 206                 logging_level=logging_level,
 207             )
 208         )
 209
 210     if len(used_input_indices) != len(inputs):
 211         unused_input_indices = set(range(len(inputs))) - used_input_indices
 212         unused_input_indices = sorted(unused_input_indices)
 213         unused_inputs = [str(inputs[x]) for x in unused_input_indices]
 214
 215         msg = (
 216             'Some auto source component specs did not produce any component: '
 217             + ', '.join(unused_inputs)
 218         )
 219         raise RuntimeError(msg)
 220
 221     return comp_specs
 222
 223
 224 # datetime.datetime or integral to nanoseconds
 225 def _get_ns(obj):
 226     if obj is None:
 227         return
 228
 229     if isinstance(obj, numbers.Real):
 230         # consider that it's already in seconds
 231         s = obj
 232     elif isinstance(obj, datetime.datetime):
 233         # s -> ns
 234         s = obj.timestamp()
 235     else:
 236         raise TypeError(
 237             '"{}" is not an integral number or a datetime.datetime object'.format(obj)
 238         )
 239
 240     return int(s * 1e9)
 241
 242
 243 class _TraceCollectionMessageIteratorProxySink(bt2_component._UserSinkComponent):
 244     def __init__(self, config, params, msg_list):
 245         assert type(msg_list) is list
 246         self._msg_list = msg_list
 247         self._add_input_port('in')
 248
 249     def _user_graph_is_configured(self):
 250         self._msg_iter = self._create_message_iterator(self._input_ports['in'])
 251
 252     def _user_consume(self):
 253         assert self._msg_list[0] is None
 254         self._msg_list[0] = next(self._msg_iter)
 255
 256
 257 class TraceCollectionMessageIterator(bt2_message_iterator._MessageIterator):
 258     def __init__(
 259         self,
 260         source_component_specs,
 261         filter_component_specs=None,
 262         stream_intersection_mode=False,
 263         begin=None,
 264         end=None,
 265         plugin_set=None,
 266     ):
 267         utils._check_bool(stream_intersection_mode)
 268         self._stream_intersection_mode = stream_intersection_mode
 269         self._begin_ns = _get_ns(begin)
 270         self._end_ns = _get_ns(end)
 271         self._msg_list = [None]
 272
 273         # If a single item is provided, convert to a list.
 274         if type(source_component_specs) in (
 275             ComponentSpec,
 276             AutoSourceComponentSpec,
 277             str,
 278         ):
 279             source_component_specs = [source_component_specs]
 280
 281         # Convert any string to an AutoSourceComponentSpec.
 282         def str_to_auto(item):
 283             if type(item) is str:
 284                 item = AutoSourceComponentSpec(item)
 285
 286             return item
 287
 288         source_component_specs = [str_to_auto(s) for s in source_component_specs]
 289
 290         if type(filter_component_specs) is ComponentSpec:
 291             filter_component_specs = [filter_component_specs]
 292         elif filter_component_specs is None:
 293             filter_component_specs = []
 294
 295         self._validate_source_component_specs(source_component_specs)
 296         self._validate_filter_component_specs(filter_component_specs)
 297
 298         # Pass any `ComponentSpec` instance as-is.
 299         self._src_comp_specs = [
 300             spec for spec in source_component_specs if type(spec) is ComponentSpec
 301         ]
 302
 303         # Convert any `AutoSourceComponentSpec` in concrete `ComponentSpec` instances.
 304         auto_src_comp_specs = [
 305             spec
 306             for spec in source_component_specs
 307             if type(spec) is AutoSourceComponentSpec
 308         ]
 309         self._src_comp_specs += _auto_discover_source_component_specs(
 310             auto_src_comp_specs, plugin_set
 311         )
 312
 313         self._flt_comp_specs = filter_component_specs
 314         self._next_suffix = 1
 315         self._connect_ports = False
 316
 317         # lists of _ComponentAndSpec
 318         self._src_comps_and_specs = []
 319         self._flt_comps_and_specs = []
 320
 321         self._build_graph()
 322
 323     def _compute_stream_intersections(self):
 324         # Pre-compute the trimmer range to use for each port in the graph, when
 325         # stream intersection mode is enabled.
 326         self._stream_inter_port_to_range = {}
 327
 328         for src_comp_and_spec in self._src_comps_and_specs:
 329             # Query the port's component for the `babeltrace.trace-infos`
 330             # object which contains the range for each stream, from which we can
 331             # compute the intersection of the streams in each trace.
 332             query_exec = bt2.QueryExecutor(
 333                 src_comp_and_spec.spec.component_class,
 334                 'babeltrace.trace-infos',
 335                 src_comp_and_spec.spec.params,
 336             )
 337             trace_infos = query_exec.query()
 338
 339             for trace_info in trace_infos:
 340                 begin = max(
 341                     [
 342                         stream['range-ns']['begin']
 343                         for stream in trace_info['stream-infos']
 344                     ]
 345                 )
 346                 end = min(
 347                     [stream['range-ns']['end'] for stream in trace_info['stream-infos']]
 348                 )
 349
 350                 # Each port associated to this trace will have this computed
 351                 # range.
 352                 for stream in trace_info['stream-infos']:
 353                     # A port name is unique within a component, but not
 354                     # necessarily across all components.  Use a component
 355                     # and port name pair to make it unique across the graph.
 356                     port_name = str(stream['port-name'])
 357                     key = (src_comp_and_spec.comp.addr, port_name)
 358                     self._stream_inter_port_to_range[key] = (begin, end)
 359
 360     def _validate_source_component_specs(self, comp_specs):
 361         for comp_spec in comp_specs:
 362             if (
 363                 type(comp_spec) is not ComponentSpec
 364                 and type(comp_spec) is not AutoSourceComponentSpec
 365             ):
 366                 raise TypeError(
 367                     '"{}" object is not a ComponentSpec or AutoSourceComponentSpec'.format(
 368                         type(comp_spec)
 369                     )
 370                 )
 371
 372     def _validate_filter_component_specs(self, comp_specs):
 373         for comp_spec in comp_specs:
 374             if type(comp_spec) is not ComponentSpec:
 375                 raise TypeError(
 376                     '"{}" object is not a ComponentSpec'.format(type(comp_spec))
 377                 )
 378
 379     def __next__(self):
 380         assert self._msg_list[0] is None
 381         self._graph.run_once()
 382         msg = self._msg_list[0]
 383         assert msg is not None
 384         self._msg_list[0] = None
 385         return msg
 386
 387     def _create_stream_intersection_trimmer(self, component, port):
 388         key = (component.addr, port.name)
 389         begin, end = self._stream_inter_port_to_range[key]
 390         name = 'trimmer-{}-{}'.format(component.name, port.name)
 391         return self._create_trimmer(begin, end, name)
 392
 393     def _create_muxer(self):
 394         plugin = bt2.find_plugin('utils')
 395
 396         if plugin is None:
 397             raise RuntimeError('cannot find "utils" plugin (needed for the muxer)')
 398
 399         if 'muxer' not in plugin.filter_component_classes:
 400             raise RuntimeError(
 401                 'cannot find "muxer" filter component class in "utils" plugin'
 402             )
 403
 404         comp_cls = plugin.filter_component_classes['muxer']
 405         return self._graph.add_component(comp_cls, 'muxer')
 406
 407     def _create_trimmer(self, begin_ns, end_ns, name):
 408         plugin = bt2.find_plugin('utils')
 409
 410         if plugin is None:
 411             raise RuntimeError('cannot find "utils" plugin (needed for the trimmer)')
 412
 413         if 'trimmer' not in plugin.filter_component_classes:
 414             raise RuntimeError(
 415                 'cannot find "trimmer" filter component class in "utils" plugin'
 416             )
 417
 418         params = {}
 419
 420         def ns_to_string(ns):
 421             s_part = ns // 1000000000
 422             ns_part = ns % 1000000000
 423             return '{}.{:09d}'.format(s_part, ns_part)
 424
 425         if begin_ns is not None:
 426             params['begin'] = ns_to_string(begin_ns)
 427
 428         if end_ns is not None:
 429             params['end'] = ns_to_string(end_ns)
 430
 431         comp_cls = plugin.filter_component_classes['trimmer']
 432         return self._graph.add_component(comp_cls, name, params)
 433
 434     def _get_unique_comp_name(self, comp_cls):
 435         name = comp_cls.name
 436         comps_and_specs = itertools.chain(
 437             self._src_comps_and_specs, self._flt_comps_and_specs
 438         )
 439
 440         if name in [comp_and_spec.comp.name for comp_and_spec in comps_and_specs]:
 441             name += '-{}'.format(self._next_suffix)
 442             self._next_suffix += 1
 443
 444         return name
 445
 446     def _create_comp(self, comp_spec):
 447         comp_cls = comp_spec.component_class
 448         name = self._get_unique_comp_name(comp_cls)
 449         comp = self._graph.add_component(
 450             comp_cls, name, comp_spec.params, comp_spec.obj, comp_spec.logging_level
 451         )
 452         return comp
 453
 454     def _get_free_muxer_input_port(self):
 455         for port in self._muxer_comp.input_ports.values():
 456             if not port.is_connected:
 457                 return port
 458
 459     def _connect_src_comp_port(self, component, port):
 460         # if this trace collection iterator is in stream intersection
 461         # mode, we need this connection:
 462         #
 463         #     port -> trimmer -> muxer
 464         #
 465         # otherwise, simply:
 466         #
 467         #     port -> muxer
 468         if self._stream_intersection_mode:
 469             trimmer_comp = self._create_stream_intersection_trimmer(component, port)
 470             self._graph.connect_ports(port, trimmer_comp.input_ports['in'])
 471             port_to_muxer = trimmer_comp.output_ports['out']
 472         else:
 473             port_to_muxer = port
 474
 475         self._graph.connect_ports(port_to_muxer, self._get_free_muxer_input_port())
 476
 477     def _graph_port_added(self, component, port):
 478         if not self._connect_ports:
 479             return
 480
 481         if type(port) is bt2_port._InputPortConst:
 482             return
 483
 484         if component not in [comp.comp for comp in self._src_comps_and_specs]:
 485             # do not care about non-source components (muxer, trimmer, etc.)
 486             return
 487
 488         self._connect_src_comp_port(component, port)
 489
 490     def _get_greatest_operative_mip_version(self):
 491         def append_comp_specs_descriptors(descriptors, comp_specs):
 492             for comp_spec in comp_specs:
 493                 descriptors.append(
 494                     bt2.ComponentDescriptor(
 495                         comp_spec.component_class, comp_spec.params, comp_spec.obj
 496                     )
 497                 )
 498
 499         descriptors = []
 500         append_comp_specs_descriptors(descriptors, self._src_comp_specs)
 501         append_comp_specs_descriptors(descriptors, self._flt_comp_specs)
 502
 503         if self._stream_intersection_mode:
 504             # we also need at least one `flt.utils.trimmer` component
 505             comp_spec = ComponentSpec.from_named_plugin_and_component_class(
 506                 'utils', 'trimmer'
 507             )
 508             append_comp_specs_descriptors(descriptors, [comp_spec])
 509
 510         mip_version = bt2.get_greatest_operative_mip_version(descriptors)
 511
 512         if mip_version is None:
 513             msg = 'failed to find an operative message interchange protocol version (components are not interoperable)'
 514             raise RuntimeError(msg)
 515
 516         return mip_version
 517
 518     def _build_graph(self):
 519         self._graph = bt2.Graph(self._get_greatest_operative_mip_version())
 520         self._graph.add_port_added_listener(self._graph_port_added)
 521         self._muxer_comp = self._create_muxer()
 522
 523         if self._begin_ns is not None or self._end_ns is not None:
 524             trimmer_comp = self._create_trimmer(self._begin_ns, self._end_ns, 'trimmer')
 525             self._graph.connect_ports(
 526                 self._muxer_comp.output_ports['out'], trimmer_comp.input_ports['in']
 527             )
 528             last_flt_out_port = trimmer_comp.output_ports['out']
 529         else:
 530             last_flt_out_port = self._muxer_comp.output_ports['out']
 531
 532         # create extra filter components (chained)
 533         for comp_spec in self._flt_comp_specs:
 534             comp = self._create_comp(comp_spec)
 535             self._flt_comps_and_specs.append(_ComponentAndSpec(comp, comp_spec))
 536
 537         # connect the extra filter chain
 538         for comp_and_spec in self._flt_comps_and_specs:
 539             in_port = list(comp_and_spec.comp.input_ports.values())[0]
 540             out_port = list(comp_and_spec.comp.output_ports.values())[0]
 541             self._graph.connect_ports(last_flt_out_port, in_port)
 542             last_flt_out_port = out_port
 543
 544         # Here we create the components, self._graph_port_added() is
 545         # called when they add ports, but the callback returns early
 546         # because self._connect_ports is False. This is because the
 547         # self._graph_port_added() could not find the associated source
 548         # component specification in self._src_comps_and_specs because
 549         # it does not exist yet (it needs the created component to
 550         # exist).
 551         for comp_spec in self._src_comp_specs:
 552             comp = self._create_comp(comp_spec)
 553             self._src_comps_and_specs.append(_ComponentAndSpec(comp, comp_spec))
 554
 555         if self._stream_intersection_mode:
 556             self._compute_stream_intersections()
 557
 558         # Now we connect the ports which exist at this point. We allow
 559         # self._graph_port_added() to automatically connect _new_ ports.
 560         self._connect_ports = True
 561
 562         for comp_and_spec in self._src_comps_and_specs:
 563             # Keep a separate list because comp_and_spec.output_ports
 564             # could change during the connection of one of its ports.
 565             # Any new port is handled by self._graph_port_added().
 566             out_ports = [port for port in comp_and_spec.comp.output_ports.values()]
 567
 568             for out_port in out_ports:
 569                 if out_port.is_connected:
 570                     continue
 571
 572                 self._connect_src_comp_port(comp_and_spec.comp, out_port)
 573
 574         # Add the proxy sink, passing our message list to share consumed
 575         # messages with this trace collection message iterator.
 576         sink = self._graph.add_component(
 577             _TraceCollectionMessageIteratorProxySink, 'proxy-sink', obj=self._msg_list
 578         )
 579         sink_in_port = sink.input_ports['in']
 580
 581         # connect last filter to proxy sink
 582         self._graph.connect_ports(last_flt_out_port, sink_in_port)