barectf/config.py

   1 # The MIT License (MIT)
   2 #
   3 # Copyright (c) 2015-2016 Philippe Proulx <pproulx@efficios.com>
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  21 # THE SOFTWARE.
  22
  23 from barectf import metadata
  24 import collections
  25 import datetime
  26 import barectf
  27 import enum
  28 import yaml
  29 import uuid
  30 import copy
  31 import re
  32 import os
  33
  34
  35 class _ConfigErrorCtx:
  36     def __init__(self, name, msg=None):
  37         self._name = name
  38         self._msg = msg
  39
  40     @property
  41     def name(self):
  42         return self._name
  43
  44     @property
  45     def msg(self):
  46         return self._msg
  47
  48
  49 class ConfigError(RuntimeError):
  50     def __init__(self, init_ctx_name, init_ctx_msg=None):
  51         self._ctx = []
  52         self.append_ctx(init_ctx_name, init_ctx_msg)
  53
  54     @property
  55     def ctx(self):
  56         return self._ctx
  57
  58     def append_ctx(self, name, msg=None):
  59         self._ctx.append(_ConfigErrorCtx(name, msg))
  60
  61
  62 class Config:
  63     def __init__(self, version, prefix, metadata, options):
  64         self.prefix = prefix
  65         self.version = version
  66         self.metadata = metadata
  67         self.options = options
  68
  69     def _validate_metadata(self, meta):
  70         try:
  71             validator = _MetadataTypesHistologyValidator()
  72             validator.validate(meta)
  73             validator = _MetadataSpecialFieldsValidator()
  74             validator.validate(meta)
  75         except ConfigError as exc:
  76             exc.append_ctx('Metadata')
  77             raise
  78
  79         try:
  80             validator = _BarectfMetadataValidator()
  81             validator.validate(meta)
  82         except ConfigError as exc:
  83             exc.append_ctx('barectf metadata')
  84             raise
  85
  86     def _augment_metadata_env(self, meta):
  87         version_tuple = barectf.get_version_tuple()
  88         base_env = {
  89             'domain': 'bare',
  90             'tracer_name': 'barectf',
  91             'tracer_major': version_tuple[0],
  92             'tracer_minor': version_tuple[1],
  93             'tracer_patch': version_tuple[2],
  94             'barectf_gen_date': str(datetime.datetime.now().isoformat()),
  95         }
  96
  97         base_env.update(meta.env)
  98         meta.env = base_env
  99
 100     @property
 101     def version(self):
 102         return self._version
 103
 104     @version.setter
 105     def version(self, value):
 106         self._version = value
 107
 108     @property
 109     def metadata(self):
 110         return self._metadata
 111
 112     @metadata.setter
 113     def metadata(self, value):
 114         self._validate_metadata(value)
 115         self._augment_metadata_env(value)
 116         self._metadata = value
 117
 118     @property
 119     def prefix(self):
 120         return self._prefix
 121
 122     @prefix.setter
 123     def prefix(self, value):
 124         if not _is_valid_identifier(value):
 125             raise ConfigError('Configuration',
 126                               'Prefix must be a valid C identifier')
 127
 128         self._prefix = value
 129
 130     @property
 131     def options(self):
 132         return self._options
 133
 134     @options.setter
 135     def options(self, options):
 136         self._options = options
 137
 138
 139 class ConfigOptions:
 140     def __init__(self):
 141         self._gen_prefix_def = False
 142         self._gen_default_stream_def = False
 143
 144     @property
 145     def gen_prefix_def(self):
 146         return self._gen_prefix_def
 147
 148     @gen_prefix_def.setter
 149     def gen_prefix_def(self, value):
 150         self._gen_prefix_def = value
 151
 152     @property
 153     def gen_default_stream_def(self):
 154         return self._gen_default_stream_def
 155
 156     @gen_default_stream_def.setter
 157     def gen_default_stream_def(self, value):
 158         self._gen_default_stream_def = value
 159
 160
 161 def _is_assoc_array_prop(node):
 162     return isinstance(node, dict)
 163
 164
 165 def _is_array_prop(node):
 166     return isinstance(node, list)
 167
 168
 169 def _is_int_prop(node):
 170     return type(node) is int
 171
 172
 173 def _is_str_prop(node):
 174     return type(node) is str
 175
 176
 177 def _is_bool_prop(node):
 178     return type(node) is bool
 179
 180
 181 def _is_valid_alignment(align):
 182     return ((align & (align - 1)) == 0) and align > 0
 183
 184
 185 def _byte_order_str_to_bo(bo_str):
 186     bo_str = bo_str.lower()
 187
 188     if bo_str == 'le':
 189         return metadata.ByteOrder.LE
 190     elif bo_str == 'be':
 191         return metadata.ByteOrder.BE
 192
 193
 194 def _encoding_str_to_encoding(encoding_str):
 195     encoding_str = encoding_str.lower()
 196
 197     if encoding_str == 'utf-8' or encoding_str == 'utf8':
 198         return metadata.Encoding.UTF8
 199     elif encoding_str == 'ascii':
 200         return metadata.Encoding.ASCII
 201     elif encoding_str == 'none':
 202         return metadata.Encoding.NONE
 203
 204
 205 _re_iden = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]*$')
 206 _ctf_keywords = set([
 207     'align',
 208     'callsite',
 209     'clock',
 210     'enum',
 211     'env',
 212     'event',
 213     'floating_point',
 214     'integer',
 215     'stream',
 216     'string',
 217     'struct',
 218     'trace',
 219     'typealias',
 220     'typedef',
 221     'variant',
 222 ])
 223
 224
 225 def _is_valid_identifier(iden):
 226     if not _re_iden.match(iden):
 227         return False
 228
 229     if _re_iden in _ctf_keywords:
 230         return False
 231
 232     return True
 233
 234
 235 def _get_first_unknown_prop(node, known_props):
 236     for prop_name in node:
 237         if prop_name in known_props:
 238             continue
 239
 240         return prop_name
 241
 242
 243 # This validator validates the configured metadata for barectf specific
 244 # needs.
 245 #
 246 # barectf needs:
 247 #
 248 #   * all header/contexts are at least byte-aligned
 249 #   * all integer and floating point number sizes to be <= 64
 250 #   * no inner structures or arrays
 251 class _BarectfMetadataValidator:
 252     def __init__(self):
 253         self._type_to_validate_type_func = {
 254             metadata.Integer: self._validate_int_type,
 255             metadata.FloatingPoint: self._validate_float_type,
 256             metadata.Enum: self._validate_enum_type,
 257             metadata.String: self._validate_string_type,
 258             metadata.Struct: self._validate_struct_type,
 259             metadata.Array: self._validate_array_type,
 260         }
 261
 262     def _validate_int_type(self, t, entity_root):
 263         if t.size > 64:
 264             raise ConfigError('Integer type', 'Size must be lesser than or equal to 64 bits')
 265
 266     def _validate_float_type(self, t, entity_root):
 267         if t.size > 64:
 268             raise ConfigError('Floating point number type', 'Size must be lesser than or equal to 64 bits')
 269
 270     def _validate_enum_type(self, t, entity_root):
 271         if t.value_type.size > 64:
 272             raise ConfigError('Enumeration type', 'Integer type\'s size must be lesser than or equal to 64 bits')
 273
 274     def _validate_string_type(self, t, entity_root):
 275         pass
 276
 277     def _validate_struct_type(self, t, entity_root):
 278         if not entity_root:
 279             raise ConfigError('Structure type', 'Inner structure types are not supported as of this version')
 280
 281         for field_name, field_type in t.fields.items():
 282             if entity_root and self._cur_entity is _Entity.TRACE_PACKET_HEADER:
 283                 if field_name == 'uuid':
 284                     # allow
 285                     continue
 286
 287             try:
 288                 self._validate_type(field_type, False)
 289             except ConfigError as exc:
 290                 exc.append_ctx('Structure type\' field "{}"'.format(field_name))
 291                 raise
 292
 293     def _validate_array_type(self, t, entity_root):
 294         raise ConfigError('Array type', 'Not supported as of this version')
 295
 296     def _validate_type(self, t, entity_root):
 297         self._type_to_validate_type_func[type(t)](t, entity_root)
 298
 299     def _validate_entity(self, t):
 300         if t is None:
 301             return
 302
 303         # make sure entity is byte-aligned
 304         if t.align < 8:
 305             raise ConfigError('Root type', 'Alignment must be at least byte-aligned')
 306
 307         # make sure entity is a structure
 308         if type(t) is not metadata.Struct:
 309             raise ConfigError('Root type', 'Expecting a structure type')
 310
 311         # validate types
 312         self._validate_type(t, True)
 313
 314     def _validate_entities_and_names(self, meta):
 315         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 316
 317         try:
 318             self._validate_entity(meta.trace.packet_header_type)
 319         except ConfigError as exc:
 320             exc.append_ctx('Trace', 'Invalid packet header type')
 321             raise
 322
 323         for stream_name, stream in meta.streams.items():
 324             if not _is_valid_identifier(stream_name):
 325                 raise ConfigError('Trace', 'Stream name "{}" is not a valid C identifier'.format(stream_name))
 326
 327             self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 328
 329             try:
 330                 self._validate_entity(stream.packet_context_type)
 331             except ConfigError as exc:
 332                 exc.append_ctx('Stream "{}"'.format(stream_name),
 333                                'Invalid packet context type')
 334                 raise
 335
 336             self._cur_entity = _Entity.STREAM_EVENT_HEADER
 337
 338             try:
 339                 self._validate_entity(stream.event_header_type)
 340             except ConfigError as exc:
 341                 exc.append_ctx('Stream "{}"'.format(stream_name),
 342                                'Invalid event header type')
 343                 raise
 344
 345             self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 346
 347             try:
 348                 self._validate_entity(stream.event_context_type)
 349             except ConfigError as exc:
 350                 exc.append_ctx('Stream "{}"'.format(stream_name),
 351                                'Invalid event context type'.format(stream_name))
 352                 raise
 353
 354             try:
 355                 for ev_name, ev in stream.events.items():
 356                     if not _is_valid_identifier(ev_name):
 357                         raise ConfigError('Stream "{}"'.format(stream_name),
 358                                           'Event name "{}" is not a valid C identifier'.format(ev_name))
 359
 360                     self._cur_entity = _Entity.EVENT_CONTEXT
 361
 362                     try:
 363                         self._validate_entity(ev.context_type)
 364                     except ConfigError as exc:
 365                         exc.append_ctx('Event "{}"'.format(ev_name),
 366                                        'Invalid context type')
 367                         raise
 368
 369                     self._cur_entity = _Entity.EVENT_PAYLOAD
 370
 371                     try:
 372                         self._validate_entity(ev.payload_type)
 373                     except ConfigError as exc:
 374                         exc.append_ctx('Event "{}"'.format(ev_name),
 375                                        'Invalid payload type')
 376                         raise
 377
 378                     if stream.is_event_empty(ev):
 379                         raise ConfigError('Event "{}"'.format(ev_name), 'Empty')
 380             except ConfigError as exc:
 381                 exc.append_ctx('Stream "{}"'.format(stream_name))
 382                 raise
 383
 384     def _validate_default_stream(self, meta):
 385         if meta.default_stream_name:
 386             if meta.default_stream_name not in meta.streams.keys():
 387                 raise ConfigError('barectf metadata', 'Default stream name ("{}") does not exist'.format(meta.default_stream_name))
 388
 389     def validate(self, meta):
 390         self._validate_entities_and_names(meta)
 391         self._validate_default_stream(meta)
 392
 393
 394 # This validator validates special fields of trace, stream, and event
 395 # types. For example, if checks that the "stream_id" field exists in the
 396 # trace packet header if there's more than one stream, and much more.
 397 class _MetadataSpecialFieldsValidator:
 398     def _validate_trace_packet_header_type(self, t):
 399         # needs "stream_id" field?
 400         if len(self._meta.streams) > 1:
 401             # yes
 402             if t is None:
 403                 raise ConfigError('"packet-header-type" property',
 404                                   'Need "stream_id" field (more than one stream), but trace packet header type is missing')
 405
 406             if type(t) is not metadata.Struct:
 407                 raise ConfigError('"packet-header-type" property',
 408                                   'Need "stream_id" field (more than one stream), but trace packet header type is not a structure type')
 409
 410             if 'stream_id' not in t.fields:
 411                 raise ConfigError('"packet-header-type" property',
 412                                   'Need "stream_id" field (more than one stream)')
 413
 414         # validate "magic" and "stream_id" types
 415         if type(t) is not metadata.Struct:
 416             return
 417
 418         for i, (field_name, field_type) in enumerate(t.fields.items()):
 419             if field_name == 'magic':
 420                 if type(field_type) is not metadata.Integer:
 421                     raise ConfigError('"packet-header-type" property',
 422                                       '"magic" field must be an integer type')
 423
 424                 if field_type.signed or field_type.size != 32:
 425                     raise ConfigError('"packet-header-type" property',
 426                                       '"magic" field must be a 32-bit unsigned integer type')
 427
 428                 if i != 0:
 429                     raise ConfigError('"packet-header-type" property',
 430                                       '"magic" field must be the first trace packet header type\'s field')
 431             elif field_name == 'stream_id':
 432                 if type(field_type) is not metadata.Integer:
 433                     raise ConfigError('"packet-header-type" property',
 434                                       '"stream_id" field must be an integer type')
 435
 436                 if field_type.signed:
 437                     raise ConfigError('"packet-header-type" property',
 438                                       '"stream_id" field must be an unsigned integer type')
 439
 440                 # "id" size can fit all event IDs
 441                 if len(self._meta.streams) > (1 << field_type.size):
 442                     raise ConfigError('"packet-header-type" property',
 443                                       '"stream_id" field\' size is too small for the number of trace streams')
 444             elif field_name == 'uuid':
 445                 if self._meta.trace.uuid is None:
 446                     raise ConfigError('"packet-header-type" property',
 447                                       '"uuid" field specified, but no trace UUID provided')
 448
 449                 if type(field_type) is not metadata.Array:
 450                     raise ConfigError('"packet-header-type" property',
 451                                       '"uuid" field must be an array')
 452
 453                 if field_type.length != 16:
 454                     raise ConfigError('"packet-header-type" property',
 455                                       '"uuid" field must be an array of 16 bytes')
 456
 457                 element_type = field_type.element_type
 458
 459                 if type(element_type) is not metadata.Integer:
 460                     raise ConfigError('"packet-header-type" property',
 461                                       '"uuid" field must be an array of 16 unsigned bytes')
 462
 463                 if element_type.size != 8:
 464                     raise ConfigError('"packet-header-type" property',
 465                                       '"uuid" field must be an array of 16 unsigned bytes')
 466
 467                 if element_type.signed:
 468                     raise ConfigError('"packet-header-type" property',
 469                                       '"uuid" field must be an array of 16 unsigned bytes')
 470
 471                 if element_type.align != 8:
 472                     raise ConfigError('"packet-header-type" property',
 473                                       '"uuid" field must be an array of 16 unsigned, byte-aligned bytes')
 474
 475     def _validate_trace(self, meta):
 476         self._validate_trace_packet_header_type(meta.trace.packet_header_type)
 477
 478     def _validate_stream_packet_context(self, stream):
 479         t = stream.packet_context_type
 480
 481         if type(t) is None:
 482             raise ConfigError('Stream',
 483                               'Missing "packet-context-type" property')
 484
 485         if type(t) is not metadata.Struct:
 486             raise ConfigError('"packet-context-type" property',
 487                               'Expecting a structure type')
 488
 489         # "timestamp_begin", if exists, is an unsigned integer type,
 490         # mapped to a clock
 491         ts_begin = None
 492
 493         if 'timestamp_begin' in t.fields:
 494             ts_begin = t.fields['timestamp_begin']
 495
 496             if type(ts_begin) is not metadata.Integer:
 497                 raise ConfigError('"packet-context-type" property',
 498                                   '"timestamp_begin" field must be an integer type')
 499
 500             if ts_begin.signed:
 501                 raise ConfigError('"packet-context-type" property',
 502                                   '"timestamp_begin" field must be an unsigned integer type')
 503
 504             if not ts_begin.property_mappings:
 505                 raise ConfigError('"packet-context-type" property',
 506                                   '"timestamp_begin" field must be mapped to a clock')
 507
 508         # "timestamp_end", if exists, is an unsigned integer type,
 509         # mapped to a clock
 510         ts_end = None
 511
 512         if 'timestamp_end' in t.fields:
 513             ts_end = t.fields['timestamp_end']
 514
 515             if type(ts_end) is not metadata.Integer:
 516                 raise ConfigError('"packet-context-type" property',
 517                                   '"timestamp_end" field must be an integer type')
 518
 519             if ts_end.signed:
 520                 raise ConfigError('"packet-context-type" property',
 521                                   '"timestamp_end" field must be an unsigned integer type')
 522
 523             if not ts_end.property_mappings:
 524                 raise ConfigError('"packet-context-type" property',
 525                                   '"timestamp_end" field must be mapped to a clock')
 526
 527         # "timestamp_begin" and "timestamp_end" exist together
 528         if (('timestamp_begin' in t.fields) ^ ('timestamp_end' in t.fields)):
 529             raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be defined together in stream packet context type')
 530
 531         # "timestamp_begin" and "timestamp_end" are mapped to the same clock
 532         if ts_begin is not None and ts_end is not None:
 533             if ts_begin.property_mappings[0].object.name != ts_end.property_mappings[0].object.name:
 534                 raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be mapped to the same clock object in stream packet context type')
 535
 536         # "events_discarded", if exists, is an unsigned integer type
 537         if 'events_discarded' in t.fields:
 538             events_discarded = t.fields['events_discarded']
 539
 540             if type(events_discarded) is not metadata.Integer:
 541                 raise ConfigError('"packet-context-type" property',
 542                                   '"events_discarded" field must be an integer type')
 543
 544             if events_discarded.signed:
 545                 raise ConfigError('"packet-context-type" property',
 546                                   '"events_discarded" field must be an unsigned integer type')
 547
 548         # "packet_size" and "content_size" must exist
 549         if 'packet_size' not in t.fields:
 550             raise ConfigError('"packet-context-type" property',
 551                               'Missing "packet_size" field in stream packet context type')
 552
 553         packet_size = t.fields['packet_size']
 554
 555         # "content_size" and "content_size" must exist
 556         if 'content_size' not in t.fields:
 557             raise ConfigError('"packet-context-type" property',
 558                               'Missing "content_size" field in stream packet context type')
 559
 560         content_size = t.fields['content_size']
 561
 562         # "packet_size" is an unsigned integer type
 563         if type(packet_size) is not metadata.Integer:
 564             raise ConfigError('"packet-context-type" property',
 565                               '"packet_size" field in stream packet context type must be an integer type')
 566
 567         if packet_size.signed:
 568             raise ConfigError('"packet-context-type" property',
 569                               '"packet_size" field in stream packet context type must be an unsigned integer type')
 570
 571         # "content_size" is an unsigned integer type
 572         if type(content_size) is not metadata.Integer:
 573             raise ConfigError('"packet-context-type" property',
 574                               '"content_size" field in stream packet context type must be an integer type')
 575
 576         if content_size.signed:
 577             raise ConfigError('"packet-context-type" property',
 578                               '"content_size" field in stream packet context type must be an unsigned integer type')
 579
 580         # "packet_size" size should be greater than or equal to "content_size" size
 581         if content_size.size > packet_size.size:
 582             raise ConfigError('"packet-context-type" property',
 583                               '"content_size" field size must be lesser than or equal to "packet_size" field size')
 584
 585     def _validate_stream_event_header(self, stream):
 586         t = stream.event_header_type
 587
 588         # needs "id" field?
 589         if len(stream.events) > 1:
 590             # yes
 591             if t is None:
 592                 raise ConfigError('"event-header-type" property',
 593                                   'Need "id" field (more than one event), but stream event header type is missing')
 594
 595             if type(t) is not metadata.Struct:
 596                 raise ConfigError('"event-header-type" property',
 597                                   'Need "id" field (more than one event), but stream event header type is not a structure type')
 598
 599             if 'id' not in t.fields:
 600                 raise ConfigError('"event-header-type" property',
 601                                   'Need "id" field (more than one event)')
 602
 603         # validate "id" and "timestamp" types
 604         if type(t) is not metadata.Struct:
 605             return
 606
 607         # "timestamp", if exists, is an unsigned integer type,
 608         # mapped to a clock
 609         if 'timestamp' in t.fields:
 610             ts = t.fields['timestamp']
 611
 612             if type(ts) is not metadata.Integer:
 613                 raise ConfigError('"event-header-type" property',
 614                                   '"timestamp" field must be an integer type')
 615
 616             if ts.signed:
 617                 raise ConfigError('"event-header-type" property',
 618                                   '"timestamp" field must be an unsigned integer type')
 619
 620             if not ts.property_mappings:
 621                 raise ConfigError('"event-header-type" property',
 622                                   '"timestamp" field must be mapped to a clock')
 623
 624         if 'id' in t.fields:
 625             eid = t.fields['id']
 626
 627             # "id" is an unsigned integer type
 628             if type(eid) is not metadata.Integer:
 629                 raise ConfigError('"event-header-type" property',
 630                                   '"id" field must be an integer type')
 631
 632             if eid.signed:
 633                 raise ConfigError('"event-header-type" property',
 634                                   '"id" field must be an unsigned integer type')
 635
 636             # "id" size can fit all event IDs
 637             if len(stream.events) > (1 << eid.size):
 638                 raise ConfigError('"event-header-type" property',
 639                                   '"id" field\' size is too small for the number of stream events')
 640
 641     def _validate_stream(self, stream):
 642         self._validate_stream_packet_context(stream)
 643         self._validate_stream_event_header(stream)
 644
 645     def validate(self, meta):
 646         self._meta = meta
 647         self._validate_trace(meta)
 648
 649         for stream in meta.streams.values():
 650             try:
 651                 self._validate_stream(stream)
 652             except ConfigError as exc:
 653                 exc.append_ctx('Stream "{}"'.format(stream.name), 'Invalid')
 654                 raise
 655
 656
 657 # Entities. Order of values is important here.
 658 @enum.unique
 659 class _Entity(enum.IntEnum):
 660     TRACE_PACKET_HEADER = 0
 661     STREAM_PACKET_CONTEXT = 1
 662     STREAM_EVENT_HEADER = 2
 663     STREAM_EVENT_CONTEXT = 3
 664     EVENT_CONTEXT = 4
 665     EVENT_PAYLOAD = 5
 666
 667
 668 # Since type inheritance allows types to be only partially defined at
 669 # any place in the configuration, this validator validates that actual
 670 # trace, stream, and event types are all complete and valid. Therefore
 671 # an invalid, but unusued type alias is accepted.
 672 class _MetadataTypesHistologyValidator:
 673     def __init__(self):
 674         self._type_to_validate_type_histology_func = {
 675             metadata.Integer: self._validate_integer_histology,
 676             metadata.FloatingPoint: self._validate_float_histology,
 677             metadata.Enum: self._validate_enum_histology,
 678             metadata.String: self._validate_string_histology,
 679             metadata.Struct: self._validate_struct_histology,
 680             metadata.Array: self._validate_array_histology,
 681         }
 682
 683     def _validate_integer_histology(self, t):
 684         # size is set
 685         if t.size is None:
 686             raise ConfigError('Integer type', 'Missing size')
 687
 688     def _validate_float_histology(self, t):
 689         # exponent digits is set
 690         if t.exp_size is None:
 691             raise ConfigError('Floating point number type',
 692                               'Missing exponent size')
 693
 694         # mantissa digits is set
 695         if t.mant_size is None:
 696             raise ConfigError('Floating point number type',
 697                               'Missing mantissa size')
 698
 699         # exponent and mantissa sum is a multiple of 8
 700         if (t.exp_size + t.mant_size) % 8 != 0:
 701             raise ConfigError('Floating point number type',
 702                               'Mantissa and exponent sizes sum must be a multiple of 8')
 703
 704     def _validate_enum_histology(self, t):
 705         # integer type is set
 706         if t.value_type is None:
 707             raise ConfigError('Enumeration type', 'Missing value type')
 708
 709         # there's at least one member
 710         if not t.members:
 711             raise ConfigError('Enumeration type', 'At least one member required')
 712
 713         # no overlapping values and all values are valid considering
 714         # the value type
 715         ranges = []
 716
 717         if t.value_type.signed:
 718             value_min = -(1 << t.value_type.size - 1)
 719             value_max = (1 << (t.value_type.size - 1)) - 1
 720         else:
 721             value_min = 0
 722             value_max = (1 << t.value_type.size) - 1
 723
 724         for label, value in t.members.items():
 725             for rg in ranges:
 726                 if value[0] <= rg[1] and rg[0] <= value[1]:
 727                     raise ConfigError('Enumeration type\'s member "{}"',
 728                                       'Overlaps another member'.format(label))
 729
 730             name_fmt = 'Enumeration type\'s member "{}"'
 731             msg_fmt = 'Value {} is outside the value type range [{}, {}]'
 732
 733             if value[0] < value_min or value[0] > value_max:
 734                 raise ConfigError(name_fmt.format(label),
 735                                   msg_fmt.format(value[0], value_min, value_max))
 736
 737             if value[1] < value_min or value[1] > value_max:
 738                 raise ConfigError(name_fmt.format(label),
 739                                   msg_fmt.format(value[0], value_min, value_max))
 740
 741             ranges.append(value)
 742
 743     def _validate_string_histology(self, t):
 744         # always valid
 745         pass
 746
 747     def _validate_struct_histology(self, t):
 748         # all fields are valid
 749         for field_name, field_type in t.fields.items():
 750             try:
 751                 self._validate_type_histology(field_type)
 752             except ConfigError as exc:
 753                 exc.append_ctx('Structure type\'s field "{}"'.format(field_name))
 754                 raise
 755
 756     def _validate_array_histology(self, t):
 757         # length is set
 758         if t.length is None:
 759             raise ConfigError('Array type', 'Missing length')
 760
 761         # element type is set
 762         if t.element_type is None:
 763             raise ConfigError('Array type', 'Missing element type')
 764
 765         # element type is valid
 766         try:
 767             self._validate_type_histology(t.element_type)
 768         except ConfigError as exc:
 769             exc.append_ctx('Array type', 'Invalid element type')
 770             raise
 771
 772     def _validate_type_histology(self, t):
 773         if t is None:
 774             return
 775
 776         self._type_to_validate_type_histology_func[type(t)](t)
 777
 778     def _validate_entity_type_histology(self, t):
 779         if t is None:
 780             return
 781
 782         if type(t) is not metadata.Struct:
 783             raise ConfigError('Root type', 'Expecting a structure type')
 784
 785         self._validate_type_histology(t)
 786
 787     def _validate_event_types_histology(self, ev):
 788         ev_name = ev.name
 789
 790         # validate event context type
 791         try:
 792             self._validate_entity_type_histology(ev.context_type)
 793         except ConfigError as exc:
 794             exc.append_ctx('Event "{}"'.format(ev.name),
 795                            'Invalid context type')
 796             raise
 797
 798         # validate event payload type
 799         try:
 800             self._validate_entity_type_histology(ev.payload_type)
 801         except ConfigError as exc:
 802             exc.append_ctx('Event "{}"'.format(ev.name),
 803                            'Invalid payload type')
 804             raise
 805
 806     def _validate_stream_types_histology(self, stream):
 807         stream_name = stream.name
 808
 809         # validate stream packet context type
 810         try:
 811             self._validate_entity_type_histology(stream.packet_context_type)
 812         except ConfigError as exc:
 813             exc.append_ctx('Stream "{}"'.format(stream_name),
 814                            'Invalid packet context type')
 815             raise
 816
 817         # validate stream event header type
 818         try:
 819             self._validate_entity_type_histology(stream.event_header_type)
 820         except ConfigError as exc:
 821             exc.append_ctx('Stream "{}"'.format(stream_name),
 822                            'Invalid event header type')
 823             raise
 824
 825         # validate stream event context type
 826         try:
 827             self._validate_entity_type_histology(stream.event_context_type)
 828         except ConfigError as exc:
 829             exc.append_ctx('Stream "{}"'.format(stream_name),
 830                            'Invalid event context type')
 831             raise
 832
 833         # validate events
 834         for ev in stream.events.values():
 835             try:
 836                 self._validate_event_types_histology(ev)
 837             except ConfigError as exc:
 838                 exc.append_ctx('Stream "{}"'.format(stream_name),
 839                                'Invalid event')
 840                 raise
 841
 842     def validate(self, meta):
 843         # validate trace packet header type
 844         try:
 845             self._validate_entity_type_histology(meta.trace.packet_header_type)
 846         except ConfigError as exc:
 847             exc.append_ctx('Metadata\'s trace', 'Invalid packet header type')
 848             raise
 849
 850         # validate streams
 851         for stream in meta.streams.values():
 852             self._validate_stream_types_histology(stream)
 853
 854
 855 class _YamlConfigParser:
 856     def __init__(self, include_dirs, ignore_include_not_found, dump_config):
 857         self._class_name_to_create_type_func = {
 858             'int': self._create_integer,
 859             'integer': self._create_integer,
 860             'flt': self._create_float,
 861             'float': self._create_float,
 862             'floating-point': self._create_float,
 863             'enum': self._create_enum,
 864             'enumeration': self._create_enum,
 865             'str': self._create_string,
 866             'string': self._create_string,
 867             'struct': self._create_struct,
 868             'structure': self._create_struct,
 869             'array': self._create_array,
 870         }
 871         self._type_to_create_type_func = {
 872             metadata.Integer: self._create_integer,
 873             metadata.FloatingPoint: self._create_float,
 874             metadata.Enum: self._create_enum,
 875             metadata.String: self._create_string,
 876             metadata.Struct: self._create_struct,
 877             metadata.Array: self._create_array,
 878         }
 879         self._include_dirs = include_dirs
 880         self._ignore_include_not_found = ignore_include_not_found
 881         self._dump_config = dump_config
 882
 883     def _set_byte_order(self, metadata_node):
 884         if 'trace' not in metadata_node:
 885             raise ConfigError('Metadata', 'Missing "trace" property')
 886
 887         trace_node = metadata_node['trace']
 888
 889         if not _is_assoc_array_prop(trace_node):
 890             raise ConfigError('Metadata\'s "trace" property',
 891                               'Must be an associative array')
 892
 893         if 'byte-order' not in trace_node:
 894             raise ConfigError('Metadata\'s "trace" property',
 895                               'Missing "byte-order" property')
 896
 897         bo_node = trace_node['byte-order']
 898
 899         if not _is_str_prop(bo_node):
 900             raise ConfigError('Metadata\'s "trace" property',
 901                               '"byte-order" property must be a string ("le" or "be")')
 902
 903         self._bo = _byte_order_str_to_bo(bo_node)
 904
 905         if self._bo is None:
 906             raise ConfigError('Metadata\'s "trace" property',
 907                               'Invalid "byte-order" property: must be "le" or "be"')
 908
 909     def _lookup_type_alias(self, name):
 910         if name in self._tas:
 911             return copy.deepcopy(self._tas[name])
 912
 913     def _set_int_clock_prop_mapping(self, int_obj, prop_mapping_node):
 914         unk_prop = _get_first_unknown_prop(prop_mapping_node, ['type', 'name', 'property'])
 915
 916         if unk_prop:
 917             raise ConfigError('Integer type\'s clock property mapping',
 918                               'Unknown property: "{}"'.format(unk_prop))
 919
 920         if 'name' not in prop_mapping_node:
 921             raise ConfigError('Integer type\'s clock property mapping',
 922                               'Missing "name" property')
 923
 924         if 'property' not in prop_mapping_node:
 925             raise ConfigError('Integer type\'s clock property mapping',
 926                               'Missing "property" property')
 927
 928         clock_name = prop_mapping_node['name']
 929         prop = prop_mapping_node['property']
 930
 931         if not _is_str_prop(clock_name):
 932             raise ConfigError('Integer type\'s clock property mapping',
 933                               '"name" property must be a string')
 934
 935         if not _is_str_prop(prop):
 936             raise ConfigError('Integer type\'s clock property mapping',
 937                               '"property" property must be a string')
 938
 939         if clock_name not in self._clocks:
 940             raise ConfigError('Integer type\'s clock property mapping',
 941                               'Invalid clock name "{}"'.format(clock_name))
 942
 943         if prop != 'value':
 944             raise ConfigError('Integer type\'s clock property mapping',
 945                               'Invalid "property" property: "{}"'.format(prop))
 946
 947         mapped_clock = self._clocks[clock_name]
 948         int_obj.property_mappings.append(metadata.PropertyMapping(mapped_clock, prop))
 949
 950     def _get_first_unknown_type_prop(self, type_node, known_props):
 951         kp = known_props + ['inherit', 'class']
 952
 953         if self._version >= 201:
 954             kp.append('$inherit')
 955
 956         return _get_first_unknown_prop(type_node, kp)
 957
 958     def _create_integer(self, obj, node):
 959         if obj is None:
 960             # create integer object
 961             obj = metadata.Integer()
 962
 963         unk_prop = self._get_first_unknown_type_prop(node, [
 964             'size',
 965             'align',
 966             'signed',
 967             'byte-order',
 968             'base',
 969             'encoding',
 970             'property-mappings',
 971         ])
 972
 973         if unk_prop:
 974             raise ConfigError('Integer type',
 975                               'Unknown property: "{}"'.format(unk_prop))
 976
 977         # size
 978         if 'size' in node:
 979             size = node['size']
 980
 981             if not _is_int_prop(size):
 982                 raise ConfigError('Integer type',
 983                                   '"size" property of integer type object must be an integer')
 984
 985             if size < 1:
 986                 raise ConfigError('Integer type',
 987                                   'Invalid integer size: {}'.format(size))
 988
 989             obj.size = size
 990
 991         # align
 992         if 'align' in node:
 993             align = node['align']
 994
 995             if align is None:
 996                 obj.set_default_align()
 997             else:
 998                 if not _is_int_prop(align):
 999                     raise ConfigError('Integer type',
1000                                       '"align" property of integer type object must be an integer')
1001
1002                 if not _is_valid_alignment(align):
1003                     raise ConfigError('Integer type',
1004                                       'Invalid alignment: {}'.format(align))
1005
1006                 obj.align = align
1007
1008         # signed
1009         if 'signed' in node:
1010             signed = node['signed']
1011
1012             if signed is None:
1013                 obj.set_default_signed()
1014             else:
1015                 if not _is_bool_prop(signed):
1016                     raise ConfigError('Integer type',
1017                                       '"signed" property of integer type object must be a boolean')
1018
1019                 obj.signed = signed
1020
1021         # byte order
1022         if 'byte-order' in node:
1023             byte_order = node['byte-order']
1024
1025             if byte_order is None:
1026                 obj.byte_order = self._bo
1027             else:
1028                 if not _is_str_prop(byte_order):
1029                     raise ConfigError('Integer type',
1030                                       '"byte-order" property of integer type object must be a string ("le" or "be")')
1031
1032                 byte_order = _byte_order_str_to_bo(byte_order)
1033
1034                 if byte_order is None:
1035                     raise ConfigError('Integer type',
1036                                       'Invalid "byte-order" property in integer type object')
1037
1038                 obj.byte_order = byte_order
1039         else:
1040             obj.byte_order = self._bo
1041
1042         # base
1043         if 'base' in node:
1044             base = node['base']
1045
1046             if base is None:
1047                 obj.set_default_base()
1048             else:
1049                 if not _is_str_prop(base):
1050                     raise ConfigError('Integer type',
1051                                       '"base" property of integer type object must be a string ("bin", "oct", "dec", or "hex")')
1052
1053                 if base == 'bin':
1054                     base = 2
1055                 elif base == 'oct':
1056                     base = 8
1057                 elif base == 'dec':
1058                     base = 10
1059                 elif base == 'hex':
1060                     base = 16
1061                 else:
1062                     raise ConfigError('Integer type',
1063                                       'Unknown "base" property value: "{}" ("bin", "oct", "dec", and "hex" are accepted)'.format(base))
1064
1065                 obj.base = base
1066
1067         # encoding
1068         if 'encoding' in node:
1069             encoding = node['encoding']
1070
1071             if encoding is None:
1072                 obj.set_default_encoding()
1073             else:
1074                 if not _is_str_prop(encoding):
1075                     raise ConfigError('Integer type',
1076                                       '"encoding" property of integer type object must be a string ("none", "ascii", or "utf-8")')
1077
1078                 encoding = _encoding_str_to_encoding(encoding)
1079
1080                 if encoding is None:
1081                     raise ConfigError('Integer type',
1082                                       'Invalid "encoding" property in integer type object')
1083
1084                 obj.encoding = encoding
1085
1086         # property mappings
1087         if 'property-mappings' in node:
1088             prop_mappings = node['property-mappings']
1089
1090             if prop_mappings is None:
1091                 obj.set_default_property_mappings()
1092             else:
1093                 if not _is_array_prop(prop_mappings):
1094                     raise ConfigError('Integer type',
1095                                       '"property-mappings" property of integer type object must be an array')
1096
1097                 if len(prop_mappings) > 1:
1098                     raise ConfigError('Integer type',
1099                                       'Length of "property-mappings" array in integer type object must be 1')
1100
1101                 for index, prop_mapping in enumerate(prop_mappings):
1102                     if not _is_assoc_array_prop(prop_mapping):
1103                         raise ConfigError('Integer type',
1104                                           'Elements of "property-mappings" property of integer type object must be associative arrays')
1105
1106                     if 'type' not in prop_mapping:
1107                         raise ConfigError('Integer type',
1108                                           'Missing "type" property in integer type object\'s "property-mappings" array\'s element #{}'.format(index))
1109
1110                     prop_type = prop_mapping['type']
1111
1112                     if not _is_str_prop(prop_type):
1113                         raise ConfigError('Integer type',
1114                                           '"type" property of integer type object\'s "property-mappings" array\'s element #{} must be a string'.format(index))
1115
1116                     if prop_type == 'clock':
1117                         self._set_int_clock_prop_mapping(obj, prop_mapping)
1118                     else:
1119                         raise ConfigError('Integer type',
1120                                           'Unknown property mapping type "{}" in integer type object\'s "property-mappings" array\'s element #{}'.format(prop_type, index))
1121
1122         return obj
1123
1124     def _create_float(self, obj, node):
1125         if obj is None:
1126             # create floating point number object
1127             obj = metadata.FloatingPoint()
1128
1129         unk_prop = self._get_first_unknown_type_prop(node, [
1130             'size',
1131             'align',
1132             'byte-order',
1133         ])
1134
1135         if unk_prop:
1136             raise ConfigError('Floating point number type',
1137                               'Unknown property: "{}"'.format(unk_prop))
1138
1139         # size
1140         if 'size' in node:
1141             size = node['size']
1142
1143             if not _is_assoc_array_prop(size):
1144                 raise ConfigError('Floating point number type',
1145                                   '"size" property must be an associative array')
1146
1147             unk_prop = _get_first_unknown_prop(size, ['exp', 'mant'])
1148
1149             if unk_prop:
1150                 raise ConfigError('Floating point number type\'s "size" property',
1151                                   'Unknown property: "{}"'.format(unk_prop))
1152
1153             if 'exp' in size:
1154                 exp = size['exp']
1155
1156                 if not _is_int_prop(exp):
1157                     raise ConfigError('Floating point number type\'s "size" property',
1158                                       '"exp" property must be an integer')
1159
1160                 if exp < 1:
1161                     raise ConfigError('Floating point number type\'s "size" property',
1162                                       'Invalid exponent size: {}')
1163
1164                 obj.exp_size = exp
1165
1166             if 'mant' in size:
1167                 mant = size['mant']
1168
1169                 if not _is_int_prop(mant):
1170                     raise ConfigError('Floating point number type\'s "size" property',
1171                                       '"mant" property must be an integer')
1172
1173                 if mant < 1:
1174                     raise ConfigError('Floating point number type\'s "size" property',
1175                                       'Invalid mantissa size: {}')
1176
1177                 obj.mant_size = mant
1178
1179         # align
1180         if 'align' in node:
1181             align = node['align']
1182
1183             if align is None:
1184                 obj.set_default_align()
1185             else:
1186                 if not _is_int_prop(align):
1187                     raise ConfigError('Floating point number type',
1188                                       '"align" property must be an integer')
1189
1190                 if not _is_valid_alignment(align):
1191                     raise ConfigError('Floating point number type',
1192                                       'Invalid alignment: {}'.format(align))
1193
1194                 obj.align = align
1195
1196         # byte order
1197         if 'byte-order' in node:
1198             byte_order = node['byte-order']
1199
1200             if byte_order is None:
1201                 obj.byte_order = self._bo
1202             else:
1203                 if not _is_str_prop(byte_order):
1204                     raise ConfigError('Floating point number type',
1205                                       '"byte-order" property must be a string ("le" or "be")')
1206
1207                 byte_order = _byte_order_str_to_bo(byte_order)
1208
1209                 if byte_order is None:
1210                     raise ConfigError('Floating point number type',
1211                                       'Invalid "byte-order" property')
1212         else:
1213             obj.byte_order = self._bo
1214
1215         return obj
1216
1217     def _create_enum(self, obj, node):
1218         if obj is None:
1219             # create enumeration object
1220             obj = metadata.Enum()
1221
1222         unk_prop = self._get_first_unknown_type_prop(node, [
1223             'value-type',
1224             'members',
1225         ])
1226
1227         if unk_prop:
1228             raise ConfigError('Enumeration type',
1229                               'Unknown property: "{}"'.format(unk_prop))
1230
1231         # value type
1232         if 'value-type' in node:
1233             value_type_node = node['value-type']
1234
1235             try:
1236                 obj.value_type = self._create_type(value_type_node)
1237             except ConfigError as exc:
1238                 exc.append_ctx('Enumeration type', 'Cannot create integer type')
1239                 raise
1240
1241         # members
1242         if 'members' in node:
1243             members_node = node['members']
1244
1245             if not _is_array_prop(members_node):
1246                 raise ConfigError('Enumeration type',
1247                                   '"members" property must be an array')
1248
1249             cur = 0
1250             last_value = obj.last_value
1251
1252             if last_value is None:
1253                 cur = 0
1254             else:
1255                 cur = last_value + 1
1256
1257             for index, m_node in enumerate(members_node):
1258                 if not _is_str_prop(m_node) and not _is_assoc_array_prop(m_node):
1259                     raise ConfigError('Enumeration type',
1260                                       'Invalid member #{}: expecting a string or an associative array'.format(index))
1261
1262                 if _is_str_prop(m_node):
1263                     label = m_node
1264                     value = (cur, cur)
1265                     cur += 1
1266                 else:
1267                     unk_prop = _get_first_unknown_prop(m_node, [
1268                         'label',
1269                         'value',
1270                     ])
1271
1272                     if unk_prop:
1273                         raise ConfigError('Enumeration type',
1274                                           'Unknown member object property: "{}"'.format(unk_prop))
1275
1276                     if 'label' not in m_node:
1277                         raise ConfigError('Enumeration type',
1278                                           'Missing "label" property in member #{}'.format(index))
1279
1280                     label = m_node['label']
1281
1282                     if not _is_str_prop(label):
1283                         raise ConfigError('Enumeration type',
1284                                           '"label" property of member #{} must be a string'.format(index))
1285
1286                     if 'value' not in m_node:
1287                         raise ConfigError('Enumeration type',
1288                                           'Missing "value" property in member ("{}")'.format(label))
1289
1290                     value = m_node['value']
1291
1292                     if not _is_int_prop(value) and not _is_array_prop(value):
1293                         raise ConfigError('Enumeration type',
1294                                           'Invalid member ("{}"): expecting an integer or an array'.format(label))
1295
1296                     if _is_int_prop(value):
1297                         cur = value + 1
1298                         value = (value, value)
1299                     else:
1300                         if len(value) != 2:
1301                             raise ConfigError('Enumeration type',
1302                                               'Invalid member ("{}"): range must have exactly two items'.format(label))
1303
1304                         mn = value[0]
1305                         mx = value[1]
1306
1307                         if mn > mx:
1308                             raise ConfigError('Enumeration type',
1309                                               'Invalid member ("{}"): invalid range ({} > {})'.format(label, mn, mx))
1310
1311                         value = (mn, mx)
1312                         cur = mx + 1
1313
1314                 obj.members[label] = value
1315
1316         return obj
1317
1318     def _create_string(self, obj, node):
1319         if obj is None:
1320             # create string object
1321             obj = metadata.String()
1322
1323         unk_prop = self._get_first_unknown_type_prop(node, [
1324             'encoding',
1325         ])
1326
1327         if unk_prop:
1328             raise ConfigError('String type',
1329                               'Unknown object property: "{}"'.format(unk_prop))
1330
1331         # encoding
1332         if 'encoding' in node:
1333             encoding = node['encoding']
1334
1335             if encoding is None:
1336                 obj.set_default_encoding()
1337             else:
1338                 if not _is_str_prop(encoding):
1339                     raise ConfigError('String type',
1340                                       '"encoding" property of must be a string ("none", "ascii", or "utf-8")')
1341
1342                 encoding = _encoding_str_to_encoding(encoding)
1343
1344                 if encoding is None:
1345                     raise ConfigError('String type',
1346                                       'Invalid "encoding" property')
1347
1348                 obj.encoding = encoding
1349
1350         return obj
1351
1352     def _create_struct(self, obj, node):
1353         if obj is None:
1354             # create structure object
1355             obj = metadata.Struct()
1356
1357         unk_prop = self._get_first_unknown_type_prop(node, [
1358             'min-align',
1359             'fields',
1360         ])
1361
1362         if unk_prop:
1363             raise ConfigError('Structure type',
1364                               'Unknown object property: "{}"'.format(unk_prop))
1365
1366         # minimum alignment
1367         if 'min-align' in node:
1368             min_align = node['min-align']
1369
1370             if min_align is None:
1371                 obj.set_default_min_align()
1372             else:
1373                 if not _is_int_prop(min_align):
1374                     raise ConfigError('Structure type',
1375                                       '"min-align" property must be an integer')
1376
1377                 if not _is_valid_alignment(min_align):
1378                     raise ConfigError('Structure type',
1379                                       'Invalid minimum alignment: {}'.format(min_align))
1380
1381                 obj.min_align = min_align
1382
1383         # fields
1384         if 'fields' in node:
1385             fields = node['fields']
1386
1387             if fields is None:
1388                 obj.set_default_fields()
1389             else:
1390                 if not _is_assoc_array_prop(fields):
1391                     raise ConfigError('Structure type',
1392                                       '"fields" property must be an associative array')
1393
1394                 for field_name, field_node in fields.items():
1395                     if not _is_valid_identifier(field_name):
1396                         raise ConfigError('Structure type',
1397                                           '"{}" is not a valid field name'.format(field_name))
1398
1399                     try:
1400                         obj.fields[field_name] = self._create_type(field_node)
1401                     except ConfigError as exc:
1402                         exc.append_ctx('Structure type',
1403                                        'Cannot create field "{}"'.format(field_name))
1404                         raise
1405
1406         return obj
1407
1408     def _create_array(self, obj, node):
1409         if obj is None:
1410             # create array object
1411             obj = metadata.Array()
1412
1413         unk_prop = self._get_first_unknown_type_prop(node, [
1414             'length',
1415             'element-type',
1416         ])
1417
1418         if unk_prop:
1419             raise ConfigError('Array type',
1420                               'Unknown property: "{}"'.format(unk_prop))
1421
1422         # length
1423         if 'length' in node:
1424             length = node['length']
1425
1426             if not _is_int_prop(length):
1427                 raise ConfigError('Array type',
1428                                   '"length" property must be an integer')
1429
1430             if type(length) is int and length < 0:
1431                 raise ConfigError('Array type',
1432                                   'Invalid length: {}'.format(length))
1433
1434             obj.length = length
1435
1436         # element type
1437         if 'element-type' in node:
1438             element_type_node = node['element-type']
1439
1440             try:
1441                 obj.element_type = self._create_type(node['element-type'])
1442             except ConfigError as exc:
1443                 exc.append_ctx('Array type', 'Cannot create element type')
1444                 raise
1445
1446         return obj
1447
1448     def _create_type(self, type_node):
1449         if type(type_node) is str:
1450             t = self._lookup_type_alias(type_node)
1451
1452             if t is None:
1453                 raise ConfigError('Type',
1454                                   'Unknown type alias "{}"'.format(type_node))
1455
1456             return t
1457
1458         if not _is_assoc_array_prop(type_node):
1459             raise ConfigError('Type',
1460                               'Expecting associative arrays or string (type alias name)')
1461
1462         # inherit:
1463         #   v2.0:  "inherit"
1464         #   v2.1+: "$inherit"
1465         inherit_node = None
1466
1467         if self._version >= 200:
1468             if 'inherit' in type_node:
1469                 inherit_prop = 'inherit'
1470                 inherit_node = type_node[inherit_prop]
1471
1472         if self._version >= 201:
1473             if '$inherit' in type_node:
1474                 if inherit_node is not None:
1475                     raise ConfigError('Type',
1476                                       'Cannot specify both "inherit" and "$inherit" properties of type object: prefer "$inherit"')
1477
1478                 inherit_prop = '$inherit'
1479                 inherit_node = type_node[inherit_prop]
1480
1481         if inherit_node is not None and 'class' in type_node:
1482             raise ConfigError('Type',
1483                               'Cannot specify both "{}" and "class" properties in type object'.format(inherit_prop))
1484
1485         if inherit_node is not None:
1486             if not _is_str_prop(inherit_node):
1487                 raise ConfigError('Type',
1488                                   '"{}" property of type object must be a string'.format(inherit_prop))
1489
1490             base = self._lookup_type_alias(inherit_node)
1491
1492             if base is None:
1493                 raise ConfigError('Type',
1494                                   'Cannot inherit from type alias "{}": type alias does not exist at this point'.format(inherit_node))
1495
1496             func = self._type_to_create_type_func[type(base)]
1497         else:
1498             if 'class' not in type_node:
1499                 raise ConfigError('Type',
1500                                   'Does not inherit, therefore must have a "class" property')
1501
1502             class_name = type_node['class']
1503
1504             if type(class_name) is not str:
1505                 raise ConfigError('Type', '"class" property must be a string')
1506
1507             if class_name not in self._class_name_to_create_type_func:
1508                 raise ConfigError('Type',
1509                                   'Unknown class "{}"'.format(class_name))
1510
1511             base = None
1512             func = self._class_name_to_create_type_func[class_name]
1513
1514         return func(base, type_node)
1515
1516     def _register_type_aliases(self, metadata_node):
1517         self._tas = dict()
1518
1519         if 'type-aliases' not in metadata_node:
1520             return
1521
1522         ta_node = metadata_node['type-aliases']
1523
1524         if ta_node is None:
1525             return
1526
1527         if not _is_assoc_array_prop(ta_node):
1528             raise ConfigError('Metadata',
1529                               '"type-aliases" property must be an associative array')
1530
1531         for ta_name, ta_type in ta_node.items():
1532             if ta_name in self._tas:
1533                 raise ConfigError('Metadata',
1534                                   'Duplicate type alias "{}"'.format(ta_name))
1535
1536             try:
1537                 t = self._create_type(ta_type)
1538             except ConfigError as exc:
1539                 exc.append_ctx('Metadata',
1540                                'Cannot create type alias "{}"'.format(ta_name))
1541                 raise
1542
1543             self._tas[ta_name] = t
1544
1545     def _create_clock(self, node):
1546         # create clock object
1547         clock = metadata.Clock()
1548
1549         if not _is_assoc_array_prop(node):
1550             raise ConfigError('Metadata',
1551                               'Clock objects must be associative arrays')
1552
1553         known_props = [
1554             'uuid',
1555             'description',
1556             'freq',
1557             'error-cycles',
1558             'offset',
1559             'absolute',
1560             'return-ctype',
1561         ]
1562
1563         if self._version >= 201:
1564             known_props.append('$return-ctype')
1565
1566         unk_prop = _get_first_unknown_prop(node, known_props)
1567
1568         if unk_prop:
1569             raise ConfigError('Clock',
1570                               'Unknown property: "{}"'.format(unk_prop))
1571
1572         # UUID
1573         if 'uuid' in node:
1574             uuidp = node['uuid']
1575
1576             if uuidp is None:
1577                 clock.set_default_uuid()
1578             else:
1579                 if not _is_str_prop(uuidp):
1580                     raise ConfigError('Clock',
1581                                       '"uuid" property must be a string')
1582
1583                 try:
1584                     uuidp = uuid.UUID(uuidp)
1585                 except:
1586                     raise ConfigError('Clock', 'Malformed UUID: "{}"'.format(uuidp))
1587
1588                 clock.uuid = uuidp
1589
1590         # description
1591         if 'description' in node:
1592             desc = node['description']
1593
1594             if desc is None:
1595                 clock.set_default_description()
1596             else:
1597                 if not _is_str_prop(desc):
1598                     raise ConfigError('Clock',
1599                                       '"description" property must be a string')
1600
1601                 clock.description = desc
1602
1603         # frequency
1604         if 'freq' in node:
1605             freq = node['freq']
1606
1607             if freq is None:
1608                 clock.set_default_freq()
1609             else:
1610                 if not _is_int_prop(freq):
1611                     raise ConfigError('Clock',
1612                                       '"freq" property must be an integer')
1613
1614                 if freq < 1:
1615                     raise ConfigError('Clock',
1616                                       'Invalid frequency: {}'.format(freq))
1617
1618                 clock.freq = freq
1619
1620         # error cycles
1621         if 'error-cycles' in node:
1622             error_cycles = node['error-cycles']
1623
1624             if error_cycles is None:
1625                 clock.set_default_error_cycles()
1626             else:
1627                 if not _is_int_prop(error_cycles):
1628                     raise ConfigError('Clock',
1629                                       '"error-cycles" property must be an integer')
1630
1631                 if error_cycles < 0:
1632                     raise ConfigError('Clock',
1633                                       'Invalid error cycles: {}'.format(error_cycles))
1634
1635                 clock.error_cycles = error_cycles
1636
1637         # offset
1638         if 'offset' in node:
1639             offset = node['offset']
1640
1641             if offset is None:
1642                 clock.set_default_offset_seconds()
1643                 clock.set_default_offset_cycles()
1644             else:
1645                 if not _is_assoc_array_prop(offset):
1646                     raise ConfigError('Clock',
1647                                       '"offset" property must be an associative array')
1648
1649                 unk_prop = _get_first_unknown_prop(offset, ['cycles', 'seconds'])
1650
1651                 if unk_prop:
1652                     raise ConfigError('Clock',
1653                                       'Unknown offset property: "{}"'.format(unk_prop))
1654
1655                 # cycles
1656                 if 'cycles' in offset:
1657                     offset_cycles = offset['cycles']
1658
1659                     if offset_cycles is None:
1660                         clock.set_default_offset_cycles()
1661                     else:
1662                         if not _is_int_prop(offset_cycles):
1663                             raise ConfigError('Clock\'s "offset" property',
1664                                               '"cycles" property must be an integer')
1665
1666                         if offset_cycles < 0:
1667                             raise ConfigError('Clock\'s "offset" property',
1668                                               'Invalid cycles: {}'.format(offset_cycles))
1669
1670                         clock.offset_cycles = offset_cycles
1671
1672                 # seconds
1673                 if 'seconds' in offset:
1674                     offset_seconds = offset['seconds']
1675
1676                     if offset_seconds is None:
1677                         clock.set_default_offset_seconds()
1678                     else:
1679                         if not _is_int_prop(offset_seconds):
1680                             raise ConfigError('Clock\'s "offset" property',
1681                                               '"seconds" property must be an integer')
1682
1683                         if offset_seconds < 0:
1684                             raise ConfigError('Clock\'s "offset" property',
1685                                               'Invalid seconds: {}'.format(offset_seconds))
1686
1687                         clock.offset_seconds = offset_seconds
1688
1689         # absolute
1690         if 'absolute' in node:
1691             absolute = node['absolute']
1692
1693             if absolute is None:
1694                 clock.set_default_absolute()
1695             else:
1696                 if not _is_bool_prop(absolute):
1697                     raise ConfigError('Clock',
1698                                       '"absolute" property must be a boolean')
1699
1700                 clock.absolute = absolute
1701
1702         # return C type:
1703         #   v2.0:  "return-ctype"
1704         #   v2.1+: "$return-ctype"
1705         return_ctype_node = None
1706
1707         if self._version >= 200:
1708             if 'return-ctype' in node:
1709                 return_ctype_prop = 'return-ctype'
1710                 return_ctype_node = node[return_ctype_prop]
1711
1712         if self._version >= 201:
1713             if '$return-ctype' in node:
1714                 if return_ctype_node is not None:
1715                     raise ConfigError('Clock',
1716                                       'Cannot specify both "return-ctype" and "$return-ctype" properties: prefer "$return-ctype"')
1717
1718                 return_ctype_prop = '$return-ctype'
1719                 return_ctype_node = node[return_ctype_prop]
1720
1721         if return_ctype_node is not None:
1722             if return_ctype_node is None:
1723                 clock.set_default_return_ctype()
1724             else:
1725                 if not _is_str_prop(return_ctype_node):
1726                     raise ConfigError('Clock',
1727                                       '"{}" property of must be a string'.format(return_ctype_prop))
1728
1729                 clock.return_ctype = return_ctype_node
1730
1731         return clock
1732
1733     def _register_clocks(self, metadata_node):
1734         self._clocks = collections.OrderedDict()
1735
1736         if 'clocks' not in metadata_node:
1737             return
1738
1739         clocks_node = metadata_node['clocks']
1740
1741         if clocks_node is None:
1742             return
1743
1744         if not _is_assoc_array_prop(clocks_node):
1745             raise ConfigError('Metadata',
1746                               '"clocks" property must be an associative array')
1747
1748         for clock_name, clock_node in clocks_node.items():
1749             if not _is_valid_identifier(clock_name):
1750                 raise ConfigError('Metadata',
1751                                   'Invalid clock name: "{}"'.format(clock_name))
1752
1753             if clock_name in self._clocks:
1754                 raise ConfigError('Metadata',
1755                                   'Duplicate clock "{}"'.format(clock_name))
1756
1757             try:
1758                 clock = self._create_clock(clock_node)
1759             except ConfigError as exc:
1760                 exc.append_ctx('Metadata',
1761                                'Cannot create clock "{}"'.format(clock_name))
1762                 raise
1763
1764             clock.name = clock_name
1765             self._clocks[clock_name] = clock
1766
1767     def _create_env(self, metadata_node):
1768         env = collections.OrderedDict()
1769
1770         if 'env' not in metadata_node:
1771             return env
1772
1773         env_node = metadata_node['env']
1774
1775         if env_node is None:
1776             return env
1777
1778         if not _is_assoc_array_prop(env_node):
1779             raise ConfigError('Metadata',
1780                               '"env" property must be an associative array')
1781
1782         for env_name, env_value in env_node.items():
1783             if env_name in env:
1784                 raise ConfigError('Metadata',
1785                                   'Duplicate environment variable "{}"'.format(env_name))
1786
1787             if not _is_valid_identifier(env_name):
1788                 raise ConfigError('Metadata',
1789                                   'Invalid environment variable name: "{}"'.format(env_name))
1790
1791             if not _is_int_prop(env_value) and not _is_str_prop(env_value):
1792                 raise ConfigError('Metadata',
1793                                   'Invalid environment variable value ("{}"): expecting integer or string'.format(env_name))
1794
1795             env[env_name] = env_value
1796
1797         return env
1798
1799     def _register_log_levels(self, metadata_node):
1800         self._log_levels = dict()
1801
1802         # log levels:
1803         #   v2.0:  "log-levels"
1804         #   v2.1+: "$log-levels"
1805         log_levels_node = None
1806
1807         if self._version >= 200:
1808             if 'log-levels' in metadata_node:
1809                 log_levels_prop = 'log-levels'
1810                 log_levels_node = metadata_node[log_levels_prop]
1811
1812         if self._version >= 201:
1813             if '$log-levels' in metadata_node:
1814                 if log_levels_node is not None:
1815                     raise ConfigError('Metadata',
1816                                       'Cannot specify both "log-levels" and "$log-levels" properties of metadata object: prefer "$log-levels"')
1817
1818                 log_levels_prop = '$log-levels'
1819                 log_levels_node = metadata_node[log_levels_prop]
1820
1821         if log_levels_node is None:
1822             return
1823
1824         if not _is_assoc_array_prop(log_levels_node):
1825             raise ConfigError('Metadata',
1826                               '"{}" property (metadata) must be an associative array'.format(log_levels_prop))
1827
1828         for ll_name, ll_value in log_levels_node.items():
1829             if ll_name in self._log_levels:
1830                 raise ConfigError('"{}" property"'.format(log_levels_prop),
1831                                   'Duplicate entry "{}"'.format(ll_name))
1832
1833             if not _is_int_prop(ll_value):
1834                 raise ConfigError('"{}" property"'.format(log_levels_prop),
1835                                   'Invalid entry ("{}"): expecting an integer'.format(ll_name))
1836
1837             if ll_value < 0:
1838                 raise ConfigError('"{}" property"'.format(log_levels_prop),
1839                                   'Invalid entry ("{}"): value must be positive'.format(ll_name))
1840
1841             self._log_levels[ll_name] = ll_value
1842
1843     def _create_trace(self, metadata_node):
1844         # create trace object
1845         trace = metadata.Trace()
1846
1847         if 'trace' not in metadata_node:
1848             raise ConfigError('Metadata', 'Missing "trace" property')
1849
1850         trace_node = metadata_node['trace']
1851
1852         if not _is_assoc_array_prop(trace_node):
1853             raise ConfigError('Metadata',
1854                               '"trace" property must be an associative array')
1855
1856         unk_prop = _get_first_unknown_prop(trace_node, [
1857             'byte-order',
1858             'uuid',
1859             'packet-header-type',
1860         ])
1861
1862         if unk_prop:
1863             raise ConfigError('Trace',
1864                               'Unknown property: "{}"'.format(unk_prop))
1865
1866         # set byte order (already parsed)
1867         trace.byte_order = self._bo
1868
1869         # UUID
1870         if 'uuid' in trace_node and trace_node['uuid'] is not None:
1871             uuidp = trace_node['uuid']
1872
1873             if not _is_str_prop(uuidp):
1874                 raise ConfigError('Trace',
1875                                   '"uuid" property must be a string')
1876
1877             if uuidp == 'auto':
1878                 uuidp = uuid.uuid1()
1879             else:
1880                 try:
1881                     uuidp = uuid.UUID(uuidp)
1882                 except:
1883                     raise ConfigError('Trace',
1884                                       'Malformed UUID: "{}"'.format(uuidp))
1885
1886             trace.uuid = uuidp
1887
1888         # packet header type
1889         if 'packet-header-type' in trace_node and trace_node['packet-header-type'] is not None:
1890             try:
1891                 ph_type = self._create_type(trace_node['packet-header-type'])
1892             except ConfigError as exc:
1893                 exc.append_ctx('Trace',
1894                                'Cannot create packet header type')
1895                 raise
1896
1897             trace.packet_header_type = ph_type
1898
1899         return trace
1900
1901     def _lookup_log_level(self, ll):
1902         if _is_int_prop(ll):
1903             return ll
1904         elif _is_str_prop(ll) and ll in self._log_levels:
1905             return self._log_levels[ll]
1906
1907     def _create_event(self, event_node):
1908         event = metadata.Event()
1909
1910         if not _is_assoc_array_prop(event_node):
1911             raise ConfigError('Event',
1912                               'Expecting associative array')
1913
1914         unk_prop = _get_first_unknown_prop(event_node, [
1915             'log-level',
1916             'context-type',
1917             'payload-type',
1918         ])
1919
1920         if unk_prop:
1921             raise ConfigError('Event',
1922                               'Unknown property: "{}"'.format(unk_prop))
1923
1924         if 'log-level' in event_node and event_node['log-level'] is not None:
1925             ll_node = event_node['log-level']
1926
1927             if _is_str_prop(ll_node):
1928                 ll_value = self._lookup_log_level(event_node['log-level'])
1929
1930                 if ll_value is None:
1931                     raise ConfigError('Event\'s "log-level" property',
1932                                       'Cannot find log level "{}"'.format(ll_node))
1933
1934                 ll = metadata.LogLevel(event_node['log-level'], ll_value)
1935             elif _is_int_prop(ll_node):
1936                 if ll_node < 0:
1937                     raise ConfigError('Event\'s "log-level" property',
1938                                       'Invalid value {}: value must be positive'.format(ll_node))
1939
1940                 ll = metadata.LogLevel(None, ll_node)
1941             else:
1942                 raise ConfigError('Event\'s "log-level" property',
1943                                   'Must be either a string or an integer')
1944
1945             event.log_level = ll
1946
1947         if 'context-type' in event_node and event_node['context-type'] is not None:
1948             ctx_type_node = event_node['context-type']
1949
1950             try:
1951                 t = self._create_type(event_node['context-type'])
1952             except ConfigError as exc:
1953                 exc.append_ctx('Event',
1954                                'Cannot create context type object')
1955                 raise
1956
1957             event.context_type = t
1958
1959         if 'payload-type' in event_node and event_node['payload-type'] is not None:
1960             try:
1961                 t = self._create_type(event_node['payload-type'])
1962             except ConfigError as exc:
1963                 exc.append_ctx('Event',
1964                                'Cannot create payload type object')
1965                 raise
1966
1967             event.payload_type = t
1968
1969         return event
1970
1971     def _create_stream(self, stream_name, stream_node):
1972         stream = metadata.Stream()
1973
1974         if not _is_assoc_array_prop(stream_node):
1975             raise ConfigError('Stream objects must be associative arrays')
1976
1977         known_props = [
1978             'packet-context-type',
1979             'event-header-type',
1980             'event-context-type',
1981             'events',
1982         ]
1983
1984         if self._version >= 202:
1985             known_props.append('$default')
1986
1987         unk_prop = _get_first_unknown_prop(stream_node, known_props)
1988
1989         if unk_prop:
1990             add = ''
1991
1992             if unk_prop == '$default':
1993                 add = ' (use version 2.2 or greater)'
1994
1995             raise ConfigError('Stream',
1996                               'Unknown property{}: "{}"'.format(add, unk_prop))
1997
1998         if 'packet-context-type' in stream_node and stream_node['packet-context-type'] is not None:
1999             try:
2000                 t = self._create_type(stream_node['packet-context-type'])
2001             except ConfigError as exc:
2002                 exc.append_ctx('Stream',
2003                                'Cannot create packet context type object')
2004                 raise
2005
2006             stream.packet_context_type = t
2007
2008         if 'event-header-type' in stream_node and stream_node['event-header-type'] is not None:
2009             try:
2010                 t = self._create_type(stream_node['event-header-type'])
2011             except ConfigError as exc:
2012                 exc.append_ctx('Stream',
2013                                'Cannot create event header type object')
2014                 raise
2015
2016             stream.event_header_type = t
2017
2018         if 'event-context-type' in stream_node and stream_node['event-context-type'] is not None:
2019             try:
2020                 t = self._create_type(stream_node['event-context-type'])
2021             except ConfigError as exc:
2022                 exc.append_ctx('Stream',
2023                                'Cannot create event context type object')
2024                 raise
2025
2026             stream.event_context_type = t
2027
2028         if 'events' not in stream_node:
2029             raise ConfigError('Stream',
2030                               'Missing "events" property')
2031
2032         events = stream_node['events']
2033
2034         if events is not None:
2035             if not _is_assoc_array_prop(events):
2036                 raise ConfigError('Stream',
2037                                   '"events" property must be an associative array')
2038
2039             if not events:
2040                 raise ConfigError('Stream', 'At least one event is needed')
2041
2042             cur_id = 0
2043
2044             for ev_name, ev_node in events.items():
2045                 try:
2046                     ev = self._create_event(ev_node)
2047                 except ConfigError as exc:
2048                     exc.append_ctx('Stream',
2049                                    'Cannot create event "{}"'.format(ev_name))
2050                     raise
2051
2052                 ev.id = cur_id
2053                 ev.name = ev_name
2054                 stream.events[ev_name] = ev
2055                 cur_id += 1
2056
2057         if '$default' in stream_node and stream_node['$default'] is not None:
2058             default_node = stream_node['$default']
2059
2060             if not _is_bool_prop(default_node):
2061                 raise ConfigError('Stream',
2062                                   'Invalid "$default" property: expecting a boolean')
2063
2064             if default_node:
2065                 if self._meta.default_stream_name is not None and self._meta.default_stream_name != stream_name:
2066                     fmt = 'Cannot specify more than one default stream (default stream already set to "{}")'
2067                     raise ConfigError('Stream',
2068                                       fmt.format(self._meta.default_stream_name))
2069
2070                 self._meta.default_stream_name = stream_name
2071
2072         return stream
2073
2074     def _create_streams(self, metadata_node):
2075         streams = collections.OrderedDict()
2076
2077         if 'streams' not in metadata_node:
2078             raise ConfigError('Metadata',
2079                               'Missing "streams" property')
2080
2081         streams_node = metadata_node['streams']
2082
2083         if not _is_assoc_array_prop(streams_node):
2084             raise ConfigError('Metadata',
2085                               '"streams" property must be an associative array')
2086
2087         if not streams_node:
2088             raise ConfigError('Metadata\'s "streams" property',
2089                               'At least one stream is needed')
2090
2091         cur_id = 0
2092
2093         for stream_name, stream_node in streams_node.items():
2094             try:
2095                 stream = self._create_stream(stream_name, stream_node)
2096             except ConfigError as exc:
2097                 exc.append_ctx('Metadata',
2098                                'Cannot create stream "{}"'.format(stream_name))
2099                 raise
2100
2101             stream.id = cur_id
2102             stream.name = str(stream_name)
2103             streams[stream_name] = stream
2104             cur_id += 1
2105
2106         return streams
2107
2108     def _create_metadata(self, root):
2109         self._meta = metadata.Metadata()
2110
2111         if 'metadata' not in root:
2112             raise ConfigError('Configuration',
2113                               'Missing "metadata" property')
2114
2115         metadata_node = root['metadata']
2116
2117         if not _is_assoc_array_prop(metadata_node):
2118             raise ConfigError('Configuration\'s "metadata" property',
2119                               'Must be an associative array')
2120
2121         known_props = [
2122             'type-aliases',
2123             'log-levels',
2124             'trace',
2125             'env',
2126             'clocks',
2127             'streams',
2128         ]
2129
2130         if self._version >= 201:
2131             known_props.append('$log-levels')
2132
2133         if self._version >= 202:
2134             known_props.append('$default-stream')
2135
2136         unk_prop = _get_first_unknown_prop(metadata_node, known_props)
2137
2138         if unk_prop:
2139             add = ''
2140
2141             if unk_prop == '$include':
2142                 add = ' (use version 2.1 or greater)'
2143
2144             if unk_prop == '$default-stream':
2145                 add = ' (use version 2.2 or greater)'
2146
2147             raise ConfigError('Metadata',
2148                               'Unknown property{}: "{}"'.format(add, unk_prop))
2149
2150         if '$default-stream' in metadata_node and metadata_node['$default-stream'] is not None:
2151             default_stream_node = metadata_node['$default-stream']
2152
2153             if not _is_str_prop(default_stream_node):
2154                 raise ConfigError('Metadata\'s "$default-stream" property',
2155                                   'Expecting a string')
2156
2157             self._meta.default_stream_name = default_stream_node
2158
2159         self._set_byte_order(metadata_node)
2160         self._register_clocks(metadata_node)
2161         self._meta.clocks = self._clocks
2162         self._register_type_aliases(metadata_node)
2163         self._meta.env = self._create_env(metadata_node)
2164         self._meta.trace = self._create_trace(metadata_node)
2165         self._register_log_levels(metadata_node)
2166         self._meta.streams = self._create_streams(metadata_node)
2167
2168         return self._meta
2169
2170     def _get_version(self, root):
2171         if 'version' not in root:
2172             raise ConfigError('Configuration',
2173                               'Missing "version" property')
2174
2175         version_node = root['version']
2176
2177         if not _is_str_prop(version_node):
2178             raise ConfigError('Configuration\'s "version" property',
2179                               'Must be a string')
2180
2181         version_node = version_node.strip()
2182
2183         if version_node not in ['2.0', '2.1', '2.2']:
2184             raise ConfigError('Configuration',
2185                               'Unsupported version ({}): versions 2.0, 2.1, and 2.2 are supported'.format(version_node))
2186
2187         # convert version string to comparable version integer
2188         parts = version_node.split('.')
2189         version = int(parts[0]) * 100 + int(parts[1])
2190
2191         return version
2192
2193     def _get_prefix(self, root):
2194         def_prefix = 'barectf_'
2195
2196         if 'prefix' not in root:
2197             return def_prefix
2198
2199         prefix_node = root['prefix']
2200
2201         if prefix_node is None:
2202             return def_prefix
2203
2204         if not _is_str_prop(prefix_node):
2205             raise ConfigError('Configuration\'s "prefix" property',
2206                               'Must be a string')
2207
2208         if not _is_valid_identifier(prefix_node):
2209             raise ConfigError('Configuration\'s "prefix" property',
2210                               'Must be a valid C identifier')
2211
2212         return prefix_node
2213
2214     def _get_options(self, root):
2215         cfg_options = ConfigOptions()
2216
2217         if 'options' not in root:
2218             return cfg_options
2219
2220         options_node = root['options']
2221
2222         if not _is_assoc_array_prop(options_node):
2223             raise ConfigError('Configuration\'s "options" property',
2224                               'Must be an associative array')
2225
2226         known_props = [
2227             'gen-prefix-def',
2228             'gen-default-stream-def',
2229         ]
2230         unk_prop = _get_first_unknown_prop(options_node, known_props)
2231
2232         if unk_prop:
2233             raise ConfigError('Configuration\'s "options" property',
2234                               'Unknown property: "{}"'.format(unk_prop))
2235
2236         if 'gen-prefix-def' in options_node and options_node['gen-prefix-def'] is not None:
2237             gen_prefix_def_node = options_node['gen-prefix-def']
2238
2239             if not _is_bool_prop(gen_prefix_def_node):
2240                 raise ConfigError('Configuration\'s "options" property',
2241                                   'Invalid option "gen-prefix-def": expecting a boolean')
2242
2243             cfg_options.gen_prefix_def = gen_prefix_def_node
2244
2245         if 'gen-default-stream-def' in options_node and options_node['gen-default-stream-def'] is not None:
2246             gen_default_stream_def_node = options_node['gen-default-stream-def']
2247
2248             if not _is_bool_prop(gen_default_stream_def_node):
2249                 raise ConfigError('Configuration\'s "options" property',
2250                                   'Invalid option "gen-default-stream-def": expecting a boolean')
2251
2252             cfg_options.gen_default_stream_def = gen_default_stream_def_node
2253
2254         return cfg_options
2255
2256     def _get_last_include_file(self):
2257         if self._include_stack:
2258             return self._include_stack[-1]
2259
2260         return self._root_yaml_path
2261
2262     def _load_include(self, yaml_path):
2263         for inc_dir in self._include_dirs:
2264             # current include dir + file name path
2265             # note: os.path.join() only takes the last arg if it's absolute
2266             inc_path = os.path.join(inc_dir, yaml_path)
2267
2268             # real path (symbolic links resolved)
2269             real_path = os.path.realpath(inc_path)
2270
2271             # normalized path (weird stuff removed!)
2272             norm_path = os.path.normpath(real_path)
2273
2274             if not os.path.isfile(norm_path):
2275                 # file does not exist: skip
2276                 continue
2277
2278             if norm_path in self._include_stack:
2279                 base_path = self._get_last_include_file()
2280                 raise ConfigError('In "{}"',
2281                                   'Cannot recursively include file "{}"'.format(base_path, norm_path))
2282
2283             self._include_stack.append(norm_path)
2284
2285             # load raw content
2286             return self._yaml_ordered_load(norm_path)
2287
2288         if not self._ignore_include_not_found:
2289             base_path = self._get_last_include_file()
2290             raise ConfigError('In "{}"',
2291                               'Cannot include file "{}": file not found in include directories'.format(base_path, yaml_path))
2292
2293         return None
2294
2295     def _get_include_paths(self, include_node):
2296         if include_node is None:
2297             return []
2298
2299         if _is_str_prop(include_node):
2300             return [include_node]
2301
2302         if _is_array_prop(include_node):
2303             for include_path in include_node:
2304                 if not _is_str_prop(include_path):
2305                     raise ConfigError('"$include" property',
2306                                       'Expecting array of strings')
2307
2308             return include_node
2309
2310         raise ConfigError('"$include" property',
2311                           'Expecting string or array of strings')
2312
2313     def _update_node(self, base_node, overlay_node):
2314         for olay_key, olay_value in overlay_node.items():
2315             if olay_key in base_node:
2316                 base_value = base_node[olay_key]
2317
2318                 if _is_assoc_array_prop(olay_value) and _is_assoc_array_prop(base_value):
2319                     # merge dictionaries
2320                     self._update_node(base_value, olay_value)
2321                 elif _is_array_prop(olay_value) and _is_array_prop(base_value):
2322                     # append extension array items to base items
2323                     base_value += olay_value
2324                 else:
2325                     # fall back to replacing
2326                     base_node[olay_key] = olay_value
2327             else:
2328                 base_node[olay_key] = olay_value
2329
2330     def _process_node_include(self, last_overlay_node, name,
2331                               process_base_include_cb,
2332                               process_children_include_cb=None):
2333         if not _is_assoc_array_prop(last_overlay_node):
2334             raise ConfigError('"$include" property',
2335                               '{} objects must be associative arrays'.format(name))
2336
2337         # process children inclusions first
2338         if process_children_include_cb:
2339             process_children_include_cb(last_overlay_node)
2340
2341         if '$include' in last_overlay_node:
2342             include_node = last_overlay_node['$include']
2343         else:
2344             # no includes!
2345             return last_overlay_node
2346
2347         include_paths = self._get_include_paths(include_node)
2348         cur_base_path = self._get_last_include_file()
2349         base_node = None
2350
2351         # keep the include paths and remove the include property
2352         include_paths = copy.deepcopy(include_paths)
2353         del last_overlay_node['$include']
2354
2355         for include_path in include_paths:
2356             # load raw YAML from included file
2357             overlay_node = self._load_include(include_path)
2358
2359             if overlay_node is None:
2360                 # cannot find include file, but we're ignoring those
2361                 # errors, otherwise _load_include() itself raises
2362                 # a config error
2363                 continue
2364
2365             # recursively process includes
2366             try:
2367                 overlay_node = process_base_include_cb(overlay_node)
2368             except ConfigError as exc:
2369                 exc.append_ctx('In "{}"'.format(cur_base_path))
2370                 raise
2371
2372             # pop include stack now that we're done including
2373             del self._include_stack[-1]
2374
2375             # at this point, base_node is fully resolved (does not
2376             # contain any include property)
2377             if base_node is None:
2378                 base_node = overlay_node
2379             else:
2380                 self._update_node(base_node, overlay_node)
2381
2382         # finally, we update the latest base node with our last overlay
2383         # node
2384         if base_node is None:
2385             # nothing was included, which is possible when we're
2386             # ignoring include errors
2387             return last_overlay_node
2388
2389         self._update_node(base_node, last_overlay_node)
2390
2391         return base_node
2392
2393     def _process_event_include(self, event_node):
2394         return self._process_node_include(event_node, 'event',
2395                                           self._process_event_include)
2396
2397     def _process_stream_include(self, stream_node):
2398         def process_children_include(stream_node):
2399             if 'events' in stream_node:
2400                 events_node = stream_node['events']
2401
2402                 if not _is_assoc_array_prop(events_node):
2403                     raise ConfigError('"$include" property',
2404                                       '"events" property must be an associative array')
2405
2406                 events_node_keys = list(events_node.keys())
2407
2408                 for key in events_node_keys:
2409                     event_node = events_node[key]
2410
2411                     try:
2412                         events_node[key] = self._process_event_include(event_node)
2413                     except ConfigError as exc:
2414                         exc.append_ctx('"$include" property',
2415                                        'Cannot process includes of event object "{}"'.format(key))
2416                         raise
2417
2418         return self._process_node_include(stream_node, 'stream',
2419                                           self._process_stream_include,
2420                                           process_children_include)
2421
2422     def _process_trace_include(self, trace_node):
2423         return self._process_node_include(trace_node, 'trace',
2424                                           self._process_trace_include)
2425
2426     def _process_clock_include(self, clock_node):
2427         return self._process_node_include(clock_node, 'clock',
2428                                           self._process_clock_include)
2429
2430     def _process_metadata_include(self, metadata_node):
2431         def process_children_include(metadata_node):
2432             if 'trace' in metadata_node:
2433                 metadata_node['trace'] = self._process_trace_include(metadata_node['trace'])
2434
2435             if 'clocks' in metadata_node:
2436                 clocks_node = metadata_node['clocks']
2437
2438                 if not _is_assoc_array_prop(clocks_node):
2439                     raise ConfigError('"$include" property',
2440                                       '"clocks" property must be an associative array')
2441
2442                 clocks_node_keys = list(clocks_node.keys())
2443
2444                 for key in clocks_node_keys:
2445                     clock_node = clocks_node[key]
2446
2447                     try:
2448                         clocks_node[key] = self._process_clock_include(clock_node)
2449                     except ConfigError as exc:
2450                         exc.append_ctx('"$include" property',
2451                                        'Cannot process includes of clock object "{}"'.format(key))
2452                         raise
2453
2454             if 'streams' in metadata_node:
2455                 streams_node = metadata_node['streams']
2456
2457                 if not _is_assoc_array_prop(streams_node):
2458                     raise ConfigError('"$include" property',
2459                                       '"streams" property must be an associative array')
2460
2461                 streams_node_keys = list(streams_node.keys())
2462
2463                 for key in streams_node_keys:
2464                     stream_node = streams_node[key]
2465
2466                     try:
2467                         streams_node[key] = self._process_stream_include(stream_node)
2468                     except ConfigError as exc:
2469                         exc.append_ctx('"$include" property',
2470                                        'Cannot process includes of stream object "{}"'.format(key))
2471                         raise
2472
2473         return self._process_node_include(metadata_node, 'metadata',
2474                                           self._process_metadata_include,
2475                                           process_children_include)
2476
2477     def _process_root_includes(self, root):
2478         # The following config objects support includes:
2479         #
2480         #   * Metadata object
2481         #   * Trace object
2482         #   * Stream object
2483         #   * Event object
2484         #
2485         # We need to process the event includes first, then the stream
2486         # includes, then the trace includes, and finally the metadata
2487         # includes.
2488         #
2489         # In each object, only one of the $include and $include-replace
2490         # special properties is allowed.
2491         #
2492         # We keep a stack of absolute paths to included files to detect
2493         # recursion.
2494         if 'metadata' in root:
2495             root['metadata'] = self._process_metadata_include(root['metadata'])
2496
2497         return root
2498
2499     def _yaml_ordered_dump(self, node, **kwds):
2500         class ODumper(yaml.Dumper):
2501             pass
2502
2503         def dict_representer(dumper, node):
2504             return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2505                                             node.items())
2506
2507         ODumper.add_representer(collections.OrderedDict, dict_representer)
2508
2509         return yaml.dump(node, Dumper=ODumper, **kwds)
2510
2511     def _yaml_ordered_load(self, yaml_path):
2512         class OLoader(yaml.Loader):
2513             pass
2514
2515         def construct_mapping(loader, node):
2516             loader.flatten_mapping(node)
2517
2518             return collections.OrderedDict(loader.construct_pairs(node))
2519
2520         OLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2521                                 construct_mapping)
2522
2523         # YAML -> Python
2524         try:
2525             with open(yaml_path, 'r') as f:
2526                 node = yaml.load(f, OLoader)
2527         except (OSError, IOError) as e:
2528             raise ConfigError('Configuration',
2529                               'Cannot open file "{}"'.format(yaml_path))
2530         except ConfigError as exc:
2531             exc.append_ctx('Configuration',
2532                            'Unknown error while trying to load file "{}"'.format(yaml_path))
2533             raise
2534
2535         # loaded node must be an associate array
2536         if not _is_assoc_array_prop(node):
2537             raise ConfigError('Configuration',
2538                               'Root of YAML file "{}" must be an associative array'.format(yaml_path))
2539
2540         return node
2541
2542     def _reset(self):
2543         self._version = None
2544         self._include_stack = []
2545
2546     def parse(self, yaml_path):
2547         self._reset()
2548         self._root_yaml_path = yaml_path
2549
2550         try:
2551             root = self._yaml_ordered_load(yaml_path)
2552         except ConfigError as exc:
2553             exc.append_ctx('Configuration',
2554                            'Cannot parse YAML file "{}"'.format(yaml_path))
2555             raise
2556
2557         if not _is_assoc_array_prop(root):
2558             raise ConfigError('Configuration',
2559                               'Must be an associative array')
2560
2561         # get the config version
2562         self._version = self._get_version(root)
2563
2564         known_props = [
2565             'version',
2566             'prefix',
2567             'metadata',
2568         ]
2569
2570         if self._version >= 202:
2571             known_props.append('options')
2572
2573         unk_prop = _get_first_unknown_prop(root, known_props)
2574
2575         if unk_prop:
2576             add = ''
2577
2578             if unk_prop == 'options':
2579                 add = ' (use version 2.2 or greater)'
2580
2581             raise ConfigError('Configuration',
2582                               'Unknown property{}: "{}"'.format(add, unk_prop))
2583
2584         # process includes if supported
2585         if self._version >= 201:
2586             root = self._process_root_includes(root)
2587
2588         # dump config if required
2589         if self._dump_config:
2590             print(self._yaml_ordered_dump(root, indent=2,
2591                                           default_flow_style=False))
2592
2593         # get prefix and metadata
2594         prefix = self._get_prefix(root)
2595         meta = self._create_metadata(root)
2596         opts = self._get_options(root)
2597
2598         return Config(self._version, prefix, meta, opts)
2599
2600
2601 def from_yaml_file(path, include_dirs, ignore_include_not_found, dump_config):
2602     try:
2603         parser = _YamlConfigParser(include_dirs, ignore_include_not_found,
2604                                    dump_config)
2605         cfg = parser.parse(path)
2606
2607         return cfg
2608     except ConfigError as exc:
2609         exc.append_ctx('Configuration',
2610                        'Cannot create configuration from YAML file "{}"'.format(path))
2611         raise