barectf/config.py

   1 # The MIT License (MIT)
   2 #
   3 # Copyright (c) 2015-2016 Philippe Proulx <pproulx@efficios.com>
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  21 # THE SOFTWARE.
  22
  23 from barectf import metadata
  24 import collections
  25 import datetime
  26 import barectf
  27 import enum
  28 import yaml
  29 import uuid
  30 import copy
  31 import re
  32 import os
  33
  34
  35 class ConfigError(RuntimeError):
  36     def __init__(self, msg, prev=None):
  37         super().__init__(msg)
  38         self._prev = prev
  39
  40     @property
  41     def prev(self):
  42         return self._prev
  43
  44
  45 class Config:
  46     def __init__(self, version, prefix, metadata, options):
  47         self.prefix = prefix
  48         self.version = version
  49         self.metadata = metadata
  50         self.options = options
  51
  52     def _validate_metadata(self, meta):
  53         try:
  54             validator = _MetadataTypesHistologyValidator()
  55             validator.validate(meta)
  56             validator = _MetadataDynamicTypesValidator()
  57             validator.validate(meta)
  58             validator = _MetadataSpecialFieldsValidator()
  59             validator.validate(meta)
  60         except Exception as e:
  61             raise ConfigError('metadata error', e)
  62
  63         try:
  64             validator = _BarectfMetadataValidator()
  65             validator.validate(meta)
  66         except Exception as e:
  67             raise ConfigError('barectf metadata error', e)
  68
  69     def _augment_metadata_env(self, meta):
  70         version_tuple = barectf.get_version_tuple()
  71         base_env = {
  72             'domain': 'bare',
  73             'tracer_name': 'barectf',
  74             'tracer_major': version_tuple[0],
  75             'tracer_minor': version_tuple[1],
  76             'tracer_patch': version_tuple[2],
  77             'barectf_gen_date': str(datetime.datetime.now().isoformat()),
  78         }
  79
  80         base_env.update(meta.env)
  81         meta.env = base_env
  82
  83     @property
  84     def version(self):
  85         return self._version
  86
  87     @version.setter
  88     def version(self, value):
  89         self._version = value
  90
  91     @property
  92     def metadata(self):
  93         return self._metadata
  94
  95     @metadata.setter
  96     def metadata(self, value):
  97         self._validate_metadata(value)
  98         self._augment_metadata_env(value)
  99         self._metadata = value
 100
 101     @property
 102     def prefix(self):
 103         return self._prefix
 104
 105     @prefix.setter
 106     def prefix(self, value):
 107         if not _is_valid_identifier(value):
 108             raise ConfigError('configuration prefix must be a valid C identifier')
 109
 110         self._prefix = value
 111
 112     @property
 113     def options(self):
 114         return self._options
 115
 116     @options.setter
 117     def options(self, options):
 118         self._options = options
 119
 120
 121 class ConfigOptions:
 122     def __init__(self):
 123         self._gen_prefix_def = False
 124         self._gen_default_stream_def = False
 125
 126     @property
 127     def gen_prefix_def(self):
 128         return self._gen_prefix_def
 129
 130     @gen_prefix_def.setter
 131     def gen_prefix_def(self, value):
 132         self._gen_prefix_def = value
 133
 134     @property
 135     def gen_default_stream_def(self):
 136         return self._gen_default_stream_def
 137
 138     @gen_default_stream_def.setter
 139     def gen_default_stream_def(self, value):
 140         self._gen_default_stream_def = value
 141
 142
 143 def _is_assoc_array_prop(node):
 144     return isinstance(node, dict)
 145
 146
 147 def _is_array_prop(node):
 148     return isinstance(node, list)
 149
 150
 151 def _is_int_prop(node):
 152     return type(node) is int
 153
 154
 155 def _is_str_prop(node):
 156     return type(node) is str
 157
 158
 159 def _is_bool_prop(node):
 160     return type(node) is bool
 161
 162
 163 def _is_valid_alignment(align):
 164     return ((align & (align - 1)) == 0) and align > 0
 165
 166
 167 def _byte_order_str_to_bo(bo_str):
 168     bo_str = bo_str.lower()
 169
 170     if bo_str == 'le':
 171         return metadata.ByteOrder.LE
 172     elif bo_str == 'be':
 173         return metadata.ByteOrder.BE
 174
 175
 176 def _encoding_str_to_encoding(encoding_str):
 177     encoding_str = encoding_str.lower()
 178
 179     if encoding_str == 'utf-8' or encoding_str == 'utf8':
 180         return metadata.Encoding.UTF8
 181     elif encoding_str == 'ascii':
 182         return metadata.Encoding.ASCII
 183     elif encoding_str == 'none':
 184         return metadata.Encoding.NONE
 185
 186
 187 _re_iden = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]*$')
 188 _ctf_keywords = set([
 189     'align',
 190     'callsite',
 191     'clock',
 192     'enum',
 193     'env',
 194     'event',
 195     'floating_point',
 196     'integer',
 197     'stream',
 198     'string',
 199     'struct',
 200     'trace',
 201     'typealias',
 202     'typedef',
 203     'variant',
 204 ])
 205
 206
 207 def _is_valid_identifier(iden):
 208     if not _re_iden.match(iden):
 209         return False
 210
 211     if _re_iden in _ctf_keywords:
 212         return False
 213
 214     return True
 215
 216
 217 def _get_first_unknown_prop(node, known_props):
 218     for prop_name in node:
 219         if prop_name in known_props:
 220             continue
 221
 222         return prop_name
 223
 224
 225 # This validator validates the configured metadata for barectf specific
 226 # needs.
 227 #
 228 # barectf needs:
 229 #
 230 #   * all header/contexts are at least byte-aligned
 231 #   * all integer and floating point number sizes to be <= 64
 232 #   * no inner structures, arrays, or variants
 233 class _BarectfMetadataValidator:
 234     def __init__(self):
 235         self._type_to_validate_type_func = {
 236             metadata.Integer: self._validate_int_type,
 237             metadata.FloatingPoint: self._validate_float_type,
 238             metadata.Enum: self._validate_enum_type,
 239             metadata.String: self._validate_string_type,
 240             metadata.Struct: self._validate_struct_type,
 241             metadata.Array: self._validate_array_type,
 242             metadata.Variant: self._validate_variant_type,
 243         }
 244
 245     def _validate_int_type(self, t, entity_root):
 246         if t.size > 64:
 247             raise ConfigError('integer type\'s size must be lesser than or equal to 64 bits')
 248
 249     def _validate_float_type(self, t, entity_root):
 250         if t.size > 64:
 251             raise ConfigError('floating point number type\'s size must be lesser than or equal to 64 bits')
 252
 253     def _validate_enum_type(self, t, entity_root):
 254         if t.value_type.size > 64:
 255             raise ConfigError('enumeration type\'s integer type\'s size must be lesser than or equal to 64 bits')
 256
 257     def _validate_string_type(self, t, entity_root):
 258         pass
 259
 260     def _validate_struct_type(self, t, entity_root):
 261         if not entity_root:
 262             raise ConfigError('inner structure types are not supported as of this version')
 263
 264         for field_name, field_type in t.fields.items():
 265             if entity_root and self._cur_entity is _Entity.TRACE_PACKET_HEADER:
 266                 if field_name == 'uuid':
 267                     # allow
 268                     continue
 269
 270             try:
 271                 self._validate_type(field_type, False)
 272             except Exception as e:
 273                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 274
 275     def _validate_array_type(self, t, entity_root):
 276         raise ConfigError('array types are not supported as of this version')
 277
 278     def _validate_variant_type(self, t, entity_root):
 279         raise ConfigError('variant types are not supported as of this version')
 280
 281     def _validate_type(self, t, entity_root):
 282         self._type_to_validate_type_func[type(t)](t, entity_root)
 283
 284     def _validate_entity(self, t):
 285         if t is None:
 286             return
 287
 288         # make sure entity is byte-aligned
 289         if t.align < 8:
 290             raise ConfigError('type\'s alignment must be at least byte-aligned')
 291
 292         # make sure entity is a structure
 293         if type(t) is not metadata.Struct:
 294             raise ConfigError('expecting a structure type')
 295
 296         # validate types
 297         self._validate_type(t, True)
 298
 299     def _validate_entities_and_names(self, meta):
 300         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 301
 302         try:
 303             self._validate_entity(meta.trace.packet_header_type)
 304         except Exception as e:
 305             raise ConfigError('invalid trace packet header type', e)
 306
 307         for stream_name, stream in meta.streams.items():
 308             if not _is_valid_identifier(stream_name):
 309                 raise ConfigError('stream name "{}" is not a valid C identifier'.format(stream_name))
 310
 311             self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 312
 313             try:
 314                 self._validate_entity(stream.packet_context_type)
 315             except Exception as e:
 316                 raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 317
 318             self._cur_entity = _Entity.STREAM_EVENT_HEADER
 319
 320             try:
 321                 self._validate_entity(stream.event_header_type)
 322             except Exception as e:
 323                 raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 324
 325             self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 326
 327             try:
 328                 self._validate_entity(stream.event_context_type)
 329             except Exception as e:
 330                 raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 331
 332             try:
 333                 for ev_name, ev in stream.events.items():
 334                     if not _is_valid_identifier(ev_name):
 335                         raise ConfigError('event name "{}" is not a valid C identifier'.format(ev_name))
 336
 337                     self._cur_entity = _Entity.EVENT_CONTEXT
 338
 339                     try:
 340                         self._validate_entity(ev.context_type)
 341                     except Exception as e:
 342                         raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 343
 344                     self._cur_entity = _Entity.EVENT_PAYLOAD
 345
 346                     try:
 347                         self._validate_entity(ev.payload_type)
 348                     except Exception as e:
 349                         raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 350
 351                     if stream.is_event_empty(ev):
 352                         raise ConfigError('event "{}" is empty'.format(ev_name))
 353             except Exception as e:
 354                 raise ConfigError('invalid stream "{}"'.format(stream_name), e)
 355
 356     def _validate_default_stream(self, meta):
 357         if meta.default_stream_name:
 358             if meta.default_stream_name not in meta.streams.keys():
 359                 raise ConfigError('default stream name ("{}") does not exist'.format(meta.default_stream_name))
 360
 361     def validate(self, meta):
 362         self._validate_entities_and_names(meta)
 363         self._validate_default_stream(meta)
 364
 365
 366 # This validator validates special fields of trace, stream, and event
 367 # types. For example, if checks that the "stream_id" field exists in the
 368 # trace packet header if there's more than one stream, and much more.
 369 class _MetadataSpecialFieldsValidator:
 370     def _validate_trace_packet_header_type(self, t):
 371         # needs "stream_id" field?
 372         if len(self._meta.streams) > 1:
 373             # yes
 374             if t is None:
 375                 raise ConfigError('need "stream_id" field in trace packet header type (more than one stream), but trace packet header type is missing')
 376
 377             if type(t) is not metadata.Struct:
 378                 raise ConfigError('need "stream_id" field in trace packet header type (more than one stream), but trace packet header type is not a structure type')
 379
 380             if 'stream_id' not in t.fields:
 381                 raise ConfigError('need "stream_id" field in trace packet header type (more than one stream)')
 382
 383         # validate "magic" and "stream_id" types
 384         if type(t) is not metadata.Struct:
 385             return
 386
 387         for i, (field_name, field_type) in enumerate(t.fields.items()):
 388             if field_name == 'magic':
 389                 if type(field_type) is not metadata.Integer:
 390                     raise ConfigError('"magic" field in trace packet header type must be an integer type')
 391
 392                 if field_type.signed or field_type.size != 32:
 393                     raise ConfigError('"magic" field in trace packet header type must be a 32-bit unsigned integer type')
 394
 395                 if i != 0:
 396                     raise ConfigError('"magic" field must be the first trace packet header type\'s field')
 397             elif field_name == 'stream_id':
 398                 if type(field_type) is not metadata.Integer:
 399                     raise ConfigError('"stream_id" field in trace packet header type must be an integer type')
 400
 401                 if field_type.signed:
 402                     raise ConfigError('"stream_id" field in trace packet header type must be an unsigned integer type')
 403
 404                 # "id" size can fit all event IDs
 405                 if len(self._meta.streams) > (1 << field_type.size):
 406                     raise ConfigError('"stream_id" field\' size in trace packet header type is too small for the number of trace streams')
 407             elif field_name == 'uuid':
 408                 if self._meta.trace.uuid is None:
 409                     raise ConfigError('"uuid" field in trace packet header type specified, but no trace UUID provided')
 410
 411                 if type(field_type) is not metadata.Array:
 412                     raise ConfigError('"uuid" field in trace packet header type must be an array')
 413
 414                 if field_type.length != 16:
 415                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 bytes')
 416
 417                 element_type = field_type.element_type
 418
 419                 if type(element_type) is not metadata.Integer:
 420                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 421
 422                 if element_type.size != 8:
 423                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 424
 425                 if element_type.signed:
 426                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 427
 428                 if element_type.align != 8:
 429                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned, byte-aligned bytes')
 430
 431     def _validate_trace(self, meta):
 432         self._validate_trace_packet_header_type(meta.trace.packet_header_type)
 433
 434     def _validate_stream_packet_context(self, stream):
 435         t = stream.packet_context_type
 436
 437         if type(t) is None:
 438             raise ConfigError('missing "packet-context-type" property in stream object')
 439
 440         if type(t) is not metadata.Struct:
 441             raise ConfigError('"packet-context-type": expecting a structure type')
 442
 443         # "timestamp_begin", if exists, is an unsigned integer type,
 444         # mapped to a clock
 445         ts_begin = None
 446
 447         if 'timestamp_begin' in t.fields:
 448             ts_begin = t.fields['timestamp_begin']
 449
 450             if type(ts_begin) is not metadata.Integer:
 451                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an integer type')
 452
 453             if ts_begin.signed:
 454                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an unsigned integer type')
 455
 456             if not ts_begin.property_mappings:
 457                 raise ConfigError('"timestamp_begin" field in stream packet context type must be mapped to a clock')
 458
 459         # "timestamp_end", if exists, is an unsigned integer type,
 460         # mapped to a clock
 461         ts_end = None
 462
 463         if 'timestamp_end' in t.fields:
 464             ts_end = t.fields['timestamp_end']
 465
 466             if type(ts_end) is not metadata.Integer:
 467                 raise ConfigError('"timestamp_end" field in stream packet context type must be an integer type')
 468
 469             if ts_end.signed:
 470                 raise ConfigError('"timestamp_end" field in stream packet context type must be an unsigned integer type')
 471
 472             if not ts_end.property_mappings:
 473                 raise ConfigError('"timestamp_end" field in stream packet context type must be mapped to a clock')
 474
 475         # "timestamp_begin" and "timestamp_end" exist together
 476         if (('timestamp_begin' in t.fields) ^ ('timestamp_end' in t.fields)):
 477             raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be defined together in stream packet context type')
 478
 479         # "timestamp_begin" and "timestamp_end" are mapped to the same clock
 480         if ts_begin is not None and ts_end is not None:
 481             if ts_begin.property_mappings[0].object.name != ts_end.property_mappings[0].object.name:
 482                 raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be mapped to the same clock object in stream packet context type')
 483
 484         # "events_discarded", if exists, is an unsigned integer type
 485         if 'events_discarded' in t.fields:
 486             events_discarded = t.fields['events_discarded']
 487
 488             if type(events_discarded) is not metadata.Integer:
 489                 raise ConfigError('"events_discarded" field in stream packet context type must be an integer type')
 490
 491             if events_discarded.signed:
 492                 raise ConfigError('"events_discarded" field in stream packet context type must be an unsigned integer type')
 493
 494         # "packet_size" and "content_size" must exist
 495         if 'packet_size' not in t.fields:
 496             raise ConfigError('missing "packet_size" field in stream packet context type')
 497
 498         packet_size = t.fields['packet_size']
 499
 500         # "content_size" and "content_size" must exist
 501         if 'content_size' not in t.fields:
 502             raise ConfigError('missing "content_size" field in stream packet context type')
 503
 504         content_size = t.fields['content_size']
 505
 506         # "packet_size" is an unsigned integer type
 507         if type(packet_size) is not metadata.Integer:
 508             raise ConfigError('"packet_size" field in stream packet context type must be an integer type')
 509
 510         if packet_size.signed:
 511             raise ConfigError('"packet_size" field in stream packet context type must be an unsigned integer type')
 512
 513         # "content_size" is an unsigned integer type
 514         if type(content_size) is not metadata.Integer:
 515             raise ConfigError('"content_size" field in stream packet context type must be an integer type')
 516
 517         if content_size.signed:
 518             raise ConfigError('"content_size" field in stream packet context type must be an unsigned integer type')
 519
 520         # "packet_size" size should be greater than or equal to "content_size" size
 521         if content_size.size > packet_size.size:
 522             raise ConfigError('"content_size" field size must be lesser than or equal to "packet_size" field size')
 523
 524     def _validate_stream_event_header(self, stream):
 525         t = stream.event_header_type
 526
 527         # needs "id" field?
 528         if len(stream.events) > 1:
 529             # yes
 530             if t is None:
 531                 raise ConfigError('need "id" field in stream event header type (more than one event), but stream event header type is missing')
 532
 533             if type(t) is not metadata.Struct:
 534                 raise ConfigError('need "id" field in stream event header type (more than one event), but stream event header type is not a structure type')
 535
 536             if 'id' not in t.fields:
 537                 raise ConfigError('need "id" field in stream event header type (more than one event)')
 538
 539         # validate "id" and "timestamp" types
 540         if type(t) is not metadata.Struct:
 541             return
 542
 543         # "timestamp", if exists, is an unsigned integer type,
 544         # mapped to a clock
 545         if 'timestamp' in t.fields:
 546             ts = t.fields['timestamp']
 547
 548             if type(ts) is not metadata.Integer:
 549                 raise ConfigError('"timestamp" field in stream event header type must be an integer type')
 550
 551             if ts.signed:
 552                 raise ConfigError('"timestamp" field in stream event header type must be an unsigned integer type')
 553
 554             if not ts.property_mappings:
 555                 raise ConfigError('"timestamp" field in stream event header type must be mapped to a clock')
 556
 557         if 'id' in t.fields:
 558             eid = t.fields['id']
 559
 560             # "id" is an unsigned integer type
 561             if type(eid) is not metadata.Integer:
 562                 raise ConfigError('"id" field in stream event header type must be an integer type')
 563
 564             if eid.signed:
 565                 raise ConfigError('"id" field in stream event header type must be an unsigned integer type')
 566
 567             # "id" size can fit all event IDs
 568             if len(stream.events) > (1 << eid.size):
 569                 raise ConfigError('"id" field\' size in stream event header type is too small for the number of stream events')
 570
 571     def _validate_stream(self, stream):
 572         self._validate_stream_packet_context(stream)
 573         self._validate_stream_event_header(stream)
 574
 575     def validate(self, meta):
 576         self._meta = meta
 577         self._validate_trace(meta)
 578
 579         for stream in meta.streams.values():
 580             try:
 581                 self._validate_stream(stream)
 582             except Exception as e:
 583                 raise ConfigError('invalid stream "{}"'.format(stream.name), e)
 584
 585
 586 class _MetadataDynamicTypesValidatorStackEntry:
 587     def __init__(self, base_t):
 588         self._base_t = base_t
 589         self._index = 0
 590
 591     @property
 592     def index(self):
 593         return self._index
 594
 595     @index.setter
 596     def index(self, value):
 597         self._index = value
 598
 599     @property
 600     def base_t(self):
 601         return self._base_t
 602
 603     @base_t.setter
 604     def base_t(self, value):
 605         self._base_t = value
 606
 607
 608 # Entities. Order of values is important here.
 609 @enum.unique
 610 class _Entity(enum.IntEnum):
 611     TRACE_PACKET_HEADER = 0
 612     STREAM_PACKET_CONTEXT = 1
 613     STREAM_EVENT_HEADER = 2
 614     STREAM_EVENT_CONTEXT = 3
 615     EVENT_CONTEXT = 4
 616     EVENT_PAYLOAD = 5
 617
 618
 619 # This validator validates dynamic metadata types, that is, it ensures
 620 # variable-length array lengths and variant tags actually point to
 621 # something that exists. It also checks that variable-length array
 622 # lengths point to integer types and variant tags to enumeration types.
 623 class _MetadataDynamicTypesValidator:
 624     def __init__(self):
 625         self._type_to_visit_type_func = {
 626             metadata.Integer: None,
 627             metadata.FloatingPoint: None,
 628             metadata.Enum: None,
 629             metadata.String: None,
 630             metadata.Struct: self._visit_struct_type,
 631             metadata.Array: self._visit_array_type,
 632             metadata.Variant: self._visit_variant_type,
 633         }
 634
 635         self._cur_trace = None
 636         self._cur_stream = None
 637         self._cur_event = None
 638
 639     def _lookup_path_from_base(self, path, parts, base, start_index,
 640                                base_is_current, from_t):
 641         index = start_index
 642         cur_t = base
 643         found_path = []
 644
 645         while index < len(parts):
 646             part = parts[index]
 647             next_t = None
 648
 649             if type(cur_t) is metadata.Struct:
 650                 enumerated_items = enumerate(cur_t.fields.items())
 651
 652                 # lookup each field
 653                 for i, (field_name, field_type) in enumerated_items:
 654                     if field_name == part:
 655                         next_t = field_type
 656                         found_path.append((i, field_type))
 657
 658                 if next_t is None:
 659                     raise ConfigError('invalid path "{}": cannot find field "{}" in structure type'.format(path, part))
 660             elif type(cur_t) is metadata.Variant:
 661                 enumerated_items = enumerate(cur_t.types.items())
 662
 663                 # lookup each type
 664                 for i, (type_name, type_type) in enumerated_items:
 665                     if type_name == part:
 666                         next_t = type_type
 667                         found_path.append((i, type_type))
 668
 669                 if next_t is None:
 670                     raise ConfigError('invalid path "{}": cannot find type "{}" in variant type'.format(path, part))
 671             else:
 672                 raise ConfigError('invalid path "{}": requesting "{}" in a non-variant, non-structure type'.format(path, part))
 673
 674             cur_t = next_t
 675             index += 1
 676
 677         # make sure that the pointed type is not the pointing type
 678         if from_t is cur_t:
 679             raise ConfigError('invalid path "{}": pointing to self'.format(path))
 680
 681         # if we're here, we found the type; however, it could be located
 682         # _after_ the variant/VLA looking for it, if the pointing
 683         # and pointed types are in the same entity, so compare the
 684         # current stack entries indexes to our index path in that case
 685         if not base_is_current:
 686             return cur_t
 687
 688         for index, entry in enumerate(self._stack):
 689             if index == len(found_path):
 690                 # end of index path; valid so far
 691                 break
 692
 693             if found_path[index][0] > entry.index:
 694                 raise ConfigError('invalid path "{}": pointed type is after pointing type'.format(path))
 695
 696         # also make sure that both pointed and pointing types share
 697         # a common structure ancestor
 698         for index, entry in enumerate(self._stack):
 699             if index == len(found_path):
 700                 break
 701
 702             if entry.base_t is not found_path[index][1]:
 703                 # found common ancestor
 704                 if type(entry.base_t) is metadata.Variant:
 705                     raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 706
 707         return cur_t
 708
 709     def _lookup_path_from_top(self, path, parts):
 710         if len(parts) != 1:
 711             raise ConfigError('invalid path "{}": multipart relative path not supported'.format(path))
 712
 713         find_name = parts[0]
 714         index = len(self._stack) - 1
 715         got_struct = False
 716
 717         # check stack entries in reversed order
 718         for entry in reversed(self._stack):
 719             # structure base type
 720             if type(entry.base_t) is metadata.Struct:
 721                 got_struct = True
 722                 enumerated_items = enumerate(entry.base_t.fields.items())
 723
 724                 # lookup each field, until the current visiting index is met
 725                 for i, (field_name, field_type) in enumerated_items:
 726                     if i == entry.index:
 727                         break
 728
 729                     if field_name == find_name:
 730                         return field_type
 731
 732             # variant base type
 733             elif type(entry.base_t) is metadata.Variant:
 734                 enumerated_items = enumerate(entry.base_t.types.items())
 735
 736                 # lookup each type, until the current visiting index is met
 737                 for i, (type_name, type_type) in enumerated_items:
 738                     if i == entry.index:
 739                         break
 740
 741                     if type_name == find_name:
 742                         if not got_struct:
 743                             raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 744
 745                         return type_type
 746
 747         # nothing returned here: cannot find type
 748         raise ConfigError('invalid path "{}": cannot find type in current context'.format(path))
 749
 750     def _lookup_path(self, path, from_t):
 751         parts = path.lower().split('.')
 752         base = None
 753         base_is_current = False
 754
 755         if len(parts) >= 3:
 756             if parts[0] == 'trace':
 757                 if parts[1] == 'packet' and parts[2] == 'header':
 758                     # make sure packet header exists
 759                     if self._cur_trace.packet_header_type is None:
 760                         raise ConfigError('invalid path "{}": no defined trace packet header type'.format(path))
 761
 762                     base = self._cur_trace.packet_header_type
 763
 764                     if self._cur_entity == _Entity.TRACE_PACKET_HEADER:
 765                         base_is_current = True
 766                 else:
 767                     raise ConfigError('invalid path "{}": unknown names after "trace"'.format(path))
 768             elif parts[0] == 'stream':
 769                 if parts[1] == 'packet' and parts[2] == 'context':
 770                     if self._cur_entity < _Entity.STREAM_PACKET_CONTEXT:
 771                         raise ConfigError('invalid path "{}": cannot access stream packet context here'.format(path))
 772
 773                     if self._cur_stream.packet_context_type is None:
 774                         raise ConfigError('invalid path "{}": no defined stream packet context type'.format(path))
 775
 776                     base = self._cur_stream.packet_context_type
 777
 778                     if self._cur_entity == _Entity.STREAM_PACKET_CONTEXT:
 779                         base_is_current = True
 780                 elif parts[1] == 'event':
 781                     if parts[2] == 'header':
 782                         if self._cur_entity < _Entity.STREAM_EVENT_HEADER:
 783                             raise ConfigError('invalid path "{}": cannot access stream event header here'.format(path))
 784
 785                         if self._cur_stream.event_header_type is None:
 786                             raise ConfigError('invalid path "{}": no defined stream event header type'.format(path))
 787
 788                         base = self._cur_stream.event_header_type
 789
 790                         if self._cur_entity == _Entity.STREAM_EVENT_HEADER:
 791                             base_is_current = True
 792                     elif parts[2] == 'context':
 793                         if self._cur_entity < _Entity.STREAM_EVENT_CONTEXT:
 794                             raise ConfigError('invalid path "{}": cannot access stream event context here'.format(path))
 795
 796                         if self._cur_stream.event_context_type is None:
 797                             raise ConfigError('invalid path "{}": no defined stream event context type'.format(path))
 798
 799                         base = self._cur_stream.event_context_type
 800
 801                         if self._cur_entity == _Entity.STREAM_EVENT_CONTEXT:
 802                             base_is_current = True
 803                     else:
 804                         raise ConfigError('invalid path "{}": unknown names after "stream.event"'.format(path))
 805                 else:
 806                     raise ConfigError('invalid path "{}": unknown names after "stream"'.format(path))
 807
 808             if base is not None:
 809                 start_index = 3
 810
 811         if len(parts) >= 2 and base is None:
 812             if parts[0] == 'event':
 813                 if parts[1] == 'context':
 814                     if self._cur_entity < _Entity.EVENT_CONTEXT:
 815                         raise ConfigError('invalid path "{}": cannot access event context here'.format(path))
 816
 817                     if self._cur_event.context_type is None:
 818                         raise ConfigError('invalid path "{}": no defined event context type'.format(path))
 819
 820                     base = self._cur_event.context_type
 821
 822                     if self._cur_entity == _Entity.EVENT_CONTEXT:
 823                         base_is_current = True
 824                 elif parts[1] == 'payload' or parts[1] == 'fields':
 825                     if self._cur_entity < _Entity.EVENT_PAYLOAD:
 826                         raise ConfigError('invalid path "{}": cannot access event payload here'.format(path))
 827
 828                     if self._cur_event.payload_type is None:
 829                         raise ConfigError('invalid path "{}": no defined event payload type'.format(path))
 830
 831                     base = self._cur_event.payload_type
 832
 833                     if self._cur_entity == _Entity.EVENT_PAYLOAD:
 834                         base_is_current = True
 835                 else:
 836                     raise ConfigError('invalid path "{}": unknown names after "event"'.format(path))
 837
 838             if base is not None:
 839                 start_index = 2
 840
 841         if base is not None:
 842             return self._lookup_path_from_base(path, parts, base, start_index,
 843                                                base_is_current, from_t)
 844         else:
 845             return self._lookup_path_from_top(path, parts)
 846
 847     def _stack_reset(self):
 848         self._stack = []
 849
 850     def _stack_push(self, base_t):
 851         entry = _MetadataDynamicTypesValidatorStackEntry(base_t)
 852         self._stack.append(entry)
 853
 854     def _stack_pop(self):
 855         self._stack.pop()
 856
 857     def _stack_incr_index(self):
 858         self._stack[-1].index += 1
 859
 860     def _visit_struct_type(self, t):
 861         self._stack_push(t)
 862
 863         for field_name, field_type in t.fields.items():
 864             try:
 865                 self._visit_type(field_type)
 866             except Exception as e:
 867                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 868
 869             self._stack_incr_index()
 870
 871         self._stack_pop()
 872
 873     def _visit_array_type(self, t):
 874         if t.is_variable_length:
 875             # find length type
 876             try:
 877                 length_type = self._lookup_path(t.length, t)
 878             except Exception as e:
 879                 raise ConfigError('invalid array type\'s length', e)
 880
 881             # make sure length type an unsigned integer
 882             if type(length_type) is not metadata.Integer:
 883                 raise ConfigError('array type\'s length does not point to an integer type')
 884
 885             if length_type.signed:
 886                 raise ConfigError('array type\'s length does not point to an unsigned integer type')
 887
 888         self._visit_type(t.element_type)
 889
 890     def _visit_variant_type(self, t):
 891         # find tag type
 892         try:
 893             tag_type = self._lookup_path(t.tag, t)
 894         except Exception as e:
 895             raise ConfigError('invalid variant type\'s tag', e)
 896
 897         # make sure tag type is an enumeration
 898         if type(tag_type) is not metadata.Enum:
 899             raise ConfigError('variant type\'s tag does not point to an enumeration type')
 900
 901         # verify that each variant type's type exists as an enumeration member
 902         for tag_name in t.types.keys():
 903             if tag_name not in tag_type.members:
 904                 raise ConfigError('cannot find variant type\'s type "{}" in pointed tag type'.format(tag_name))
 905
 906         self._stack_push(t)
 907
 908         for type_name, type_type in t.types.items():
 909             try:
 910                 self._visit_type(type_type)
 911             except Exception as e:
 912                 raise ConfigError('in variant type\'s type "{}"'.format(type_name), e)
 913
 914             self._stack_incr_index()
 915
 916         self._stack_pop()
 917
 918     def _visit_type(self, t):
 919         if t is None:
 920             return
 921
 922         if type(t) in self._type_to_visit_type_func:
 923             func = self._type_to_visit_type_func[type(t)]
 924
 925             if func is not None:
 926                 func(t)
 927
 928     def _visit_event(self, ev):
 929         ev_name = ev.name
 930
 931         # set current event
 932         self._cur_event = ev
 933
 934         # visit event context type
 935         self._stack_reset()
 936         self._cur_entity = _Entity.EVENT_CONTEXT
 937
 938         try:
 939             self._visit_type(ev.context_type)
 940         except Exception as e:
 941             raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 942
 943         # visit event payload type
 944         self._stack_reset()
 945         self._cur_entity = _Entity.EVENT_PAYLOAD
 946
 947         try:
 948             self._visit_type(ev.payload_type)
 949         except Exception as e:
 950             raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 951
 952     def _visit_stream(self, stream):
 953         stream_name = stream.name
 954
 955         # set current stream
 956         self._cur_stream = stream
 957
 958         # reset current event
 959         self._cur_event = None
 960
 961         # visit stream packet context type
 962         self._stack_reset()
 963         self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 964
 965         try:
 966             self._visit_type(stream.packet_context_type)
 967         except Exception as e:
 968             raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 969
 970         # visit stream event header type
 971         self._stack_reset()
 972         self._cur_entity = _Entity.STREAM_EVENT_HEADER
 973
 974         try:
 975             self._visit_type(stream.event_header_type)
 976         except Exception as e:
 977             raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 978
 979         # visit stream event context type
 980         self._stack_reset()
 981         self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 982
 983         try:
 984             self._visit_type(stream.event_context_type)
 985         except Exception as e:
 986             raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 987
 988         # visit events
 989         for ev in stream.events.values():
 990             try:
 991                 self._visit_event(ev)
 992             except Exception as e:
 993                 raise ConfigError('invalid stream "{}"'.format(stream_name))
 994
 995     def validate(self, meta):
 996         # set current trace
 997         self._cur_trace = meta.trace
 998
 999         # visit trace packet header type
1000         self._stack_reset()
1001         self._cur_entity = _Entity.TRACE_PACKET_HEADER
1002
1003         try:
1004             self._visit_type(meta.trace.packet_header_type)
1005         except Exception as e:
1006             raise ConfigError('invalid packet header type in trace', e)
1007
1008         # visit streams
1009         for stream in meta.streams.values():
1010             self._visit_stream(stream)
1011
1012
1013 # Since type inheritance allows types to be only partially defined at
1014 # any place in the configuration, this validator validates that actual
1015 # trace, stream, and event types are all complete and valid. Therefore
1016 # an invalid, but unusued type alias is accepted.
1017 class _MetadataTypesHistologyValidator:
1018     def __init__(self):
1019         self._type_to_validate_type_histology_func = {
1020             metadata.Integer: self._validate_integer_histology,
1021             metadata.FloatingPoint: self._validate_float_histology,
1022             metadata.Enum: self._validate_enum_histology,
1023             metadata.String: self._validate_string_histology,
1024             metadata.Struct: self._validate_struct_histology,
1025             metadata.Array: self._validate_array_histology,
1026             metadata.Variant: self._validate_variant_histology,
1027         }
1028
1029     def _validate_integer_histology(self, t):
1030         # size is set
1031         if t.size is None:
1032             raise ConfigError('missing integer type\'s size')
1033
1034     def _validate_float_histology(self, t):
1035         # exponent digits is set
1036         if t.exp_size is None:
1037             raise ConfigError('missing floating point number type\'s exponent size')
1038
1039         # mantissa digits is set
1040         if t.mant_size is None:
1041             raise ConfigError('missing floating point number type\'s mantissa size')
1042
1043         # exponent and mantissa sum is a multiple of 8
1044         if (t.exp_size + t.mant_size) % 8 != 0:
1045             raise ConfigError('floating point number type\'s mantissa and exponent sizes sum must be a multiple of 8')
1046
1047     def _validate_enum_histology(self, t):
1048         # integer type is set
1049         if t.value_type is None:
1050             raise ConfigError('missing enumeration type\'s value type')
1051
1052         # there's at least one member
1053         if not t.members:
1054             raise ConfigError('enumeration type needs at least one member')
1055
1056         # no overlapping values and all values are valid considering
1057         # the value type
1058         ranges = []
1059
1060         if t.value_type.signed:
1061             value_min = -(1 << t.value_type.size - 1)
1062             value_max = (1 << (t.value_type.size - 1)) - 1
1063         else:
1064             value_min = 0
1065             value_max = (1 << t.value_type.size) - 1
1066
1067         for label, value in t.members.items():
1068             for rg in ranges:
1069                 if value[0] <= rg[1] and rg[0] <= value[1]:
1070                     raise ConfigError('enumeration type\'s member "{}" overlaps another member'.format(label))
1071
1072             fmt = 'enumeration type\'s member "{}": value {} is outside the value type range [{}, {}]'
1073
1074             if value[0] < value_min or value[0] > value_max:
1075                 raise ConfigError(fmt.format(label, value[0], value_min, value_max))
1076
1077             if value[1] < value_min or value[1] > value_max:
1078                 raise ConfigError(fmt.format(label, value[1], value_min, value_max))
1079
1080             ranges.append(value)
1081
1082     def _validate_string_histology(self, t):
1083         # always valid
1084         pass
1085
1086     def _validate_struct_histology(self, t):
1087         # all fields are valid
1088         for field_name, field_type in t.fields.items():
1089             try:
1090                 self._validate_type_histology(field_type)
1091             except Exception as e:
1092                 raise ConfigError('invalid structure type\'s field "{}"'.format(field_name), e)
1093
1094     def _validate_array_histology(self, t):
1095         # length is set
1096         if t.length is None:
1097             raise ConfigError('missing array type\'s length')
1098
1099         # element type is set
1100         if t.element_type is None:
1101             raise ConfigError('missing array type\'s element type')
1102
1103         # element type is valid
1104         try:
1105             self._validate_type_histology(t.element_type)
1106         except Exception as e:
1107             raise ConfigError('invalid array type\'s element type', e)
1108
1109     def _validate_variant_histology(self, t):
1110         # tag is set
1111         if t.tag is None:
1112             raise ConfigError('missing variant type\'s tag')
1113
1114         # there's at least one type
1115         if not t.types:
1116             raise ConfigError('variant type needs at least one type')
1117
1118         # all types are valid
1119         for type_name, type_t in t.types.items():
1120             try:
1121                 self._validate_type_histology(type_t)
1122             except Exception as e:
1123                 raise ConfigError('invalid variant type\'s type "{}"'.format(type_name), e)
1124
1125     def _validate_type_histology(self, t):
1126         if t is None:
1127             return
1128
1129         self._type_to_validate_type_histology_func[type(t)](t)
1130
1131     def _validate_entity_type_histology(self, t):
1132         if t is None:
1133             return
1134
1135         if type(t) is not metadata.Struct:
1136             raise ConfigError('expecting a structure type')
1137
1138         self._validate_type_histology(t)
1139
1140     def _validate_event_types_histology(self, ev):
1141         ev_name = ev.name
1142
1143         # validate event context type
1144         try:
1145             self._validate_entity_type_histology(ev.context_type)
1146         except Exception as e:
1147             raise ConfigError('invalid event context type for event "{}"'.format(ev_name), e)
1148
1149         # validate event payload type
1150         try:
1151             self._validate_entity_type_histology(ev.payload_type)
1152         except Exception as e:
1153             raise ConfigError('invalid event payload type for event "{}"'.format(ev_name), e)
1154
1155     def _validate_stream_types_histology(self, stream):
1156         stream_name = stream.name
1157
1158         # validate stream packet context type
1159         try:
1160             self._validate_entity_type_histology(stream.packet_context_type)
1161         except Exception as e:
1162             raise ConfigError('invalid stream packet context type for stream "{}"'.format(stream_name), e)
1163
1164         # validate stream event header type
1165         try:
1166             self._validate_entity_type_histology(stream.event_header_type)
1167         except Exception as e:
1168             raise ConfigError('invalid stream event header type for stream "{}"'.format(stream_name), e)
1169
1170         # validate stream event context type
1171         try:
1172             self._validate_entity_type_histology(stream.event_context_type)
1173         except Exception as e:
1174             raise ConfigError('invalid stream event context type for stream "{}"'.format(stream_name), e)
1175
1176         # validate events
1177         for ev in stream.events.values():
1178             try:
1179                 self._validate_event_types_histology(ev)
1180             except Exception as e:
1181                 raise ConfigError('invalid event in stream "{}"'.format(stream_name), e)
1182
1183     def validate(self, meta):
1184         # validate trace packet header type
1185         try:
1186             self._validate_entity_type_histology(meta.trace.packet_header_type)
1187         except Exception as e:
1188             raise ConfigError('invalid trace packet header type', e)
1189
1190         # validate streams
1191         for stream in meta.streams.values():
1192             self._validate_stream_types_histology(stream)
1193
1194
1195 class _YamlConfigParser:
1196     def __init__(self, include_dirs, ignore_include_not_found, dump_config):
1197         self._class_name_to_create_type_func = {
1198             'int': self._create_integer,
1199             'integer': self._create_integer,
1200             'flt': self._create_float,
1201             'float': self._create_float,
1202             'floating-point': self._create_float,
1203             'enum': self._create_enum,
1204             'enumeration': self._create_enum,
1205             'str': self._create_string,
1206             'string': self._create_string,
1207             'struct': self._create_struct,
1208             'structure': self._create_struct,
1209             'array': self._create_array,
1210             'var': self._create_variant,
1211             'variant': self._create_variant,
1212         }
1213         self._type_to_create_type_func = {
1214             metadata.Integer: self._create_integer,
1215             metadata.FloatingPoint: self._create_float,
1216             metadata.Enum: self._create_enum,
1217             metadata.String: self._create_string,
1218             metadata.Struct: self._create_struct,
1219             metadata.Array: self._create_array,
1220             metadata.Variant: self._create_variant,
1221         }
1222         self._include_dirs = include_dirs
1223         self._ignore_include_not_found = ignore_include_not_found
1224         self._dump_config = dump_config
1225
1226     def _set_byte_order(self, metadata_node):
1227         if 'trace' not in metadata_node:
1228             raise ConfigError('missing "trace" property (metadata)')
1229
1230         trace_node = metadata_node['trace']
1231
1232         if not _is_assoc_array_prop(trace_node):
1233             raise ConfigError('"trace" property (metadata) must be an associative array')
1234
1235         if 'byte-order' not in trace_node:
1236             raise ConfigError('missing "byte-order" property (trace)')
1237
1238         bo_node = trace_node['byte-order']
1239
1240         if not _is_str_prop(bo_node):
1241             raise ConfigError('"byte-order" property of trace object must be a string ("le" or "be")')
1242
1243         self._bo = _byte_order_str_to_bo(bo_node)
1244
1245         if self._bo is None:
1246             raise ConfigError('invalid "byte-order" property (trace): must be "le" or "be"')
1247
1248     def _lookup_type_alias(self, name):
1249         if name in self._tas:
1250             return copy.deepcopy(self._tas[name])
1251
1252     def _set_int_clock_prop_mapping(self, int_obj, prop_mapping_node):
1253         unk_prop = _get_first_unknown_prop(prop_mapping_node, ['type', 'name', 'property'])
1254
1255         if unk_prop:
1256             raise ConfigError('unknown property in integer type object\'s clock property mapping: "{}"'.format(unk_prop))
1257
1258         if 'name' not in prop_mapping_node:
1259             raise ConfigError('missing "name" property in integer type object\'s clock property mapping')
1260
1261         if 'property' not in prop_mapping_node:
1262             raise ConfigError('missing "property" property in integer type object\'s clock property mapping')
1263
1264         clock_name = prop_mapping_node['name']
1265         prop = prop_mapping_node['property']
1266
1267         if not _is_str_prop(clock_name):
1268             raise ConfigError('"name" property of integer type object\'s clock property mapping must be a string')
1269
1270         if not _is_str_prop(prop):
1271             raise ConfigError('"property" property of integer type object\'s clock property mapping must be a string')
1272
1273         if clock_name not in self._clocks:
1274             raise ConfigError('invalid clock name "{}" in integer type object\'s clock property mapping'.format(clock_name))
1275
1276         if prop != 'value':
1277             raise ConfigError('invalid "property" property in integer type object\'s clock property mapping: "{}"'.format(prop))
1278
1279         mapped_clock = self._clocks[clock_name]
1280         int_obj.property_mappings.append(metadata.PropertyMapping(mapped_clock, prop))
1281
1282     def _get_first_unknown_type_prop(self, type_node, known_props):
1283         kp = known_props + ['inherit', 'class']
1284
1285         if self._version >= 201:
1286             kp.append('$inherit')
1287
1288         return _get_first_unknown_prop(type_node, kp)
1289
1290     def _create_integer(self, obj, node):
1291         if obj is None:
1292             # create integer object
1293             obj = metadata.Integer()
1294
1295         unk_prop = self._get_first_unknown_type_prop(node, [
1296             'size',
1297             'align',
1298             'signed',
1299             'byte-order',
1300             'base',
1301             'encoding',
1302             'property-mappings',
1303         ])
1304
1305         if unk_prop:
1306             raise ConfigError('unknown integer type object property: "{}"'.format(unk_prop))
1307
1308         # size
1309         if 'size' in node:
1310             size = node['size']
1311
1312             if not _is_int_prop(size):
1313                 raise ConfigError('"size" property of integer type object must be an integer')
1314
1315             if size < 1:
1316                 raise ConfigError('invalid integer size: {}'.format(size))
1317
1318             obj.size = size
1319
1320         # align
1321         if 'align' in node:
1322             align = node['align']
1323
1324             if align is None:
1325                 obj.set_default_align()
1326             else:
1327                 if not _is_int_prop(align):
1328                     raise ConfigError('"align" property of integer type object must be an integer')
1329
1330                 if not _is_valid_alignment(align):
1331                     raise ConfigError('invalid alignment: {}'.format(align))
1332
1333                 obj.align = align
1334
1335         # signed
1336         if 'signed' in node:
1337             signed = node['signed']
1338
1339             if signed is None:
1340                 obj.set_default_signed()
1341             else:
1342                 if not _is_bool_prop(signed):
1343                     raise ConfigError('"signed" property of integer type object must be a boolean')
1344
1345                 obj.signed = signed
1346
1347         # byte order
1348         if 'byte-order' in node:
1349             byte_order = node['byte-order']
1350
1351             if byte_order is None:
1352                 obj.byte_order = self._bo
1353             else:
1354                 if not _is_str_prop(byte_order):
1355                     raise ConfigError('"byte-order" property of integer type object must be a string ("le" or "be")')
1356
1357                 byte_order = _byte_order_str_to_bo(byte_order)
1358
1359                 if byte_order is None:
1360                     raise ConfigError('invalid "byte-order" property in integer type object')
1361
1362                 obj.byte_order = byte_order
1363         else:
1364             obj.byte_order = self._bo
1365
1366         # base
1367         if 'base' in node:
1368             base = node['base']
1369
1370             if base is None:
1371                 obj.set_default_base()
1372             else:
1373                 if not _is_str_prop(base):
1374                     raise ConfigError('"base" property of integer type object must be a string ("bin", "oct", "dec", or "hex")')
1375
1376                 if base == 'bin':
1377                     base = 2
1378                 elif base == 'oct':
1379                     base = 8
1380                 elif base == 'dec':
1381                     base = 10
1382                 elif base == 'hex':
1383                     base = 16
1384                 else:
1385                     raise ConfigError('unknown "base" property value: "{}" ("bin", "oct", "dec", and "hex" are accepted)'.format(base))
1386
1387                 obj.base = base
1388
1389         # encoding
1390         if 'encoding' in node:
1391             encoding = node['encoding']
1392
1393             if encoding is None:
1394                 obj.set_default_encoding()
1395             else:
1396                 if not _is_str_prop(encoding):
1397                     raise ConfigError('"encoding" property of integer type object must be a string ("none", "ascii", or "utf-8")')
1398
1399                 encoding = _encoding_str_to_encoding(encoding)
1400
1401                 if encoding is None:
1402                     raise ConfigError('invalid "encoding" property in integer type object')
1403
1404                 obj.encoding = encoding
1405
1406         # property mappings
1407         if 'property-mappings' in node:
1408             prop_mappings = node['property-mappings']
1409
1410             if prop_mappings is None:
1411                 obj.set_default_property_mappings()
1412             else:
1413                 if not _is_array_prop(prop_mappings):
1414                     raise ConfigError('"property-mappings" property of integer type object must be an array')
1415
1416                 if len(prop_mappings) > 1:
1417                     raise ConfigError('length of "property-mappings" array in integer type object must be 1')
1418
1419                 for index, prop_mapping in enumerate(prop_mappings):
1420                     if not _is_assoc_array_prop(prop_mapping):
1421                         raise ConfigError('elements of "property-mappings" property of integer type object must be associative arrays')
1422
1423                     if 'type' not in prop_mapping:
1424                         raise ConfigError('missing "type" property in integer type object\'s "property-mappings" array\'s element #{}'.format(index))
1425
1426                     prop_type = prop_mapping['type']
1427
1428                     if not _is_str_prop(prop_type):
1429                         raise ConfigError('"type" property of integer type object\'s "property-mappings" array\'s element #{} must be a string'.format(index))
1430
1431                     if prop_type == 'clock':
1432                         self._set_int_clock_prop_mapping(obj, prop_mapping)
1433                     else:
1434                         raise ConfigError('unknown property mapping type "{}" in integer type object\'s "property-mappings" array\'s element #{}'.format(prop_type, index))
1435
1436         return obj
1437
1438     def _create_float(self, obj, node):
1439         if obj is None:
1440             # create floating point number object
1441             obj = metadata.FloatingPoint()
1442
1443         unk_prop = self._get_first_unknown_type_prop(node, [
1444             'size',
1445             'align',
1446             'byte-order',
1447         ])
1448
1449         if unk_prop:
1450             raise ConfigError('unknown floating point number type object property: "{}"'.format(unk_prop))
1451
1452         # size
1453         if 'size' in node:
1454             size = node['size']
1455
1456             if not _is_assoc_array_prop(size):
1457                 raise ConfigError('"size" property of floating point number type object must be an associative array')
1458
1459             unk_prop = _get_first_unknown_prop(size, ['exp', 'mant'])
1460
1461             if unk_prop:
1462                 raise ConfigError('unknown floating point number type object\'s "size" property: "{}"'.format(unk_prop))
1463
1464             if 'exp' in size:
1465                 exp = size['exp']
1466
1467                 if not _is_int_prop(exp):
1468                     raise ConfigError('"exp" property of floating point number type object\'s "size" property must be an integer')
1469
1470                 if exp < 1:
1471                     raise ConfigError('invalid floating point number exponent size: {}')
1472
1473                 obj.exp_size = exp
1474
1475             if 'mant' in size:
1476                 mant = size['mant']
1477
1478                 if not _is_int_prop(mant):
1479                     raise ConfigError('"mant" property of floating point number type object\'s "size" property must be an integer')
1480
1481                 if mant < 1:
1482                     raise ConfigError('invalid floating point number mantissa size: {}')
1483
1484                 obj.mant_size = mant
1485
1486         # align
1487         if 'align' in node:
1488             align = node['align']
1489
1490             if align is None:
1491                 obj.set_default_align()
1492             else:
1493                 if not _is_int_prop(align):
1494                     raise ConfigError('"align" property of floating point number type object must be an integer')
1495
1496                 if not _is_valid_alignment(align):
1497                     raise ConfigError('invalid alignment: {}'.format(align))
1498
1499                 obj.align = align
1500
1501         # byte order
1502         if 'byte-order' in node:
1503             byte_order = node['byte-order']
1504
1505             if byte_order is None:
1506                 obj.byte_order = self._bo
1507             else:
1508                 if not _is_str_prop(byte_order):
1509                     raise ConfigError('"byte-order" property of floating point number type object must be a string ("le" or "be")')
1510
1511                 byte_order = _byte_order_str_to_bo(byte_order)
1512
1513                 if byte_order is None:
1514                     raise ConfigError('invalid "byte-order" property in floating point number type object')
1515         else:
1516             obj.byte_order = self._bo
1517
1518         return obj
1519
1520     def _create_enum(self, obj, node):
1521         if obj is None:
1522             # create enumeration object
1523             obj = metadata.Enum()
1524
1525         unk_prop = self._get_first_unknown_type_prop(node, [
1526             'value-type',
1527             'members',
1528         ])
1529
1530         if unk_prop:
1531             raise ConfigError('unknown enumeration type object property: "{}"'.format(unk_prop))
1532
1533         # value type
1534         if 'value-type' in node:
1535             value_type_node = node['value-type']
1536
1537             try:
1538                 obj.value_type = self._create_type(value_type_node)
1539             except Exception as e:
1540                 raise ConfigError('cannot create enumeration type\'s integer type', e)
1541
1542         # members
1543         if 'members' in node:
1544             members_node = node['members']
1545
1546             if not _is_array_prop(members_node):
1547                 raise ConfigError('"members" property of enumeration type object must be an array')
1548
1549             cur = 0
1550             last_value = obj.last_value
1551
1552             if last_value is None:
1553                 cur = 0
1554             else:
1555                 cur = last_value + 1
1556
1557             for index, m_node in enumerate(members_node):
1558                 if not _is_str_prop(m_node) and not _is_assoc_array_prop(m_node):
1559                     raise ConfigError('invalid enumeration member #{}: expecting a string or an associative array'.format(index))
1560
1561                 if _is_str_prop(m_node):
1562                     label = m_node
1563                     value = (cur, cur)
1564                     cur += 1
1565                 else:
1566                     unk_prop = _get_first_unknown_prop(m_node, [
1567                         'label',
1568                         'value',
1569                     ])
1570
1571                     if unk_prop:
1572                         raise ConfigError('unknown enumeration type member object property: "{}"'.format(unk_prop))
1573
1574                     if 'label' not in m_node:
1575                         raise ConfigError('missing "label" property in enumeration member #{}'.format(index))
1576
1577                     label = m_node['label']
1578
1579                     if not _is_str_prop(label):
1580                         raise ConfigError('"label" property of enumeration member #{} must be a string'.format(index))
1581
1582                     if 'value' not in m_node:
1583                         raise ConfigError('missing "value" property in enumeration member ("{}")'.format(label))
1584
1585                     value = m_node['value']
1586
1587                     if not _is_int_prop(value) and not _is_array_prop(value):
1588                         raise ConfigError('invalid enumeration member ("{}"): expecting an integer or an array'.format(label))
1589
1590                     if _is_int_prop(value):
1591                         cur = value + 1
1592                         value = (value, value)
1593                     else:
1594                         if len(value) != 2:
1595                             raise ConfigError('invalid enumeration member ("{}"): range must have exactly two items'.format(label))
1596
1597                         mn = value[0]
1598                         mx = value[1]
1599
1600                         if mn > mx:
1601                             raise ConfigError('invalid enumeration member ("{}"): invalid range ({} > {})'.format(label, mn, mx))
1602
1603                         value = (mn, mx)
1604                         cur = mx + 1
1605
1606                 obj.members[label] = value
1607
1608         return obj
1609
1610     def _create_string(self, obj, node):
1611         if obj is None:
1612             # create string object
1613             obj = metadata.String()
1614
1615         unk_prop = self._get_first_unknown_type_prop(node, [
1616             'encoding',
1617         ])
1618
1619         if unk_prop:
1620             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1621
1622         # encoding
1623         if 'encoding' in node:
1624             encoding = node['encoding']
1625
1626             if encoding is None:
1627                 obj.set_default_encoding()
1628             else:
1629                 if not _is_str_prop(encoding):
1630                     raise ConfigError('"encoding" property of string type object must be a string ("none", "ascii", or "utf-8")')
1631
1632                 encoding = _encoding_str_to_encoding(encoding)
1633
1634                 if encoding is None:
1635                     raise ConfigError('invalid "encoding" property in string type object')
1636
1637                 obj.encoding = encoding
1638
1639         return obj
1640
1641     def _create_struct(self, obj, node):
1642         if obj is None:
1643             # create structure object
1644             obj = metadata.Struct()
1645
1646         unk_prop = self._get_first_unknown_type_prop(node, [
1647             'min-align',
1648             'fields',
1649         ])
1650
1651         if unk_prop:
1652             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1653
1654         # minimum alignment
1655         if 'min-align' in node:
1656             min_align = node['min-align']
1657
1658             if min_align is None:
1659                 obj.set_default_min_align()
1660             else:
1661                 if not _is_int_prop(min_align):
1662                     raise ConfigError('"min-align" property of structure type object must be an integer')
1663
1664                 if not _is_valid_alignment(min_align):
1665                     raise ConfigError('invalid minimum alignment: {}'.format(min_align))
1666
1667                 obj.min_align = min_align
1668
1669         # fields
1670         if 'fields' in node:
1671             fields = node['fields']
1672
1673             if fields is None:
1674                 obj.set_default_fields()
1675             else:
1676                 if not _is_assoc_array_prop(fields):
1677                     raise ConfigError('"fields" property of structure type object must be an associative array')
1678
1679                 for field_name, field_node in fields.items():
1680                     if not _is_valid_identifier(field_name):
1681                         raise ConfigError('"{}" is not a valid field name for structure type'.format(field_name))
1682
1683                     try:
1684                         obj.fields[field_name] = self._create_type(field_node)
1685                     except Exception as e:
1686                         raise ConfigError('cannot create structure type\'s field "{}"'.format(field_name), e)
1687
1688         return obj
1689
1690     def _create_array(self, obj, node):
1691         if obj is None:
1692             # create array object
1693             obj = metadata.Array()
1694
1695         unk_prop = self._get_first_unknown_type_prop(node, [
1696             'length',
1697             'element-type',
1698         ])
1699
1700         if unk_prop:
1701             raise ConfigError('unknown array type object property: "{}"'.format(unk_prop))
1702
1703         # length
1704         if 'length' in node:
1705             length = node['length']
1706
1707             if not _is_int_prop(length) and not _is_str_prop(length):
1708                 raise ConfigError('"length" property of array type object must be an integer or a string')
1709
1710             if type(length) is int and length < 0:
1711                 raise ConfigError('invalid static array length: {}'.format(length))
1712
1713             obj.length = length
1714
1715         # element type
1716         if 'element-type' in node:
1717             element_type_node = node['element-type']
1718
1719             try:
1720                 obj.element_type = self._create_type(node['element-type'])
1721             except Exception as e:
1722                 raise ConfigError('cannot create array type\'s element type', e)
1723
1724         return obj
1725
1726     def _create_variant(self, obj, node):
1727         if obj is None:
1728             # create variant object
1729             obj = metadata.Variant()
1730
1731         unk_prop = self._get_first_unknown_type_prop(node, [
1732             'tag',
1733             'types',
1734         ])
1735
1736         if unk_prop:
1737             raise ConfigError('unknown variant type object property: "{}"'.format(unk_prop))
1738
1739         # tag
1740         if 'tag' in node:
1741             tag = node['tag']
1742
1743             if not _is_str_prop(tag):
1744                 raise ConfigError('"tag" property of variant type object must be a string')
1745
1746             # do not validate variant tag for the moment; will be done in a
1747             # second phase
1748             obj.tag = tag
1749
1750         # element type
1751         if 'types' in node:
1752             types = node['types']
1753
1754             if not _is_assoc_array_prop(types):
1755                 raise ConfigError('"types" property of variant type object must be an associative array')
1756
1757             # do not validate type names for the moment; will be done in a
1758             # second phase
1759             for type_name, type_node in types.items():
1760                 if not _is_valid_identifier(type_name):
1761                     raise ConfigError('"{}" is not a valid type name for variant type'.format(type_name))
1762
1763                 try:
1764                     obj.types[type_name] = self._create_type(type_node)
1765                 except Exception as e:
1766                     raise ConfigError('cannot create variant type\'s type "{}"'.format(type_name), e)
1767
1768         return obj
1769
1770     def _create_type(self, type_node):
1771         if type(type_node) is str:
1772             t = self._lookup_type_alias(type_node)
1773
1774             if t is None:
1775                 raise ConfigError('unknown type alias "{}"'.format(type_node))
1776
1777             return t
1778
1779         if not _is_assoc_array_prop(type_node):
1780             raise ConfigError('type objects must be associative arrays or strings (type alias name)')
1781
1782         # inherit:
1783         #   v2.0:  "inherit"
1784         #   v2.1+: "$inherit"
1785         inherit_node = None
1786
1787         if self._version >= 200:
1788             if 'inherit' in type_node:
1789                 inherit_prop = 'inherit'
1790                 inherit_node = type_node[inherit_prop]
1791
1792         if self._version >= 201:
1793             if '$inherit' in type_node:
1794                 if inherit_node is not None:
1795                     raise ConfigError('cannot specify both "inherit" and "$inherit" properties of type object: prefer "$inherit"')
1796
1797                 inherit_prop = '$inherit'
1798                 inherit_node = type_node[inherit_prop]
1799
1800         if inherit_node is not None and 'class' in type_node:
1801             raise ConfigError('cannot specify both "{}" and "class" properties in type object'.format(inherit_prop))
1802
1803         if inherit_node is not None:
1804             if not _is_str_prop(inherit_node):
1805                 raise ConfigError('"{}" property of type object must be a string'.format(inherit_prop))
1806
1807             base = self._lookup_type_alias(inherit_node)
1808
1809             if base is None:
1810                 raise ConfigError('cannot inherit from type alias "{}": type alias does not exist at this point'.format(inherit_node))
1811
1812             func = self._type_to_create_type_func[type(base)]
1813         else:
1814             if 'class' not in type_node:
1815                 raise ConfigError('type objects which do not inherit must have a "class" property')
1816
1817             class_name = type_node['class']
1818
1819             if type(class_name) is not str:
1820                 raise ConfigError('type objects\' "class" property must be a string')
1821
1822             if class_name not in self._class_name_to_create_type_func:
1823                 raise ConfigError('unknown type class "{}"'.format(class_name))
1824
1825             base = None
1826             func = self._class_name_to_create_type_func[class_name]
1827
1828         return func(base, type_node)
1829
1830     def _register_type_aliases(self, metadata_node):
1831         self._tas = dict()
1832
1833         if 'type-aliases' not in metadata_node:
1834             return
1835
1836         ta_node = metadata_node['type-aliases']
1837
1838         if ta_node is None:
1839             return
1840
1841         if not _is_assoc_array_prop(ta_node):
1842             raise ConfigError('"type-aliases" property (metadata) must be an associative array')
1843
1844         for ta_name, ta_type in ta_node.items():
1845             if ta_name in self._tas:
1846                 raise ConfigError('duplicate type alias "{}"'.format(ta_name))
1847
1848             try:
1849                 t = self._create_type(ta_type)
1850             except Exception as e:
1851                 raise ConfigError('cannot create type alias "{}"'.format(ta_name), e)
1852
1853             self._tas[ta_name] = t
1854
1855     def _create_clock(self, node):
1856         # create clock object
1857         clock = metadata.Clock()
1858
1859         if not _is_assoc_array_prop(node):
1860             raise ConfigError('clock objects must be associative arrays')
1861
1862         known_props = [
1863             'uuid',
1864             'description',
1865             'freq',
1866             'error-cycles',
1867             'offset',
1868             'absolute',
1869             'return-ctype',
1870         ]
1871
1872         if self._version >= 201:
1873             known_props.append('$return-ctype')
1874
1875         unk_prop = _get_first_unknown_prop(node, known_props)
1876
1877         if unk_prop:
1878             raise ConfigError('unknown clock object property: "{}"'.format(unk_prop))
1879
1880         # UUID
1881         if 'uuid' in node:
1882             uuidp = node['uuid']
1883
1884             if uuidp is None:
1885                 clock.set_default_uuid()
1886             else:
1887                 if not _is_str_prop(uuidp):
1888                     raise ConfigError('"uuid" property of clock object must be a string')
1889
1890                 try:
1891                     uuidp = uuid.UUID(uuidp)
1892                 except:
1893                     raise ConfigError('malformed UUID (clock object): "{}"'.format(uuidp))
1894
1895                 clock.uuid = uuidp
1896
1897         # description
1898         if 'description' in node:
1899             desc = node['description']
1900
1901             if desc is None:
1902                 clock.set_default_description()
1903             else:
1904                 if not _is_str_prop(desc):
1905                     raise ConfigError('"description" property of clock object must be a string')
1906
1907                 clock.description = desc
1908
1909         # frequency
1910         if 'freq' in node:
1911             freq = node['freq']
1912
1913             if freq is None:
1914                 clock.set_default_freq()
1915             else:
1916                 if not _is_int_prop(freq):
1917                     raise ConfigError('"freq" property of clock object must be an integer')
1918
1919                 if freq < 1:
1920                     raise ConfigError('invalid clock frequency: {}'.format(freq))
1921
1922                 clock.freq = freq
1923
1924         # error cycles
1925         if 'error-cycles' in node:
1926             error_cycles = node['error-cycles']
1927
1928             if error_cycles is None:
1929                 clock.set_default_error_cycles()
1930             else:
1931                 if not _is_int_prop(error_cycles):
1932                     raise ConfigError('"error-cycles" property of clock object must be an integer')
1933
1934                 if error_cycles < 0:
1935                     raise ConfigError('invalid clock error cycles: {}'.format(error_cycles))
1936
1937                 clock.error_cycles = error_cycles
1938
1939         # offset
1940         if 'offset' in node:
1941             offset = node['offset']
1942
1943             if offset is None:
1944                 clock.set_default_offset_seconds()
1945                 clock.set_default_offset_cycles()
1946             else:
1947                 if not _is_assoc_array_prop(offset):
1948                     raise ConfigError('"offset" property of clock object must be an associative array')
1949
1950                 unk_prop = _get_first_unknown_prop(offset, ['cycles', 'seconds'])
1951
1952                 if unk_prop:
1953                     raise ConfigError('unknown clock object\'s offset property: "{}"'.format(unk_prop))
1954
1955                 # cycles
1956                 if 'cycles' in offset:
1957                     offset_cycles = offset['cycles']
1958
1959                     if offset_cycles is None:
1960                         clock.set_default_offset_cycles()
1961                     else:
1962                         if not _is_int_prop(offset_cycles):
1963                             raise ConfigError('"cycles" property of clock object\'s offset property must be an integer')
1964
1965                         if offset_cycles < 0:
1966                             raise ConfigError('invalid clock offset cycles: {}'.format(offset_cycles))
1967
1968                         clock.offset_cycles = offset_cycles
1969
1970                 # seconds
1971                 if 'seconds' in offset:
1972                     offset_seconds = offset['seconds']
1973
1974                     if offset_seconds is None:
1975                         clock.set_default_offset_seconds()
1976                     else:
1977                         if not _is_int_prop(offset_seconds):
1978                             raise ConfigError('"seconds" property of clock object\'s offset property must be an integer')
1979
1980                         if offset_seconds < 0:
1981                             raise ConfigError('invalid clock offset seconds: {}'.format(offset_seconds))
1982
1983                         clock.offset_seconds = offset_seconds
1984
1985         # absolute
1986         if 'absolute' in node:
1987             absolute = node['absolute']
1988
1989             if absolute is None:
1990                 clock.set_default_absolute()
1991             else:
1992                 if not _is_bool_prop(absolute):
1993                     raise ConfigError('"absolute" property of clock object must be a boolean')
1994
1995                 clock.absolute = absolute
1996
1997         # return C type:
1998         #   v2.0:  "return-ctype"
1999         #   v2.1+: "$return-ctype"
2000         return_ctype_node = None
2001
2002         if self._version >= 200:
2003             if 'return-ctype' in node:
2004                 return_ctype_prop = 'return-ctype'
2005                 return_ctype_node = node[return_ctype_prop]
2006
2007         if self._version >= 201:
2008             if '$return-ctype' in node:
2009                 if return_ctype_node is not None:
2010                     raise ConfigError('cannot specify both "return-ctype" and "$return-ctype" properties of clock object: prefer "$return-ctype"')
2011
2012                 return_ctype_prop = '$return-ctype'
2013                 return_ctype_node = node[return_ctype_prop]
2014
2015         if return_ctype_node is not None:
2016             if return_ctype_node is None:
2017                 clock.set_default_return_ctype()
2018             else:
2019                 if not _is_str_prop(return_ctype_node):
2020                     raise ConfigError('"{}" property of clock object must be a string'.format(return_ctype_prop))
2021
2022                 clock.return_ctype = return_ctype_node
2023
2024         return clock
2025
2026     def _register_clocks(self, metadata_node):
2027         self._clocks = collections.OrderedDict()
2028
2029         if 'clocks' not in metadata_node:
2030             return
2031
2032         clocks_node = metadata_node['clocks']
2033
2034         if clocks_node is None:
2035             return
2036
2037         if not _is_assoc_array_prop(clocks_node):
2038             raise ConfigError('"clocks" property (metadata) must be an associative array')
2039
2040         for clock_name, clock_node in clocks_node.items():
2041             if not _is_valid_identifier(clock_name):
2042                 raise ConfigError('invalid clock name: "{}"'.format(clock_name))
2043
2044             if clock_name in self._clocks:
2045                 raise ConfigError('duplicate clock "{}"'.format(clock_name))
2046
2047             try:
2048                 clock = self._create_clock(clock_node)
2049             except Exception as e:
2050                 raise ConfigError('cannot create clock "{}"'.format(clock_name), e)
2051
2052             clock.name = clock_name
2053             self._clocks[clock_name] = clock
2054
2055     def _create_env(self, metadata_node):
2056         env = collections.OrderedDict()
2057
2058         if 'env' not in metadata_node:
2059             return env
2060
2061         env_node = metadata_node['env']
2062
2063         if env_node is None:
2064             return env
2065
2066         if not _is_assoc_array_prop(env_node):
2067             raise ConfigError('"env" property (metadata) must be an associative array')
2068
2069         for env_name, env_value in env_node.items():
2070             if env_name in env:
2071                 raise ConfigError('duplicate environment variable "{}"'.format(env_name))
2072
2073             if not _is_valid_identifier(env_name):
2074                 raise ConfigError('invalid environment variable name: "{}"'.format(env_name))
2075
2076             if not _is_int_prop(env_value) and not _is_str_prop(env_value):
2077                 raise ConfigError('invalid environment variable value ("{}"): expecting integer or string'.format(env_name))
2078
2079             env[env_name] = env_value
2080
2081         return env
2082
2083     def _register_log_levels(self, metadata_node):
2084         self._log_levels = dict()
2085
2086         # log levels:
2087         #   v2.0:  "log-levels"
2088         #   v2.1+: "$log-levels"
2089         log_levels_node = None
2090
2091         if self._version >= 200:
2092             if 'log-levels' in metadata_node:
2093                 log_levels_prop = 'log-levels'
2094                 log_levels_node = metadata_node[log_levels_prop]
2095
2096         if self._version >= 201:
2097             if '$log-levels' in metadata_node:
2098                 if log_levels_node is not None:
2099                     raise ConfigError('cannot specify both "log-levels" and "$log-levels" properties of metadata object: prefer "$log-levels"')
2100
2101                 log_levels_prop = '$log-levels'
2102                 log_levels_node = metadata_node[log_levels_prop]
2103
2104         if log_levels_node is None:
2105             return
2106
2107         if not _is_assoc_array_prop(log_levels_node):
2108             raise ConfigError('"{}" property (metadata) must be an associative array'.format(log_levels_prop))
2109
2110         for ll_name, ll_value in log_levels_node.items():
2111             if ll_name in self._log_levels:
2112                 raise ConfigError('duplicate log level entry "{}"'.format(ll_name))
2113
2114             if not _is_int_prop(ll_value):
2115                 raise ConfigError('invalid log level entry ("{}"): expecting an integer'.format(ll_name))
2116
2117             if ll_value < 0:
2118                 raise ConfigError('invalid log level entry ("{}"): log level value must be positive'.format(ll_name))
2119
2120             self._log_levels[ll_name] = ll_value
2121
2122     def _create_trace(self, metadata_node):
2123         # create trace object
2124         trace = metadata.Trace()
2125
2126         if 'trace' not in metadata_node:
2127             raise ConfigError('missing "trace" property (metadata)')
2128
2129         trace_node = metadata_node['trace']
2130
2131         if not _is_assoc_array_prop(trace_node):
2132             raise ConfigError('"trace" property (metadata) must be an associative array')
2133
2134         unk_prop = _get_first_unknown_prop(trace_node, [
2135             'byte-order',
2136             'uuid',
2137             'packet-header-type',
2138         ])
2139
2140         if unk_prop:
2141             raise ConfigError('unknown trace object property: "{}"'.format(unk_prop))
2142
2143         # set byte order (already parsed)
2144         trace.byte_order = self._bo
2145
2146         # UUID
2147         if 'uuid' in trace_node and trace_node['uuid'] is not None:
2148             uuidp = trace_node['uuid']
2149
2150             if not _is_str_prop(uuidp):
2151                 raise ConfigError('"uuid" property of trace object must be a string')
2152
2153             if uuidp == 'auto':
2154                 uuidp = uuid.uuid1()
2155             else:
2156                 try:
2157                     uuidp = uuid.UUID(uuidp)
2158                 except:
2159                     raise ConfigError('malformed UUID (trace object): "{}"'.format(uuidp))
2160
2161             trace.uuid = uuidp
2162
2163         # packet header type
2164         if 'packet-header-type' in trace_node and trace_node['packet-header-type'] is not None:
2165             try:
2166                 ph_type = self._create_type(trace_node['packet-header-type'])
2167             except Exception as e:
2168                 raise ConfigError('cannot create packet header type (trace)', e)
2169
2170             trace.packet_header_type = ph_type
2171
2172         return trace
2173
2174     def _lookup_log_level(self, ll):
2175         if _is_int_prop(ll):
2176             return ll
2177         elif _is_str_prop(ll) and ll in self._log_levels:
2178             return self._log_levels[ll]
2179
2180     def _create_event(self, event_node):
2181         event = metadata.Event()
2182
2183         if not _is_assoc_array_prop(event_node):
2184             raise ConfigError('event objects must be associative arrays')
2185
2186         unk_prop = _get_first_unknown_prop(event_node, [
2187             'log-level',
2188             'context-type',
2189             'payload-type',
2190         ])
2191
2192         if unk_prop:
2193             raise ConfigError('unknown event object property: "{}"'.format(unk_prop))
2194
2195         if 'log-level' in event_node and event_node['log-level'] is not None:
2196             ll_node = event_node['log-level']
2197
2198             if _is_str_prop(ll_node):
2199                 ll_value = self._lookup_log_level(event_node['log-level'])
2200
2201                 if ll_value is None:
2202                     raise ConfigError('cannot find log level "{}"'.format(ll_node))
2203
2204                 ll = metadata.LogLevel(event_node['log-level'], ll_value)
2205             elif _is_int_prop(ll_node):
2206                 if ll_node < 0:
2207                     raise ConfigError('invalid log level value {}: value must be positive'.format(ll_node))
2208
2209                 ll = metadata.LogLevel(None, ll_node)
2210             else:
2211                 raise ConfigError('"log-level" property must be either a string or an integer')
2212
2213             event.log_level = ll
2214
2215         if 'context-type' in event_node and event_node['context-type'] is not None:
2216             ctx_type_node = event_node['context-type']
2217
2218             try:
2219                 t = self._create_type(event_node['context-type'])
2220             except Exception as e:
2221                 raise ConfigError('cannot create event\'s context type object', e)
2222
2223             event.context_type = t
2224
2225         if 'payload-type' in event_node and event_node['payload-type'] is not None:
2226             try:
2227                 t = self._create_type(event_node['payload-type'])
2228             except Exception as e:
2229                 raise ConfigError('cannot create event\'s payload type object', e)
2230
2231             event.payload_type = t
2232
2233         return event
2234
2235     def _create_stream(self, stream_name, stream_node):
2236         stream = metadata.Stream()
2237
2238         if not _is_assoc_array_prop(stream_node):
2239             raise ConfigError('stream objects must be associative arrays')
2240
2241         known_props = [
2242             'packet-context-type',
2243             'event-header-type',
2244             'event-context-type',
2245             'events',
2246         ]
2247
2248         if self._version >= 202:
2249             known_props.append('$default')
2250
2251         unk_prop = _get_first_unknown_prop(stream_node, known_props)
2252
2253         if unk_prop:
2254             add = ''
2255
2256             if unk_prop == '$default':
2257                 add = ' (use version 2.2 or greater)'
2258
2259             raise ConfigError('unknown stream object property{}: "{}"'.format(add, unk_prop))
2260
2261         if 'packet-context-type' in stream_node and stream_node['packet-context-type'] is not None:
2262             try:
2263                 t = self._create_type(stream_node['packet-context-type'])
2264             except Exception as e:
2265                 raise ConfigError('cannot create stream\'s packet context type object', e)
2266
2267             stream.packet_context_type = t
2268
2269         if 'event-header-type' in stream_node and stream_node['event-header-type'] is not None:
2270             try:
2271                 t = self._create_type(stream_node['event-header-type'])
2272             except Exception as e:
2273                 raise ConfigError('cannot create stream\'s event header type object', e)
2274
2275             stream.event_header_type = t
2276
2277         if 'event-context-type' in stream_node and stream_node['event-context-type'] is not None:
2278             try:
2279                 t = self._create_type(stream_node['event-context-type'])
2280             except Exception as e:
2281                 raise ConfigError('cannot create stream\'s event context type object', e)
2282
2283             stream.event_context_type = t
2284
2285         if 'events' not in stream_node:
2286             raise ConfigError('missing "events" property in stream object')
2287
2288         events = stream_node['events']
2289
2290         if events is not None:
2291             if not _is_assoc_array_prop(events):
2292                 raise ConfigError('"events" property of stream object must be an associative array')
2293
2294             if not events:
2295                 raise ConfigError('at least one event is needed within a stream object')
2296
2297             cur_id = 0
2298
2299             for ev_name, ev_node in events.items():
2300                 try:
2301                     ev = self._create_event(ev_node)
2302                 except Exception as e:
2303                     raise ConfigError('cannot create event "{}"'.format(ev_name), e)
2304
2305                 ev.id = cur_id
2306                 ev.name = ev_name
2307                 stream.events[ev_name] = ev
2308                 cur_id += 1
2309
2310         if '$default' in stream_node and stream_node['$default'] is not None:
2311             default_node = stream_node['$default']
2312
2313             if not _is_bool_prop(default_node):
2314                 raise ConfigError('invalid "$default" property in stream object: expecting a boolean')
2315
2316             if default_node:
2317                 if self._meta.default_stream_name is not None and self._meta.default_stream_name != stream_name:
2318                     fmt = 'cannot specify more than one default stream (default stream already set to "{}")'
2319                     raise ConfigError(fmt.format(self._meta.default_stream_name))
2320
2321                 self._meta.default_stream_name = stream_name
2322
2323         return stream
2324
2325     def _create_streams(self, metadata_node):
2326         streams = collections.OrderedDict()
2327
2328         if 'streams' not in metadata_node:
2329             raise ConfigError('missing "streams" property (metadata)')
2330
2331         streams_node = metadata_node['streams']
2332
2333         if not _is_assoc_array_prop(streams_node):
2334             raise ConfigError('"streams" property (metadata) must be an associative array')
2335
2336         if not streams_node:
2337             raise ConfigError('at least one stream is needed (metadata)')
2338
2339         cur_id = 0
2340
2341         for stream_name, stream_node in streams_node.items():
2342             try:
2343                 stream = self._create_stream(stream_name, stream_node)
2344             except Exception as e:
2345                 raise ConfigError('cannot create stream "{}"'.format(stream_name), e)
2346
2347             stream.id = cur_id
2348             stream.name = str(stream_name)
2349             streams[stream_name] = stream
2350             cur_id += 1
2351
2352         return streams
2353
2354     def _create_metadata(self, root):
2355         self._meta = metadata.Metadata()
2356
2357         if 'metadata' not in root:
2358             raise ConfigError('missing "metadata" property (configuration)')
2359
2360         metadata_node = root['metadata']
2361
2362         if not _is_assoc_array_prop(metadata_node):
2363             raise ConfigError('"metadata" property (configuration) must be an associative array')
2364
2365         known_props = [
2366             'type-aliases',
2367             'log-levels',
2368             'trace',
2369             'env',
2370             'clocks',
2371             'streams',
2372         ]
2373
2374         if self._version >= 201:
2375             known_props.append('$log-levels')
2376
2377         if self._version >= 202:
2378             known_props.append('$default-stream')
2379
2380         unk_prop = _get_first_unknown_prop(metadata_node, known_props)
2381
2382         if unk_prop:
2383             add = ''
2384
2385             if unk_prop == '$include':
2386                 add = ' (use version 2.1 or greater)'
2387
2388             if unk_prop == '$default-stream':
2389                 add = ' (use version 2.2 or greater)'
2390
2391             raise ConfigError('unknown metadata property{}: "{}"'.format(add, unk_prop))
2392
2393         if '$default-stream' in metadata_node and metadata_node['$default-stream'] is not None:
2394             default_stream_node = metadata_node['$default-stream']
2395
2396             if not _is_str_prop(default_stream_node):
2397                 raise ConfigError('invalid "$default-stream" property (metadata): expecting a string')
2398
2399             self._meta.default_stream_name = default_stream_node
2400
2401         self._set_byte_order(metadata_node)
2402         self._register_clocks(metadata_node)
2403         self._meta.clocks = self._clocks
2404         self._register_type_aliases(metadata_node)
2405         self._meta.env = self._create_env(metadata_node)
2406         self._meta.trace = self._create_trace(metadata_node)
2407         self._register_log_levels(metadata_node)
2408         self._meta.streams = self._create_streams(metadata_node)
2409
2410         return self._meta
2411
2412     def _get_version(self, root):
2413         if 'version' not in root:
2414             raise ConfigError('missing "version" property (configuration)')
2415
2416         version_node = root['version']
2417
2418         if not _is_str_prop(version_node):
2419             raise ConfigError('"version" property (configuration) must be a string')
2420
2421         version_node = version_node.strip()
2422
2423         if version_node not in ['2.0', '2.1', '2.2']:
2424             raise ConfigError('unsupported version ({}): versions 2.0, 2.1, and 2.2 are supported'.format(version_node))
2425
2426         # convert version string to comparable version integer
2427         parts = version_node.split('.')
2428         version = int(parts[0]) * 100 + int(parts[1])
2429
2430         return version
2431
2432     def _get_prefix(self, root):
2433         def_prefix = 'barectf_'
2434
2435         if 'prefix' not in root:
2436             return def_prefix
2437
2438         prefix_node = root['prefix']
2439
2440         if prefix_node is None:
2441             return def_prefix
2442
2443         if not _is_str_prop(prefix_node):
2444             raise ConfigError('"prefix" property (configuration) must be a string')
2445
2446         if not _is_valid_identifier(prefix_node):
2447             raise ConfigError('"prefix" property (configuration) must be a valid C identifier')
2448
2449         return prefix_node
2450
2451     def _get_options(self, root):
2452         cfg_options = ConfigOptions()
2453
2454         if 'options' not in root:
2455             return cfg_options
2456
2457         options_node = root['options']
2458
2459         if not _is_assoc_array_prop(options_node):
2460             raise ConfigError('"options" property (configuration) must be an associative array')
2461
2462         known_props = [
2463             'gen-prefix-def',
2464             'gen-default-stream-def',
2465         ]
2466         unk_prop = _get_first_unknown_prop(options_node, known_props)
2467
2468         if unk_prop:
2469             raise ConfigError('unknown configuration option property: "{}"'.format(unk_prop))
2470
2471         if 'gen-prefix-def' in options_node and options_node['gen-prefix-def'] is not None:
2472             gen_prefix_def_node = options_node['gen-prefix-def']
2473
2474             if not _is_bool_prop(gen_prefix_def_node):
2475                 raise ConfigError('invalid configuration option "gen-prefix-def": expecting a boolean')
2476
2477             cfg_options.gen_prefix_def = gen_prefix_def_node
2478
2479         if 'gen-default-stream-def' in options_node and options_node['gen-default-stream-def'] is not None:
2480             gen_default_stream_def_node = options_node['gen-default-stream-def']
2481
2482             if not _is_bool_prop(gen_default_stream_def_node):
2483                 raise ConfigError('invalid configuration option "gen-default-stream-def": expecting a boolean')
2484
2485             cfg_options.gen_default_stream_def = gen_default_stream_def_node
2486
2487         return cfg_options
2488
2489     def _get_last_include_file(self):
2490         if self._include_stack:
2491             return self._include_stack[-1]
2492
2493         return self._root_yaml_path
2494
2495     def _load_include(self, yaml_path):
2496         for inc_dir in self._include_dirs:
2497             # current include dir + file name path
2498             # note: os.path.join() only takes the last arg if it's absolute
2499             inc_path = os.path.join(inc_dir, yaml_path)
2500
2501             # real path (symbolic links resolved)
2502             real_path = os.path.realpath(inc_path)
2503
2504             # normalized path (weird stuff removed!)
2505             norm_path = os.path.normpath(real_path)
2506
2507             if not os.path.isfile(norm_path):
2508                 # file does not exist: skip
2509                 continue
2510
2511             if norm_path in self._include_stack:
2512                 base_path = self._get_last_include_file()
2513                 raise ConfigError('in "{}": cannot recursively include file "{}"'.format(base_path, norm_path))
2514
2515             self._include_stack.append(norm_path)
2516
2517             # load raw content
2518             return self._yaml_ordered_load(norm_path)
2519
2520         if not self._ignore_include_not_found:
2521             base_path = self._get_last_include_file()
2522             raise ConfigError('in "{}": cannot include file "{}": file not found in include directories'.format(base_path, yaml_path))
2523
2524         return None
2525
2526     def _get_include_paths(self, include_node):
2527         if include_node is None:
2528             return []
2529
2530         if _is_str_prop(include_node):
2531             return [include_node]
2532
2533         if _is_array_prop(include_node):
2534             for include_path in include_node:
2535                 if not _is_str_prop(include_path):
2536                     raise ConfigError('invalid include property: expecting array of strings')
2537
2538             return include_node
2539
2540         raise ConfigError('invalid include property: expecting string or array of strings')
2541
2542     def _update_node(self, base_node, overlay_node):
2543         for olay_key, olay_value in overlay_node.items():
2544             if olay_key in base_node:
2545                 base_value = base_node[olay_key]
2546
2547                 if _is_assoc_array_prop(olay_value) and _is_assoc_array_prop(base_value):
2548                     # merge dictionaries
2549                     self._update_node(base_value, olay_value)
2550                 elif _is_array_prop(olay_value) and _is_array_prop(base_value):
2551                     # append extension array items to base items
2552                     base_value += olay_value
2553                 else:
2554                     # fall back to replacing
2555                     base_node[olay_key] = olay_value
2556             else:
2557                 base_node[olay_key] = olay_value
2558
2559     def _process_node_include(self, last_overlay_node, name,
2560                               process_base_include_cb,
2561                               process_children_include_cb=None):
2562         if not _is_assoc_array_prop(last_overlay_node):
2563             raise ConfigError('{} objects must be associative arrays'.format(name))
2564
2565         # process children inclusions first
2566         if process_children_include_cb:
2567             process_children_include_cb(last_overlay_node)
2568
2569         if '$include' in last_overlay_node:
2570             include_node = last_overlay_node['$include']
2571         else:
2572             # no includes!
2573             return last_overlay_node
2574
2575         include_paths = self._get_include_paths(include_node)
2576         cur_base_path = self._get_last_include_file()
2577         base_node = None
2578
2579         # keep the include paths and remove the include property
2580         include_paths = copy.deepcopy(include_paths)
2581         del last_overlay_node['$include']
2582
2583         for include_path in include_paths:
2584             # load raw YAML from included file
2585             overlay_node = self._load_include(include_path)
2586
2587             if overlay_node is None:
2588                 # cannot find include file, but we're ignoring those
2589                 # errors, otherwise _load_include() itself raises
2590                 # a config error
2591                 continue
2592
2593             # recursively process includes
2594             try:
2595                 overlay_node = process_base_include_cb(overlay_node)
2596             except Exception as e:
2597                 raise ConfigError('in "{}"'.format(cur_base_path), e)
2598
2599             # pop include stack now that we're done including
2600             del self._include_stack[-1]
2601
2602             # at this point, base_node is fully resolved (does not
2603             # contain any include property)
2604             if base_node is None:
2605                 base_node = overlay_node
2606             else:
2607                 self._update_node(base_node, overlay_node)
2608
2609         # finally, we update the latest base node with our last overlay
2610         # node
2611         if base_node is None:
2612             # nothing was included, which is possible when we're
2613             # ignoring include errors
2614             return last_overlay_node
2615
2616         self._update_node(base_node, last_overlay_node)
2617
2618         return base_node
2619
2620     def _process_event_include(self, event_node):
2621         return self._process_node_include(event_node, 'event',
2622                                           self._process_event_include)
2623
2624     def _process_stream_include(self, stream_node):
2625         def process_children_include(stream_node):
2626             if 'events' in stream_node:
2627                 events_node = stream_node['events']
2628
2629                 if not _is_assoc_array_prop(events_node):
2630                     raise ConfigError('"events" property must be an associative array')
2631
2632                 events_node_keys = list(events_node.keys())
2633
2634                 for key in events_node_keys:
2635                     event_node = events_node[key]
2636
2637                     try:
2638                         events_node[key] = self._process_event_include(event_node)
2639                     except Exception as e:
2640                         raise ConfigError('cannot process includes of event object "{}"'.format(key), e)
2641
2642         return self._process_node_include(stream_node, 'stream',
2643                                           self._process_stream_include,
2644                                           process_children_include)
2645
2646     def _process_trace_include(self, trace_node):
2647         return self._process_node_include(trace_node, 'trace',
2648                                           self._process_trace_include)
2649
2650     def _process_clock_include(self, clock_node):
2651         return self._process_node_include(clock_node, 'clock',
2652                                           self._process_clock_include)
2653
2654     def _process_metadata_include(self, metadata_node):
2655         def process_children_include(metadata_node):
2656             if 'trace' in metadata_node:
2657                 metadata_node['trace'] = self._process_trace_include(metadata_node['trace'])
2658
2659             if 'clocks' in metadata_node:
2660                 clocks_node = metadata_node['clocks']
2661
2662                 if not _is_assoc_array_prop(clocks_node):
2663                     raise ConfigError('"clocks" property (metadata) must be an associative array')
2664
2665                 clocks_node_keys = list(clocks_node.keys())
2666
2667                 for key in clocks_node_keys:
2668                     clock_node = clocks_node[key]
2669
2670                     try:
2671                         clocks_node[key] = self._process_clock_include(clock_node)
2672                     except Exception as e:
2673                         raise ConfigError('cannot process includes of clock object "{}"'.format(key), e)
2674
2675             if 'streams' in metadata_node:
2676                 streams_node = metadata_node['streams']
2677
2678                 if not _is_assoc_array_prop(streams_node):
2679                     raise ConfigError('"streams" property (metadata) must be an associative array')
2680
2681                 streams_node_keys = list(streams_node.keys())
2682
2683                 for key in streams_node_keys:
2684                     stream_node = streams_node[key]
2685
2686                     try:
2687                         streams_node[key] = self._process_stream_include(stream_node)
2688                     except Exception as e:
2689                         raise ConfigError('cannot process includes of stream object "{}"'.format(key), e)
2690
2691         return self._process_node_include(metadata_node, 'metadata',
2692                                           self._process_metadata_include,
2693                                           process_children_include)
2694
2695     def _process_root_includes(self, root):
2696         # The following config objects support includes:
2697         #
2698         #   * Metadata object
2699         #   * Trace object
2700         #   * Stream object
2701         #   * Event object
2702         #
2703         # We need to process the event includes first, then the stream
2704         # includes, then the trace includes, and finally the metadata
2705         # includes.
2706         #
2707         # In each object, only one of the $include and $include-replace
2708         # special properties is allowed.
2709         #
2710         # We keep a stack of absolute paths to included files to detect
2711         # recursion.
2712         if 'metadata' in root:
2713             root['metadata'] = self._process_metadata_include(root['metadata'])
2714
2715         return root
2716
2717     def _yaml_ordered_dump(self, node, **kwds):
2718         class ODumper(yaml.Dumper):
2719             pass
2720
2721         def dict_representer(dumper, node):
2722             return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2723                                             node.items())
2724
2725         ODumper.add_representer(collections.OrderedDict, dict_representer)
2726
2727         return yaml.dump(node, Dumper=ODumper, **kwds)
2728
2729     def _yaml_ordered_load(self, yaml_path):
2730         class OLoader(yaml.Loader):
2731             pass
2732
2733         def construct_mapping(loader, node):
2734             loader.flatten_mapping(node)
2735
2736             return collections.OrderedDict(loader.construct_pairs(node))
2737
2738         OLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2739                                 construct_mapping)
2740
2741         # YAML -> Python
2742         try:
2743             with open(yaml_path, 'r') as f:
2744                 node = yaml.load(f, OLoader)
2745         except (OSError, IOError) as e:
2746             raise ConfigError('cannot open file "{}"'.format(yaml_path))
2747         except Exception as e:
2748             raise ConfigError('unknown error while trying to load file "{}"'.format(yaml_path), e)
2749
2750         # loaded node must be an associate array
2751         if not _is_assoc_array_prop(node):
2752             raise ConfigError('root of YAML file "{}" must be an associative array'.format(yaml_path))
2753
2754         return node
2755
2756     def _reset(self):
2757         self._version = None
2758         self._include_stack = []
2759
2760     def parse(self, yaml_path):
2761         self._reset()
2762         self._root_yaml_path = yaml_path
2763
2764         try:
2765             root = self._yaml_ordered_load(yaml_path)
2766         except Exception as e:
2767             raise ConfigError('cannot parse YAML file "{}"'.format(yaml_path), e)
2768
2769         if not _is_assoc_array_prop(root):
2770             raise ConfigError('configuration must be an associative array')
2771
2772         # get the config version
2773         self._version = self._get_version(root)
2774
2775         known_props = [
2776             'version',
2777             'prefix',
2778             'metadata',
2779         ]
2780
2781         if self._version >= 202:
2782             known_props.append('options')
2783
2784         unk_prop = _get_first_unknown_prop(root, known_props)
2785
2786         if unk_prop:
2787             add = ''
2788
2789             if unk_prop == 'options':
2790                 add = ' (use version 2.2 or greater)'
2791
2792             raise ConfigError('unknown configuration property{}: "{}"'.format(add, unk_prop))
2793
2794         # process includes if supported
2795         if self._version >= 201:
2796             root = self._process_root_includes(root)
2797
2798         # dump config if required
2799         if self._dump_config:
2800             print(self._yaml_ordered_dump(root, indent=2,
2801                                           default_flow_style=False))
2802
2803         # get prefix and metadata
2804         prefix = self._get_prefix(root)
2805         meta = self._create_metadata(root)
2806         opts = self._get_options(root)
2807
2808         return Config(self._version, prefix, meta, opts)
2809
2810
2811 def from_yaml_file(path, include_dirs, ignore_include_not_found, dump_config):
2812     try:
2813         parser = _YamlConfigParser(include_dirs, ignore_include_not_found,
2814                                    dump_config)
2815         cfg = parser.parse(path)
2816
2817         return cfg
2818     except Exception as e:
2819         raise ConfigError('cannot create configuration from YAML file "{}"'.format(path), e)