barectf/config.py

   1 # The MIT License (MIT)
   2 #
   3 # Copyright (c) 2015 Philippe Proulx <pproulx@efficios.com>
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  21 # THE SOFTWARE.
  22
  23 from barectf import metadata
  24 import collections
  25 import datetime
  26 import barectf
  27 import enum
  28 import yaml
  29 import uuid
  30 import copy
  31 import re
  32 import os
  33
  34
  35 class ConfigError(RuntimeError):
  36     def __init__(self, msg, prev=None):
  37         super().__init__(msg)
  38         self._prev = prev
  39
  40     @property
  41     def prev(self):
  42         return self._prev
  43
  44
  45 class Config:
  46     def __init__(self, version, prefix, metadata):
  47         self.prefix = prefix
  48         self.version = version
  49         self.metadata = metadata
  50
  51     def _validate_metadata(self, meta):
  52         try:
  53             validator = _MetadataTypesHistologyValidator()
  54             validator.validate(meta)
  55             validator = _MetadataDynamicTypesValidator()
  56             validator.validate(meta)
  57             validator = _MetadataSpecialFieldsValidator()
  58             validator.validate(meta)
  59         except Exception as e:
  60             raise ConfigError('metadata error', e)
  61
  62         try:
  63             validator = _BarectfMetadataValidator()
  64             validator.validate(meta)
  65         except Exception as e:
  66             raise ConfigError('barectf metadata error', e)
  67
  68     def _augment_metadata_env(self, meta):
  69         env = meta.env
  70
  71         env['domain'] = 'bare'
  72         env['tracer_name'] = 'barectf'
  73         version_tuple = barectf.get_version_tuple()
  74         env['tracer_major'] = version_tuple[0]
  75         env['tracer_minor'] = version_tuple[1]
  76         env['tracer_patch'] = version_tuple[2]
  77         env['barectf_gen_date'] = str(datetime.datetime.now().isoformat())
  78
  79     @property
  80     def version(self):
  81         return self._version
  82
  83     @version.setter
  84     def version(self, value):
  85         self._version = value
  86
  87     @property
  88     def metadata(self):
  89         return self._metadata
  90
  91     @metadata.setter
  92     def metadata(self, value):
  93         self._validate_metadata(value)
  94         self._augment_metadata_env(value)
  95         self._metadata = value
  96
  97     @property
  98     def prefix(self):
  99         return self._prefix
 100
 101     @prefix.setter
 102     def prefix(self, value):
 103         if not is_valid_identifier(value):
 104             raise ConfigError('prefix must be a valid C identifier')
 105
 106         self._prefix = value
 107
 108
 109 def _is_assoc_array_prop(node):
 110     return isinstance(node, dict)
 111
 112
 113 def _is_array_prop(node):
 114     return isinstance(node, list)
 115
 116
 117 def _is_int_prop(node):
 118     return type(node) is int
 119
 120
 121 def _is_str_prop(node):
 122     return type(node) is str
 123
 124
 125 def _is_bool_prop(node):
 126     return type(node) is bool
 127
 128
 129 def _is_valid_alignment(align):
 130     return ((align & (align - 1)) == 0) and align > 0
 131
 132
 133 def _byte_order_str_to_bo(bo_str):
 134     bo_str = bo_str.lower()
 135
 136     if bo_str == 'le':
 137         return metadata.ByteOrder.LE
 138     elif bo_str == 'be':
 139         return metadata.ByteOrder.BE
 140
 141
 142 def _encoding_str_to_encoding(encoding_str):
 143     encoding_str = encoding_str.lower()
 144
 145     if encoding_str == 'utf-8' or encoding_str == 'utf8':
 146         return metadata.Encoding.UTF8
 147     elif encoding_str == 'ascii':
 148         return metadata.Encoding.ASCII
 149     elif encoding_str == 'none':
 150         return metadata.Encoding.NONE
 151
 152
 153 _re_iden = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]*$')
 154 _ctf_keywords = set([
 155     'align',
 156     'callsite',
 157     'clock',
 158     'enum',
 159     'env',
 160     'event',
 161     'floating_point',
 162     'integer',
 163     'stream',
 164     'string',
 165     'struct',
 166     'trace',
 167     'typealias',
 168     'typedef',
 169     'variant',
 170 ])
 171
 172
 173 def is_valid_identifier(iden):
 174     if not _re_iden.match(iden):
 175         return False
 176
 177     if _re_iden in _ctf_keywords:
 178         return False
 179
 180     return True
 181
 182
 183 def _get_first_unknown_prop(node, known_props):
 184     for prop_name in node:
 185         if prop_name in known_props:
 186             continue
 187
 188         return prop_name
 189
 190
 191 # This validator validates the configured metadata for barectf specific
 192 # needs.
 193 #
 194 # barectf needs:
 195 #
 196 #   * all header/contexts are at least byte-aligned
 197 #   * all integer and floating point number sizes to be <= 64
 198 #   * no inner structures, arrays, or variants
 199 class _BarectfMetadataValidator:
 200     def __init__(self):
 201         self._type_to_validate_type_func = {
 202             metadata.Integer: self._validate_int_type,
 203             metadata.FloatingPoint: self._validate_float_type,
 204             metadata.Enum: self._validate_enum_type,
 205             metadata.String: self._validate_string_type,
 206             metadata.Struct: self._validate_struct_type,
 207             metadata.Array: self._validate_array_type,
 208             metadata.Variant: self._validate_variant_type,
 209         }
 210
 211     def _validate_int_type(self, t, entity_root):
 212         if t.size > 64:
 213             raise ConfigError('integer type\'s size must be lesser than or equal to 64 bits')
 214
 215     def _validate_float_type(self, t, entity_root):
 216         if t.size > 64:
 217             raise ConfigError('floating point number type\'s size must be lesser than or equal to 64 bits')
 218
 219     def _validate_enum_type(self, t, entity_root):
 220         if t.value_type.size > 64:
 221             raise ConfigError('enumeration type\'s integer type\'s size must be lesser than or equal to 64 bits')
 222
 223     def _validate_string_type(self, t, entity_root):
 224         pass
 225
 226     def _validate_struct_type(self, t, entity_root):
 227         if not entity_root:
 228             raise ConfigError('inner structure types are not supported as of this version')
 229
 230         for field_name, field_type in t.fields.items():
 231             if entity_root and self._cur_entity is _Entity.TRACE_PACKET_HEADER:
 232                 if field_name == 'uuid':
 233                     # allow
 234                     continue
 235
 236             try:
 237                 self._validate_type(field_type, False)
 238             except Exception as e:
 239                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 240
 241     def _validate_array_type(self, t, entity_root):
 242         raise ConfigError('array types are not supported as of this version')
 243
 244     def _validate_variant_type(self, t, entity_root):
 245         raise ConfigError('variant types are not supported as of this version')
 246
 247     def _validate_type(self, t, entity_root):
 248         self._type_to_validate_type_func[type(t)](t, entity_root)
 249
 250     def _validate_entity(self, t):
 251         if t is None:
 252             return
 253
 254         # make sure entity is byte-aligned
 255         if t.align < 8:
 256             raise ConfigError('type\'s alignment must be at least byte-aligned')
 257
 258         # make sure entity is a structure
 259         if type(t) is not metadata.Struct:
 260             raise ConfigError('expecting a structure type')
 261
 262         # validate types
 263         self._validate_type(t, True)
 264
 265     def _validate_entities_and_names(self, meta):
 266         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 267
 268         try:
 269             self._validate_entity(meta.trace.packet_header_type)
 270         except Exception as e:
 271             raise ConfigError('invalid trace packet header type', e)
 272
 273         for stream_name, stream in meta.streams.items():
 274             if not is_valid_identifier(stream_name):
 275                 raise ConfigError('stream name "{}" is not a valid C identifier'.format(stream_name))
 276
 277             self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 278
 279             try:
 280                 self._validate_entity(stream.packet_context_type)
 281             except Exception as e:
 282                 raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 283
 284             self._cur_entity = _Entity.STREAM_EVENT_HEADER
 285
 286             try:
 287                 self._validate_entity(stream.event_header_type)
 288             except Exception as e:
 289                 raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 290
 291             self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 292
 293             try:
 294                 self._validate_entity(stream.event_context_type)
 295             except Exception as e:
 296                 raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 297
 298             try:
 299                 for ev_name, ev in stream.events.items():
 300                     if not is_valid_identifier(ev_name):
 301                         raise ConfigError('event name "{}" is not a valid C identifier'.format(ev_name))
 302
 303                     self._cur_entity = _Entity.EVENT_CONTEXT
 304
 305                     try:
 306                         self._validate_entity(ev.context_type)
 307                     except Exception as e:
 308                         raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 309
 310                     self._cur_entity = _Entity.EVENT_PAYLOAD
 311
 312                     if ev.payload_type is None:
 313                         raise ConfigError('missing payload type in event "{}"'.format(ev_name), e)
 314
 315                     try:
 316                         self._validate_entity(ev.payload_type)
 317                     except Exception as e:
 318                         raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 319
 320                     if not ev.payload_type.fields:
 321                         raise ConfigError('empty payload type in event "{}"'.format(ev_name), e)
 322             except Exception as e:
 323                 raise ConfigError('invalid stream "{}"'.format(stream_name), e)
 324
 325     def validate(self, meta):
 326         self._validate_entities_and_names(meta)
 327
 328
 329 # This validator validates special fields of trace, stream, and event
 330 # types. For example, if checks that the "stream_id" field exists in the
 331 # trace packet header if there's more than one stream, and much more.
 332 class _MetadataSpecialFieldsValidator:
 333     def _validate_trace_packet_header_type(self, t):
 334         # needs "stream_id" field?
 335         if len(self._meta.streams) > 1:
 336             # yes
 337             if t is None:
 338                 raise ConfigError('need "stream_id" field in trace packet header type, but trace packet header type is missing')
 339
 340             if type(t) is not metadata.Struct:
 341                 raise ConfigError('need "stream_id" field in trace packet header type, but trace packet header type is not a structure type')
 342
 343             if 'stream_id' not in t.fields:
 344                 raise ConfigError('need "stream_id" field in trace packet header type')
 345
 346         # validate "magic" and "stream_id" types
 347         if type(t) is not metadata.Struct:
 348             return
 349
 350         for i, (field_name, field_type) in enumerate(t.fields.items()):
 351             if field_name == 'magic':
 352                 if type(field_type) is not metadata.Integer:
 353                     raise ConfigError('"magic" field in trace packet header type must be an integer type')
 354
 355                 if field_type.signed or field_type.size != 32:
 356                     raise ConfigError('"magic" field in trace packet header type must be a 32-bit unsigned integer type')
 357
 358                 if i != 0:
 359                     raise ConfigError('"magic" field must be the first trace packet header type\'s field')
 360             elif field_name == 'stream_id':
 361                 if type(field_type) is not metadata.Integer:
 362                     raise ConfigError('"stream_id" field in trace packet header type must be an integer type')
 363
 364                 if field_type.signed:
 365                     raise ConfigError('"stream_id" field in trace packet header type must be an unsigned integer type')
 366             elif field_name == 'uuid':
 367                 if self._meta.trace.uuid is None:
 368                     raise ConfigError('"uuid" field in trace packet header type specified, but no trace UUID provided')
 369
 370                 if type(field_type) is not metadata.Array:
 371                     raise ConfigError('"uuid" field in trace packet header type must be an array')
 372
 373                 if field_type.length != 16:
 374                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 bytes')
 375
 376                 element_type = field_type.element_type
 377
 378                 if type(element_type) is not metadata.Integer:
 379                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 380
 381                 if element_type.size != 8:
 382                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 383
 384                 if element_type.signed:
 385                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 386
 387                 if element_type.align != 8:
 388                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned, byte-aligned bytes')
 389
 390     def _validate_trace(self, meta):
 391         self._validate_trace_packet_header_type(meta.trace.packet_header_type)
 392
 393     def _validate_stream_packet_context(self, stream):
 394         t = stream.packet_context_type
 395
 396         if type(t) is None:
 397             return
 398
 399         if type(t) is not metadata.Struct:
 400             return
 401
 402         # "timestamp_begin", if exists, is an unsigned integer type,
 403         # mapped to a clock
 404         if 'timestamp_begin' in t.fields:
 405             ts_begin = t.fields['timestamp_begin']
 406
 407             if type(ts_begin) is not metadata.Integer:
 408                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an integer type')
 409
 410             if ts_begin.signed:
 411                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an unsigned integer type')
 412
 413             if not ts_begin.property_mappings:
 414                 raise ConfigError('"timestamp_begin" field in stream packet context type must be mapped to a clock')
 415
 416         # "timestamp_end", if exists, is an unsigned integer type,
 417         # mapped to a clock
 418         if 'timestamp_end' in t.fields:
 419             ts_end = t.fields['timestamp_end']
 420
 421             if type(ts_end) is not metadata.Integer:
 422                 raise ConfigError('"timestamp_end" field in stream packet context type must be an integer type')
 423
 424             if ts_end.signed:
 425                 raise ConfigError('"timestamp_end" field in stream packet context type must be an unsigned integer type')
 426
 427             if not ts_end.property_mappings:
 428                 raise ConfigError('"timestamp_end" field in stream packet context type must be mapped to a clock')
 429
 430         # "timestamp_begin" and "timestamp_end" exist together
 431         if (('timestamp_begin' in t.fields) ^ ('timestamp_end' in t.fields)):
 432             raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be defined together in stream packet context type')
 433
 434         # "events_discarded", if exists, is an unsigned integer type
 435         if 'events_discarded' in t.fields:
 436             events_discarded = t.fields['events_discarded']
 437
 438             if type(events_discarded) is not metadata.Integer:
 439                 raise ConfigError('"events_discarded" field in stream packet context type must be an integer type')
 440
 441             if events_discarded.signed:
 442                 raise ConfigError('"events_discarded" field in stream packet context type must be an unsigned integer type')
 443
 444         # "packet_size" and "content_size" must exist
 445         if 'packet_size' not in t.fields:
 446             raise ConfigError('missing "packet_size" field in stream packet context type')
 447
 448         packet_size = t.fields['packet_size']
 449
 450         # "content_size" and "content_size" must exist
 451         if 'content_size' not in t.fields:
 452             raise ConfigError('missing "content_size" field in stream packet context type')
 453
 454         content_size = t.fields['content_size']
 455
 456         # "packet_size" is an unsigned integer type
 457         if type(packet_size) is not metadata.Integer:
 458             raise ConfigError('"packet_size" field in stream packet context type must be an integer type')
 459
 460         if packet_size.signed:
 461             raise ConfigError('"packet_size" field in stream packet context type must be an unsigned integer type')
 462
 463         # "content_size" is an unsigned integer type
 464         if type(content_size) is not metadata.Integer:
 465             raise ConfigError('"content_size" field in stream packet context type must be an integer type')
 466
 467         if content_size.signed:
 468             raise ConfigError('"content_size" field in stream packet context type must be an unsigned integer type')
 469
 470     def _validate_stream_event_header(self, stream):
 471         t = stream.event_header_type
 472
 473         # needs "id" field?
 474         if len(stream.events) > 1:
 475             # yes
 476             if t is None:
 477                 raise ConfigError('need "id" field in stream event header type, but stream event header type is missing')
 478
 479             if type(t) is not metadata.Struct:
 480                 raise ConfigError('need "id" field in stream event header type, but stream event header type is not a structure type')
 481
 482             if 'id' not in t.fields:
 483                 raise ConfigError('need "id" field in stream event header type')
 484
 485         # validate "id" and "timestamp" types
 486         if type(t) is not metadata.Struct:
 487             return
 488
 489         # "timestamp", if exists, is an unsigned integer type,
 490         # mapped to a clock
 491         if 'timestamp' in t.fields:
 492             ts = t.fields['timestamp']
 493
 494             if type(ts) is not metadata.Integer:
 495                 raise ConfigError('"ts" field in stream event header type must be an integer type')
 496
 497             if ts.signed:
 498                 raise ConfigError('"ts" field in stream event header type must be an unsigned integer type')
 499
 500             if not ts.property_mappings:
 501                 raise ConfigError('"ts" field in stream event header type must be mapped to a clock')
 502
 503         # "id" is an unsigned integer type
 504         if 'id' in t.fields:
 505             eid = t.fields['id']
 506
 507             if type(eid) is not metadata.Integer:
 508                 raise ConfigError('"id" field in stream event header type must be an integer type')
 509
 510             if eid.signed:
 511                 raise ConfigError('"id" field in stream event header type must be an unsigned integer type')
 512
 513     def _validate_stream(self, stream):
 514         self._validate_stream_packet_context(stream)
 515         self._validate_stream_event_header(stream)
 516
 517     def validate(self, meta):
 518         self._meta = meta
 519         self._validate_trace(meta)
 520
 521         for stream in meta.streams.values():
 522             try:
 523                 self._validate_stream(stream)
 524             except Exception as e:
 525                 raise ConfigError('invalid stream "{}"'.format(stream.name), e)
 526
 527
 528 class _MetadataDynamicTypesValidatorStackEntry:
 529     def __init__(self, base_t):
 530         self._base_t = base_t
 531         self._index = 0
 532
 533     @property
 534     def index(self):
 535         return self._index
 536
 537     @index.setter
 538     def index(self, value):
 539         self._index = value
 540
 541     @property
 542     def base_t(self):
 543         return self._base_t
 544
 545     @base_t.setter
 546     def base_t(self, value):
 547         self._base_t = value
 548
 549
 550 # Entities. Order of values is important here.
 551 @enum.unique
 552 class _Entity(enum.IntEnum):
 553     TRACE_PACKET_HEADER = 0
 554     STREAM_PACKET_CONTEXT = 1
 555     STREAM_EVENT_HEADER = 2
 556     STREAM_EVENT_CONTEXT = 3
 557     EVENT_CONTEXT = 4
 558     EVENT_PAYLOAD = 5
 559
 560
 561 # This validator validates dynamic metadata types, that is, it ensures
 562 # variable-length array lengths and variant tags actually point to
 563 # something that exists. It also checks that variable-length array
 564 # lengths point to integer types and variant tags to enumeration types.
 565 class _MetadataDynamicTypesValidator:
 566     def __init__(self):
 567         self._type_to_visit_type_func = {
 568             metadata.Integer: None,
 569             metadata.FloatingPoint: None,
 570             metadata.Enum: None,
 571             metadata.String: None,
 572             metadata.Struct: self._visit_struct_type,
 573             metadata.Array: self._visit_array_type,
 574             metadata.Variant: self._visit_variant_type,
 575         }
 576
 577         self._cur_trace = None
 578         self._cur_stream = None
 579         self._cur_event = None
 580
 581     def _lookup_path_from_base(self, path, parts, base, start_index,
 582                                base_is_current, from_t):
 583         index = start_index
 584         cur_t = base
 585         found_path = []
 586
 587         while index < len(parts):
 588             part = parts[index]
 589             next_t = None
 590
 591             if type(cur_t) is metadata.Struct:
 592                 enumerated_items = enumerate(cur_t.fields.items())
 593
 594                 # lookup each field
 595                 for i, (field_name, field_type) in enumerated_items:
 596                     if field_name == part:
 597                         next_t = field_type
 598                         found_path.append((i, field_type))
 599
 600                 if next_t is None:
 601                     raise ConfigError('invalid path "{}": cannot find field "{}" in structure type'.format(path, part))
 602             elif type(cur_t) is metadata.Variant:
 603                 enumerated_items = enumerate(cur_t.types.items())
 604
 605                 # lookup each type
 606                 for i, (type_name, type_type) in enumerated_items:
 607                     if type_name == part:
 608                         next_t = type_type
 609                         found_path.append((i, type_type))
 610
 611                 if next_t is None:
 612                     raise ConfigError('invalid path "{}": cannot find type "{}" in variant type'.format(path, part))
 613             else:
 614                 raise ConfigError('invalid path "{}": requesting "{}" in a non-variant, non-structure type'.format(path, part))
 615
 616             cur_t = next_t
 617             index += 1
 618
 619         # make sure that the pointed type is not the pointing type
 620         if from_t is cur_t:
 621             raise ConfigError('invalid path "{}": pointing to self'.format(path))
 622
 623         # if we're here, we found the type; however, it could be located
 624         # _after_ the variant/VLA looking for it, if the pointing
 625         # and pointed types are in the same entity, so compare the
 626         # current stack entries indexes to our index path in that case
 627         if not base_is_current:
 628             return cur_t
 629
 630         for index, entry in enumerate(self._stack):
 631             if index == len(found_path):
 632                 # end of index path; valid so far
 633                 break
 634
 635             if found_path[index][0] > entry.index:
 636                 raise ConfigError('invalid path "{}": pointed type is after pointing type'.format(path))
 637
 638         # also make sure that both pointed and pointing types share
 639         # a common structure ancestor
 640         for index, entry in enumerate(self._stack):
 641             if index == len(found_path):
 642                 break
 643
 644             if entry.base_t is not found_path[index][1]:
 645                 # found common ancestor
 646                 if type(entry.base_t) is metadata.Variant:
 647                     raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 648
 649         return cur_t
 650
 651     def _lookup_path_from_top(self, path, parts):
 652         if len(parts) != 1:
 653             raise ConfigError('invalid path "{}": multipart relative path not supported'.format(path))
 654
 655         find_name = parts[0]
 656         index = len(self._stack) - 1
 657         got_struct = False
 658
 659         # check stack entries in reversed order
 660         for entry in reversed(self._stack):
 661             # structure base type
 662             if type(entry.base_t) is metadata.Struct:
 663                 got_struct = True
 664                 enumerated_items = enumerate(entry.base_t.fields.items())
 665
 666                 # lookup each field, until the current visiting index is met
 667                 for i, (field_name, field_type) in enumerated_items:
 668                     if i == entry.index:
 669                         break
 670
 671                     if field_name == find_name:
 672                         return field_type
 673
 674             # variant base type
 675             elif type(entry.base_t) is metadata.Variant:
 676                 enumerated_items = enumerate(entry.base_t.types.items())
 677
 678                 # lookup each type, until the current visiting index is met
 679                 for i, (type_name, type_type) in enumerated_items:
 680                     if i == entry.index:
 681                         break
 682
 683                     if type_name == find_name:
 684                         if not got_struct:
 685                             raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 686
 687                         return type_type
 688
 689         # nothing returned here: cannot find type
 690         raise ConfigError('invalid path "{}": cannot find type in current context'.format(path))
 691
 692     def _lookup_path(self, path, from_t):
 693         parts = path.lower().split('.')
 694         base = None
 695         base_is_current = False
 696
 697         if len(parts) >= 3:
 698             if parts[0] == 'trace':
 699                 if parts[1] == 'packet' and parts[2] == 'header':
 700                     # make sure packet header exists
 701                     if self._cur_trace.packet_header_type is None:
 702                         raise ConfigError('invalid path "{}": no defined trace packet header type'.format(path))
 703
 704                     base = self._cur_trace.packet_header_type
 705
 706                     if self._cur_entity == _Entity.TRACE_PACKET_HEADER:
 707                         base_is_current = True
 708                 else:
 709                     raise ConfigError('invalid path "{}": unknown names after "trace"'.format(path))
 710             elif parts[0] == 'stream':
 711                 if parts[1] == 'packet' and parts[2] == 'context':
 712                     if self._cur_entity < _Entity.STREAM_PACKET_CONTEXT:
 713                         raise ConfigError('invalid path "{}": cannot access stream packet context here'.format(path))
 714
 715                     if self._cur_stream.packet_context_type is None:
 716                         raise ConfigError('invalid path "{}": no defined stream packet context type'.format(path))
 717
 718                     base = self._cur_stream.packet_context_type
 719
 720                     if self._cur_entity == _Entity.STREAM_PACKET_CONTEXT:
 721                         base_is_current = True
 722                 elif parts[1] == 'event':
 723                     if parts[2] == 'header':
 724                         if self._cur_entity < _Entity.STREAM_EVENT_HEADER:
 725                             raise ConfigError('invalid path "{}": cannot access stream event header here'.format(path))
 726
 727                         if self._cur_stream.event_header_type is None:
 728                             raise ConfigError('invalid path "{}": no defined stream event header type'.format(path))
 729
 730                         base = self._cur_stream.event_header_type
 731
 732                         if self._cur_entity == _Entity.STREAM_EVENT_HEADER:
 733                             base_is_current = True
 734                     elif parts[2] == 'context':
 735                         if self._cur_entity < _Entity.STREAM_EVENT_CONTEXT:
 736                             raise ConfigError('invalid path "{}": cannot access stream event context here'.format(path))
 737
 738                         if self._cur_stream.event_context_type is None:
 739                             raise ConfigError('invalid path "{}": no defined stream event context type'.format(path))
 740
 741                         base = self._cur_stream.event_context_type
 742
 743                         if self._cur_entity == _Entity.STREAM_EVENT_CONTEXT:
 744                             base_is_current = True
 745                     else:
 746                         raise ConfigError('invalid path "{}": unknown names after "stream.event"'.format(path))
 747                 else:
 748                     raise ConfigError('invalid path "{}": unknown names after "stream"'.format(path))
 749
 750             if base is not None:
 751                 start_index = 3
 752
 753         if len(parts) >= 2 and base is None:
 754             if parts[0] == 'event':
 755                 if parts[1] == 'context':
 756                     if self._cur_entity < _Entity.EVENT_CONTEXT:
 757                         raise ConfigError('invalid path "{}": cannot access event context here'.format(path))
 758
 759                     if self._cur_event.context_type is None:
 760                         raise ConfigError('invalid path "{}": no defined event context type'.format(path))
 761
 762                     base = self._cur_event.context_type
 763
 764                     if self._cur_entity == _Entity.EVENT_CONTEXT:
 765                         base_is_current = True
 766                 elif parts[1] == 'payload' or parts[1] == 'fields':
 767                     if self._cur_entity < _Entity.EVENT_PAYLOAD:
 768                         raise ConfigError('invalid path "{}": cannot access event payload here'.format(path))
 769
 770                     if self._cur_event.payload_type is None:
 771                         raise ConfigError('invalid path "{}": no defined event payload type'.format(path))
 772
 773                     base = self._cur_event.payload_type
 774
 775                     if self._cur_entity == _Entity.EVENT_PAYLOAD:
 776                         base_is_current = True
 777                 else:
 778                     raise ConfigError('invalid path "{}": unknown names after "event"'.format(path))
 779
 780             if base is not None:
 781                 start_index = 2
 782
 783         if base is not None:
 784             return self._lookup_path_from_base(path, parts, base, start_index,
 785                                                base_is_current, from_t)
 786         else:
 787             return self._lookup_path_from_top(path, parts)
 788
 789     def _stack_reset(self):
 790         self._stack = []
 791
 792     def _stack_push(self, base_t):
 793         entry = _MetadataDynamicTypesValidatorStackEntry(base_t)
 794         self._stack.append(entry)
 795
 796     def _stack_pop(self):
 797         self._stack.pop()
 798
 799     def _stack_incr_index(self):
 800         self._stack[-1].index += 1
 801
 802     def _visit_struct_type(self, t):
 803         self._stack_push(t)
 804
 805         for field_name, field_type in t.fields.items():
 806             try:
 807                 self._visit_type(field_type)
 808             except Exception as e:
 809                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 810
 811             self._stack_incr_index()
 812
 813         self._stack_pop()
 814
 815     def _visit_array_type(self, t):
 816         if not t.is_static:
 817             # find length type
 818             try:
 819                 length_type = self._lookup_path(t.length, t)
 820             except Exception as e:
 821                 raise ConfigError('invalid array type\'s length', e)
 822
 823             # make sure length type an unsigned integer
 824             if type(length_type) is not metadata.Integer:
 825                 raise ConfigError('array type\'s length does not point to an integer type')
 826
 827             if length_type.signed:
 828                 raise ConfigError('array type\'s length does not point to an unsigned integer type')
 829
 830         self._visit_type(t.element_type)
 831
 832     def _visit_variant_type(self, t):
 833         # find tag type
 834         try:
 835             tag_type = self._lookup_path(t.tag, t)
 836         except Exception as e:
 837             raise ConfigError('invalid variant type\'s tag', e)
 838
 839         # make sure tag type is an enumeration
 840         if type(tag_type) is not metadata.Enum:
 841             raise ConfigError('variant type\'s tag does not point to an enumeration type')
 842
 843         # verify that each variant type's type exists as an enumeration member
 844         for tag_name in t.types.keys():
 845             if tag_name not in tag_type.members:
 846                 raise ConfigError('cannot find variant type\'s type "{}" in pointed tag type'.format(tag_name))
 847
 848         self._stack_push(t)
 849
 850         for type_name, type_type in t.types.items():
 851             try:
 852                 self._visit_type(type_type)
 853             except Exception as e:
 854                 raise ConfigError('in variant type\'s type "{}"'.format(type_name), e)
 855
 856             self._stack_incr_index()
 857
 858         self._stack_pop()
 859
 860     def _visit_type(self, t):
 861         if t is None:
 862             return
 863
 864         if type(t) in self._type_to_visit_type_func:
 865             func = self._type_to_visit_type_func[type(t)]
 866
 867             if func is not None:
 868                 func(t)
 869
 870     def _visit_event(self, ev):
 871         ev_name = ev.name
 872
 873         # set current event
 874         self._cur_event = ev
 875
 876         # visit event context type
 877         self._stack_reset()
 878         self._cur_entity = _Entity.EVENT_CONTEXT
 879
 880         try:
 881             self._visit_type(ev.context_type)
 882         except Exception as e:
 883             raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 884
 885         # visit event payload type
 886         self._stack_reset()
 887         self._cur_entity = _Entity.EVENT_PAYLOAD
 888
 889         try:
 890             self._visit_type(ev.payload_type)
 891         except Exception as e:
 892             raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 893
 894     def _visit_stream(self, stream):
 895         stream_name = stream.name
 896
 897         # set current stream
 898         self._cur_stream = stream
 899
 900         # reset current event
 901         self._cur_event = None
 902
 903         # visit stream packet context type
 904         self._stack_reset()
 905         self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 906
 907         try:
 908             self._visit_type(stream.packet_context_type)
 909         except Exception as e:
 910             raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 911
 912         # visit stream event header type
 913         self._stack_reset()
 914         self._cur_entity = _Entity.STREAM_EVENT_HEADER
 915
 916         try:
 917             self._visit_type(stream.event_header_type)
 918         except Exception as e:
 919             raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 920
 921         # visit stream event context type
 922         self._stack_reset()
 923         self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 924
 925         try:
 926             self._visit_type(stream.event_context_type)
 927         except Exception as e:
 928             raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 929
 930         # visit events
 931         for ev in stream.events.values():
 932             try:
 933                 self._visit_event(ev)
 934             except Exception as e:
 935                 raise ConfigError('invalid stream "{}"'.format(stream_name))
 936
 937     def validate(self, meta):
 938         # set current trace
 939         self._cur_trace = meta.trace
 940
 941         # visit trace packet header type
 942         self._stack_reset()
 943         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 944
 945         try:
 946             self._visit_type(meta.trace.packet_header_type)
 947         except Exception as e:
 948             raise ConfigError('invalid packet header type in trace', e)
 949
 950         # visit streams
 951         for stream in meta.streams.values():
 952             self._visit_stream(stream)
 953
 954
 955 # Since type inheritance allows types to be only partially defined at
 956 # any place in the configuration, this validator validates that actual
 957 # trace, stream, and event types are all complete and valid.
 958 class _MetadataTypesHistologyValidator:
 959     def __init__(self):
 960         self._type_to_validate_type_histology_func = {
 961             metadata.Integer: self._validate_integer_histology,
 962             metadata.FloatingPoint: self._validate_float_histology,
 963             metadata.Enum: self._validate_enum_histology,
 964             metadata.String: self._validate_string_histology,
 965             metadata.Struct: self._validate_struct_histology,
 966             metadata.Array: self._validate_array_histology,
 967             metadata.Variant: self._validate_variant_histology,
 968         }
 969
 970     def _validate_integer_histology(self, t):
 971         # size is set
 972         if t.size is None:
 973             raise ConfigError('missing integer type\'s size')
 974
 975     def _validate_float_histology(self, t):
 976         # exponent digits is set
 977         if t.exp_size is None:
 978             raise ConfigError('missing floating point number type\'s exponent size')
 979
 980         # mantissa digits is set
 981         if t.mant_size is None:
 982             raise ConfigError('missing floating point number type\'s mantissa size')
 983
 984         # exponent and mantissa sum is a multiple of 8
 985         if (t.exp_size + t.mant_size) % 8 != 0:
 986             raise ConfigError('floating point number type\'s mantissa and exponent sizes sum must be a multiple of 8')
 987
 988     def _validate_enum_histology(self, t):
 989         # integer type is set
 990         if t.value_type is None:
 991             raise ConfigError('missing enumeration type\'s value type')
 992
 993         # there's at least one member
 994         if not t.members:
 995             raise ConfigError('enumeration type needs at least one member')
 996
 997         # no overlapping values
 998         ranges = []
 999
1000         for label, value in t.members.items():
1001             for rg in ranges:
1002                 if value[0] <= rg[1] and rg[0] <= value[1]:
1003                     raise ConfigError('enumeration type\'s member "{}" overlaps another member'.format(label))
1004
1005             ranges.append(value)
1006
1007     def _validate_string_histology(self, t):
1008         # always valid
1009         pass
1010
1011     def _validate_struct_histology(self, t):
1012         # all fields are valid
1013         for field_name, field_type in t.fields.items():
1014             try:
1015                 self._validate_type_histology(field_type)
1016             except Exception as e:
1017                 raise ConfigError('invalid structure type\'s field "{}"'.format(field_name), e)
1018
1019     def _validate_array_histology(self, t):
1020         # length is set
1021         if t.length is None:
1022             raise ConfigError('missing array type\'s length')
1023
1024         # element type is set
1025         if t.element_type is None:
1026             raise ConfigError('missing array type\'s element type')
1027
1028         # element type is valid
1029         try:
1030             self._validate_type_histology(t.element_type)
1031         except Exception as e:
1032             raise ConfigError('invalid array type\'s element type', e)
1033
1034     def _validate_variant_histology(self, t):
1035         # tag is set
1036         if t.tag is None:
1037             raise ConfigError('missing variant type\'s tag')
1038
1039         # there's at least one type
1040         if not t.types:
1041             raise ConfigError('variant type needs at least one type')
1042
1043         # all types are valid
1044         for type_name, type_t in t.types.items():
1045             try:
1046                 self._validate_type_histology(type_t)
1047             except Exception as e:
1048                 raise ConfigError('invalid variant type\'s type "{}"'.format(type_name), e)
1049
1050     def _validate_type_histology(self, t):
1051         if t is None:
1052             return
1053
1054         self._type_to_validate_type_histology_func[type(t)](t)
1055
1056     def _validate_entity_type_histology(self, t):
1057         if t is None:
1058             return
1059
1060         # entity cannot be an array
1061         if type(t) is metadata.Array:
1062             raise ConfigError('cannot use an array here')
1063
1064         self._validate_type_histology(t)
1065
1066     def _validate_event_types_histology(self, ev):
1067         ev_name = ev.name
1068
1069         # validate event context type
1070         try:
1071             self._validate_entity_type_histology(ev.context_type)
1072         except Exception as e:
1073             raise ConfigError('invalid event context type for event "{}"'.format(ev_name), e)
1074
1075         # validate event payload type
1076         if ev.payload_type is None:
1077             raise ConfigError('event payload type must exist in event "{}"'.format(ev_name))
1078
1079         # TODO: also check arrays, sequences, and variants
1080         if type(ev.payload_type) is metadata.Struct:
1081             if not ev.payload_type.fields:
1082                 raise ConfigError('event payload type must have at least one field for event "{}"'.format(ev_name))
1083
1084         try:
1085             self._validate_entity_type_histology(ev.payload_type)
1086         except Exception as e:
1087             raise ConfigError('invalid event payload type for event "{}"'.format(ev_name), e)
1088
1089     def _validate_stream_types_histology(self, stream):
1090         stream_name = stream.name
1091
1092         # validate stream packet context type
1093         try:
1094             self._validate_entity_type_histology(stream.packet_context_type)
1095         except Exception as e:
1096             raise ConfigError('invalid stream packet context type for stream "{}"'.format(stream_name), e)
1097
1098         # validate stream event header type
1099         try:
1100             self._validate_entity_type_histology(stream.event_header_type)
1101         except Exception as e:
1102             raise ConfigError('invalid stream event header type for stream "{}"'.format(stream_name), e)
1103
1104         # validate stream event context type
1105         try:
1106             self._validate_entity_type_histology(stream.event_context_type)
1107         except Exception as e:
1108             raise ConfigError('invalid stream event context type for stream "{}"'.format(stream_name), e)
1109
1110         # validate events
1111         for ev in stream.events.values():
1112             try:
1113                 self._validate_event_types_histology(ev)
1114             except Exception as e:
1115                 raise ConfigError('invalid event in stream "{}"'.format(stream_name), e)
1116
1117     def validate(self, meta):
1118         # validate trace packet header type
1119         try:
1120             self._validate_entity_type_histology(meta.trace.packet_header_type)
1121         except Exception as e:
1122             raise ConfigError('invalid trace packet header type', e)
1123
1124         # validate streams
1125         for stream in meta.streams.values():
1126             self._validate_stream_types_histology(stream)
1127
1128
1129 class _YamlConfigParser:
1130     def __init__(self, include_dirs, ignore_include_not_found, dump_config):
1131         self._class_name_to_create_type_func = {
1132             'int': self._create_integer,
1133             'integer': self._create_integer,
1134             'flt': self._create_float,
1135             'float': self._create_float,
1136             'floating-point': self._create_float,
1137             'enum': self._create_enum,
1138             'enumeration': self._create_enum,
1139             'str': self._create_string,
1140             'string': self._create_string,
1141             'struct': self._create_struct,
1142             'structure': self._create_struct,
1143             'array': self._create_array,
1144             'var': self._create_variant,
1145             'variant': self._create_variant,
1146         }
1147         self._type_to_create_type_func = {
1148             metadata.Integer: self._create_integer,
1149             metadata.FloatingPoint: self._create_float,
1150             metadata.Enum: self._create_enum,
1151             metadata.String: self._create_string,
1152             metadata.Struct: self._create_struct,
1153             metadata.Array: self._create_array,
1154             metadata.Variant: self._create_variant,
1155         }
1156         self._include_dirs = include_dirs
1157         self._include_dirs.append(os.getcwd())
1158         self._ignore_include_not_found = ignore_include_not_found
1159         self._dump_config = dump_config
1160
1161     def _set_byte_order(self, metadata_node):
1162         if 'trace' not in metadata_node:
1163             raise ConfigError('missing "trace" property (metadata)')
1164
1165         trace_node = metadata_node['trace']
1166
1167         if not _is_assoc_array_prop(trace_node):
1168             raise ConfigError('"trace" property (metadata) must be an associative array')
1169
1170         if 'byte-order' not in trace_node:
1171             raise ConfigError('missing "byte-order" property (trace)')
1172
1173         bo_node = trace_node['byte-order']
1174
1175         if not _is_str_prop(bo_node):
1176             raise ConfigError('"byte-order" property of trace object must be a string ("le" or "be")')
1177
1178         self._bo = _byte_order_str_to_bo(bo_node)
1179
1180         if self._bo is None:
1181             raise ConfigError('invalid "byte-order" property (trace): must be "le" or "be"')
1182
1183     def _lookup_type_alias(self, name):
1184         if name in self._tas:
1185             return copy.deepcopy(self._tas[name])
1186
1187     def _set_int_clock_prop_mapping(self, int_obj, prop_mapping_node):
1188         unk_prop = _get_first_unknown_prop(prop_mapping_node, ['type', 'name', 'property'])
1189
1190         if unk_prop:
1191             raise ConfigError('unknown property in integer type object\'s clock property mapping: "{}"'.format(unk_prop))
1192
1193         if 'name' not in prop_mapping_node:
1194             raise ConfigError('missing "name" property in integer type object\'s clock property mapping')
1195
1196         if 'property' not in prop_mapping_node:
1197             raise ConfigError('missing "property" property in integer type object\'s clock property mapping')
1198
1199         clock_name = prop_mapping_node['name']
1200         prop = prop_mapping_node['property']
1201
1202         if not _is_str_prop(clock_name):
1203             raise ConfigError('"name" property of integer type object\'s clock property mapping must be a string')
1204
1205         if not _is_str_prop(prop):
1206             raise ConfigError('"property" property of integer type object\'s clock property mapping must be a string')
1207
1208         if clock_name not in self._clocks:
1209             raise ConfigError('invalid clock name "{}" in integer type object\'s clock property mapping'.format(clock_name))
1210
1211         if prop != 'value':
1212             raise ConfigError('invalid "property" property in integer type object\'s clock property mapping: "{}"'.format(prop))
1213
1214         mapped_clock = self._clocks[clock_name]
1215         int_obj.property_mappings.append(metadata.PropertyMapping(mapped_clock, prop))
1216
1217     def _get_first_unknown_type_prop(self, type_node, known_props):
1218         kp = known_props + ['inherit', 'class']
1219
1220         if self._version >= 201:
1221             kp.append('$inherit')
1222
1223         return _get_first_unknown_prop(type_node, kp)
1224
1225     def _create_integer(self, obj, node):
1226         if obj is None:
1227             # create integer object
1228             obj = metadata.Integer()
1229
1230         unk_prop = self._get_first_unknown_type_prop(node, [
1231             'size',
1232             'align',
1233             'signed',
1234             'byte-order',
1235             'base',
1236             'encoding',
1237             'property-mappings',
1238         ])
1239
1240         if unk_prop:
1241             raise ConfigError('unknown integer type object property: "{}"'.format(unk_prop))
1242
1243         # size
1244         if 'size' in node:
1245             size = node['size']
1246
1247             if not _is_int_prop(size):
1248                 raise ConfigError('"size" property of integer type object must be an integer')
1249
1250             if size < 1:
1251                 raise ConfigError('invalid integer size: {}'.format(size))
1252
1253             obj.size = size
1254
1255         # align
1256         if 'align' in node:
1257             align = node['align']
1258
1259             if not _is_int_prop(align):
1260                 raise ConfigError('"align" property of integer type object must be an integer')
1261
1262             if not _is_valid_alignment(align):
1263                 raise ConfigError('invalid alignment: {}'.format(align))
1264
1265             obj.align = align
1266
1267         # signed
1268         if 'signed' in node:
1269             signed = node['signed']
1270
1271             if not _is_bool_prop(signed):
1272                 raise ConfigError('"signed" property of integer type object must be a boolean')
1273
1274             obj.signed = signed
1275
1276         # byte order
1277         if 'byte-order' in node:
1278             byte_order = node['byte-order']
1279
1280             if not _is_str_prop(byte_order):
1281                 raise ConfigError('"byte-order" property of integer type object must be a string ("le" or "be")')
1282
1283             byte_order = _byte_order_str_to_bo(byte_order)
1284
1285             if byte_order is None:
1286                 raise ConfigError('invalid "byte-order" property in integer type object')
1287         else:
1288             byte_order = self._bo
1289
1290         obj.byte_order = byte_order
1291
1292         # base
1293         if 'base' in node:
1294             base = node['base']
1295
1296             if not _is_str_prop(base):
1297                 raise ConfigError('"base" property of integer type object must be a string ("bin", "oct", "dec", or "hex")')
1298
1299             if base == 'bin':
1300                 base = 2
1301             elif base == 'oct':
1302                 base = 8
1303             elif base == 'dec':
1304                 base = 10
1305             elif base == 'hex':
1306                 base = 16
1307             else:
1308                 raise ConfigError('unknown "base" property value: "{}" ("bin", "oct", "dec", and "hex" are accepted)'.format(base))
1309
1310             obj.base = base
1311
1312         # encoding
1313         if 'encoding' in node:
1314             encoding = node['encoding']
1315
1316             if not _is_str_prop(encoding):
1317                 raise ConfigError('"encoding" property of integer type object must be a string ("none", "ascii", or "utf-8")')
1318
1319             encoding = _encoding_str_to_encoding(encoding)
1320
1321             if encoding is None:
1322                 raise ConfigError('invalid "encoding" property in integer type object')
1323
1324             obj.encoding = encoding
1325
1326         # property mappings
1327         if 'property-mappings' in node:
1328             prop_mappings = node['property-mappings']
1329
1330             if not _is_array_prop(prop_mappings):
1331                 raise ConfigError('"property-mappings" property of integer type object must be an array')
1332
1333             if len(prop_mappings) > 1:
1334                 raise ConfigError('length of "property-mappings" array in integer type object must be 1')
1335
1336             del obj.property_mappings[:]
1337
1338             for index, prop_mapping in enumerate(prop_mappings):
1339                 if not _is_assoc_array_prop(prop_mapping):
1340                     raise ConfigError('elements of "property-mappings" property of integer type object must be associative arrays')
1341
1342                 if 'type' not in prop_mapping:
1343                     raise ConfigError('missing "type" property in integer type object\'s "property-mappings" array\'s element #{}'.format(index))
1344
1345                 prop_type = prop_mapping['type']
1346
1347                 if not _is_str_prop(prop_type):
1348                     raise ConfigError('"type" property of integer type object\'s "property-mappings" array\'s element #{} must be a string'.format(index))
1349
1350                 if prop_type == 'clock':
1351                     self._set_int_clock_prop_mapping(obj, prop_mapping)
1352                 else:
1353                     raise ConfigError('unknown property mapping type "{}" in integer type object\'s "property-mappings" array\'s element #{}'.format(prop_type, index))
1354
1355         return obj
1356
1357     def _create_float(self, obj, node):
1358         if obj is None:
1359             # create floating point number object
1360             obj = metadata.FloatingPoint()
1361
1362         unk_prop = self._get_first_unknown_type_prop(node, [
1363             'size',
1364             'align',
1365             'byte-order',
1366         ])
1367
1368         if unk_prop:
1369             raise ConfigError('unknown floating point number type object property: "{}"'.format(unk_prop))
1370
1371         # size
1372         if 'size' in node:
1373             size = node['size']
1374
1375             if not _is_assoc_array_prop(size):
1376                 raise ConfigError('"size" property of floating point number type object must be an associative array')
1377
1378             unk_prop = _get_first_unknown_prop(size, ['exp', 'mant'])
1379
1380             if unk_prop:
1381                 raise ConfigError('unknown floating point number type object\'s "size" property: "{}"'.format(unk_prop))
1382
1383             if 'exp' in size:
1384                 exp = size['exp']
1385
1386                 if not _is_int_prop(exp):
1387                     raise ConfigError('"exp" property of floating point number type object\'s "size" property must be an integer')
1388
1389                 if exp < 1:
1390                     raise ConfigError('invalid floating point number exponent size: {}')
1391
1392                 obj.exp_size = exp
1393
1394             if 'mant' in size:
1395                 mant = size['mant']
1396
1397                 if not _is_int_prop(mant):
1398                     raise ConfigError('"mant" property of floating point number type object\'s "size" property must be an integer')
1399
1400                 if mant < 1:
1401                     raise ConfigError('invalid floating point number mantissa size: {}')
1402
1403                 obj.mant_size = mant
1404
1405         # align
1406         if 'align' in node:
1407             align = node['align']
1408
1409             if not _is_int_prop(align):
1410                 raise ConfigError('"align" property of floating point number type object must be an integer')
1411
1412             if not _is_valid_alignment(align):
1413                 raise ConfigError('invalid alignment: {}'.format(align))
1414
1415             obj.align = align
1416
1417         # byte order
1418         if 'byte-order' in node:
1419             byte_order = node['byte-order']
1420
1421             if not _is_str_prop(byte_order):
1422                 raise ConfigError('"byte-order" property of floating point number type object must be a string ("le" or "be")')
1423
1424             byte_order = _byte_order_str_to_bo(byte_order)
1425
1426             if byte_order is None:
1427                 raise ConfigError('invalid "byte-order" property in floating point number type object')
1428         else:
1429             byte_order = self._bo
1430
1431         obj.byte_order = byte_order
1432
1433         return obj
1434
1435     def _create_enum(self, obj, node):
1436         if obj is None:
1437             # create enumeration object
1438             obj = metadata.Enum()
1439
1440         unk_prop = self._get_first_unknown_type_prop(node, [
1441             'value-type',
1442             'members',
1443         ])
1444
1445         if unk_prop:
1446             raise ConfigError('unknown enumeration type object property: "{}"'.format(unk_prop))
1447
1448         # value type
1449         if 'value-type' in node:
1450             try:
1451                 obj.value_type = self._create_type(node['value-type'])
1452             except Exception as e:
1453                 raise ConfigError('cannot create enumeration type\'s integer type', e)
1454
1455         # members
1456         if 'members' in node:
1457             members_node = node['members']
1458
1459             if not _is_array_prop(members_node):
1460                 raise ConfigError('"members" property of enumeration type object must be an array')
1461
1462             cur = 0
1463
1464             for index, m_node in enumerate(members_node):
1465                 if not _is_str_prop(m_node) and not _is_assoc_array_prop(m_node):
1466                     raise ConfigError('invalid enumeration member #{}: expecting a string or an associative array'.format(index))
1467
1468                 if _is_str_prop(m_node):
1469                     label = m_node
1470                     value = (cur, cur)
1471                     cur += 1
1472                 else:
1473                     unk_prop = _get_first_unknown_prop(m_node, [
1474                         'label',
1475                         'value',
1476                     ])
1477
1478                     if unk_prop:
1479                         raise ConfigError('unknown enumeration type member object property: "{}"'.format(unk_prop))
1480
1481                     if 'label' not in m_node:
1482                         raise ConfigError('missing "label" property in enumeration member #{}'.format(index))
1483
1484                     label = m_node['label']
1485
1486                     if not _is_str_prop(label):
1487                         raise ConfigError('"label" property of enumeration member #{} must be a string'.format(index))
1488
1489                     if 'value' not in m_node:
1490                         raise ConfigError('missing "value" property in enumeration member ("{}")'.format(label))
1491
1492                     value = m_node['value']
1493
1494                     if not _is_int_prop(value) and not _is_array_prop(value):
1495                         raise ConfigError('invalid enumeration member ("{}"): expecting an integer or an array'.format(label))
1496
1497                     if _is_int_prop(value):
1498                         cur = value + 1
1499                         value = (value, value)
1500                     else:
1501                         if len(value) != 2:
1502                             raise ConfigError('invalid enumeration member ("{}"): range must have exactly two items'.format(label))
1503
1504                         mn = value[0]
1505                         mx = value[1]
1506
1507                         if mn > mx:
1508                             raise ConfigError('invalid enumeration member ("{}"): invalid range ({} > {})'.format(label, mn, mx))
1509
1510                         value = (mn, mx)
1511                         cur = mx + 1
1512
1513                 obj.members[label] = value
1514
1515         return obj
1516
1517     def _create_string(self, obj, node):
1518         if obj is None:
1519             # create string object
1520             obj = metadata.String()
1521
1522         unk_prop = self._get_first_unknown_type_prop(node, [
1523             'encoding',
1524         ])
1525
1526         if unk_prop:
1527             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1528
1529         # encoding
1530         if 'encoding' in node:
1531             encoding = node['encoding']
1532
1533             if not _is_str_prop(encoding):
1534                 raise ConfigError('"encoding" property of string type object must be a string ("none", "ascii", or "utf-8")')
1535
1536             encoding = _encoding_str_to_encoding(encoding)
1537
1538             if encoding is None:
1539                 raise ConfigError('invalid "encoding" property in string type object')
1540
1541             obj.encoding = encoding
1542
1543         return obj
1544
1545     def _create_struct(self, obj, node):
1546         if obj is None:
1547             # create structure object
1548             obj = metadata.Struct()
1549
1550         unk_prop = self._get_first_unknown_type_prop(node, [
1551             'min-align',
1552             'fields',
1553         ])
1554
1555         if unk_prop:
1556             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1557
1558         # minimum alignment
1559         if 'min-align' in node:
1560             min_align = node['min-align']
1561
1562             if not _is_int_prop(min_align):
1563                 raise ConfigError('"min-align" property of structure type object must be an integer')
1564
1565             if not _is_valid_alignment(min_align):
1566                 raise ConfigError('invalid minimum alignment: {}'.format(min_align))
1567
1568             obj.min_align = min_align
1569
1570         # fields
1571         if 'fields' in node:
1572             fields = node['fields']
1573
1574             if not _is_assoc_array_prop(fields):
1575                 raise ConfigError('"fields" property of structure type object must be an associative array')
1576
1577             for field_name, field_node in fields.items():
1578                 if not is_valid_identifier(field_name):
1579                     raise ConfigError('"{}" is not a valid field name for structure type'.format(field_name))
1580
1581                 try:
1582                     obj.fields[field_name] = self._create_type(field_node)
1583                 except Exception as e:
1584                     raise ConfigError('cannot create structure type\'s field "{}"'.format(field_name), e)
1585
1586         return obj
1587
1588     def _create_array(self, obj, node):
1589         if obj is None:
1590             # create array object
1591             obj = metadata.Array()
1592
1593         unk_prop = self._get_first_unknown_type_prop(node, [
1594             'length',
1595             'element-type',
1596         ])
1597
1598         if unk_prop:
1599             raise ConfigError('unknown array type object property: "{}"'.format(unk_prop))
1600
1601         # length
1602         if 'length' in node:
1603             length = node['length']
1604
1605             if not _is_int_prop(length) and not _is_str_prop(length):
1606                 raise ConfigError('"length" property of array type object must be an integer or a string')
1607
1608             if type(length) is int and length < 0:
1609                 raise ConfigError('invalid static array length: {}'.format(length))
1610
1611             obj.length = length
1612
1613         # element type
1614         if 'element-type' in node:
1615             try:
1616                 obj.element_type = self._create_type(node['element-type'])
1617             except Exception as e:
1618                 raise ConfigError('cannot create array type\'s element type', e)
1619
1620         return obj
1621
1622     def _create_variant(self, obj, node):
1623         if obj is None:
1624             # create variant object
1625             obj = metadata.Variant()
1626
1627         unk_prop = self._get_first_unknown_type_prop(node, [
1628             'tag',
1629             'types',
1630         ])
1631
1632         if unk_prop:
1633             raise ConfigError('unknown variant type object property: "{}"'.format(unk_prop))
1634
1635         # tag
1636         if 'tag' in node:
1637             tag = node['tag']
1638
1639             if not _is_str_prop(tag):
1640                 raise ConfigError('"tag" property of variant type object must be a string')
1641
1642             # do not validate variant tag for the moment; will be done in a
1643             # second phase
1644             obj.tag = tag
1645
1646         # element type
1647         if 'types' in node:
1648             types = node['types']
1649
1650             if not _is_assoc_array_prop(types):
1651                 raise ConfigError('"types" property of variant type object must be an associative array')
1652
1653             # do not validate type names for the moment; will be done in a
1654             # second phase
1655             for type_name, type_node in types.items():
1656                 if not is_valid_identifier(type_name):
1657                     raise ConfigError('"{}" is not a valid type name for variant type'.format(type_name))
1658
1659                 try:
1660                     obj.types[type_name] = self._create_type(type_node)
1661                 except Exception as e:
1662                     raise ConfigError('cannot create variant type\'s type "{}"'.format(type_name), e)
1663
1664         return obj
1665
1666     def _create_type(self, type_node):
1667         if type(type_node) is str:
1668             t = self._lookup_type_alias(type_node)
1669
1670             if t is None:
1671                 raise ConfigError('unknown type alias "{}"'.format(type_node))
1672
1673             return t
1674
1675         if not _is_assoc_array_prop(type_node):
1676             raise ConfigError('type objects must be associative arrays or strings (type alias name)')
1677
1678         # inherit:
1679         #   v2.0:  "inherit"
1680         #   v2.1+: "$inherit"
1681         inherit_node = None
1682
1683         if self._version >= 200:
1684             if 'inherit' in type_node:
1685                 inherit_prop = 'inherit'
1686                 inherit_node = type_node[inherit_prop]
1687
1688         if self._version >= 201:
1689             if '$inherit' in type_node:
1690                 if inherit_node is not None:
1691                     raise ConfigError('cannot specify both "inherit" and "$inherit" properties of type object: prefer "$inherit"')
1692
1693                 inherit_prop = '$inherit'
1694                 inherit_node = type_node[inherit_prop]
1695
1696         if inherit_node is not None and 'class' in type_node:
1697             raise ConfigError('cannot specify both "{}" and "class" properties in type object'.format(inherit_prop))
1698
1699         if inherit_node is not None:
1700             if not _is_str_prop(inherit_node):
1701                 raise ConfigError('"{}" property of type object must be a string'.format(inherit_prop))
1702
1703             base = self._lookup_type_alias(inherit_node)
1704
1705             if base is None:
1706                 raise ConfigError('cannot inherit from type alias "{}": type alias does not exist at this point'.format(inherit_node))
1707
1708             func = self._type_to_create_type_func[type(base)]
1709         else:
1710             if 'class' not in type_node:
1711                 raise ConfigError('type objects which do not inherit must have a "class" property')
1712
1713             class_name = type_node['class']
1714
1715             if type(class_name) is not str:
1716                 raise ConfigError('type objects\' "class" property must be a string')
1717
1718             if class_name not in self._class_name_to_create_type_func:
1719                 raise ConfigError('unknown type class "{}"'.format(class_name))
1720
1721             base = None
1722             func = self._class_name_to_create_type_func[class_name]
1723
1724         return func(base, type_node)
1725
1726     def _register_type_aliases(self, metadata_node):
1727         self._tas = dict()
1728
1729         if 'type-aliases' not in metadata_node:
1730             return
1731
1732         ta_node = metadata_node['type-aliases']
1733
1734         if not _is_assoc_array_prop(ta_node):
1735             raise ConfigError('"type-aliases" property (metadata) must be an associative array')
1736
1737         for ta_name, ta_type in ta_node.items():
1738             if ta_name in self._tas:
1739                 raise ConfigError('duplicate type alias "{}"'.format(ta_name))
1740
1741             try:
1742                 t = self._create_type(ta_type)
1743             except Exception as e:
1744                 raise ConfigError('cannot create type alias "{}"'.format(ta_name), e)
1745
1746             self._tas[ta_name] = t
1747
1748     def _create_clock(self, node):
1749         # create clock object
1750         clock = metadata.Clock()
1751
1752         if not _is_assoc_array_prop(node):
1753             raise ConfigError('clock objects must be associative arrays')
1754
1755         known_props = [
1756             'uuid',
1757             'description',
1758             'freq',
1759             'error-cycles',
1760             'offset',
1761             'absolute',
1762             'return-ctype',
1763         ]
1764
1765         if self._version >= 201:
1766             known_props.append('$return-ctype')
1767
1768         unk_prop = _get_first_unknown_prop(node, known_props)
1769
1770         if unk_prop:
1771             raise ConfigError('unknown clock object property: "{}"'.format(unk_prop))
1772
1773         # UUID
1774         if 'uuid' in node:
1775             uuidp = node['uuid']
1776
1777             if not _is_str_prop(uuidp):
1778                 raise ConfigError('"uuid" property of clock object must be a string')
1779
1780             try:
1781                 uuidp = uuid.UUID(uuidp)
1782             except:
1783                 raise ConfigError('malformed UUID (clock object): "{}"'.format(uuidp))
1784
1785             clock.uuid = uuidp
1786
1787         # description
1788         if 'description' in node:
1789             desc = node['description']
1790
1791             if not _is_str_prop(desc):
1792                 raise ConfigError('"description" property of clock object must be a string')
1793
1794             clock.description = desc
1795
1796         # frequency
1797         if 'freq' in node:
1798             freq = node['freq']
1799
1800             if not _is_int_prop(freq):
1801                 raise ConfigError('"freq" property of clock object must be an integer')
1802
1803             if freq < 1:
1804                 raise ConfigError('invalid clock frequency: {}'.format(freq))
1805
1806             clock.freq = freq
1807
1808         # error cycles
1809         if 'error-cycles' in node:
1810             error_cycles = node['error-cycles']
1811
1812             if not _is_int_prop(error_cycles):
1813                 raise ConfigError('"error-cycles" property of clock object must be an integer')
1814
1815             if error_cycles < 0:
1816                 raise ConfigError('invalid clock error cycles: {}'.format(error_cycles))
1817
1818             clock.error_cycles = error_cycles
1819
1820         # offset
1821         if 'offset' in node:
1822             offset = node['offset']
1823
1824             if not _is_assoc_array_prop(offset):
1825                 raise ConfigError('"offset" property of clock object must be an associative array')
1826
1827             unk_prop = _get_first_unknown_prop(offset, ['cycles', 'seconds'])
1828
1829             if unk_prop:
1830                 raise ConfigError('unknown clock object\'s offset property: "{}"'.format(unk_prop))
1831
1832             # cycles
1833             if 'cycles' in offset:
1834                 offset_cycles = offset['cycles']
1835
1836                 if not _is_int_prop(offset_cycles):
1837                     raise ConfigError('"cycles" property of clock object\'s offset property must be an integer')
1838
1839                 if offset_cycles < 0:
1840                     raise ConfigError('invalid clock offset cycles: {}'.format(offset_cycles))
1841
1842                 clock.offset_cycles = offset_cycles
1843
1844             # seconds
1845             if 'seconds' in offset:
1846                 offset_seconds = offset['seconds']
1847
1848                 if not _is_int_prop(offset_seconds):
1849                     raise ConfigError('"seconds" property of clock object\'s offset property must be an integer')
1850
1851                 if offset_seconds < 0:
1852                     raise ConfigError('invalid clock offset seconds: {}'.format(offset_seconds))
1853
1854                 clock.offset_seconds = offset_seconds
1855
1856         # absolute
1857         if 'absolute' in node:
1858             absolute = node['absolute']
1859
1860             if not _is_bool_prop(absolute):
1861                 raise ConfigError('"absolute" property of clock object must be a boolean')
1862
1863             clock.absolute = absolute
1864
1865         # return C type:
1866         #   v2.0:  "return-ctype"
1867         #   v2.1+: "$return-ctype"
1868         return_ctype_node = None
1869
1870         if self._version >= 200:
1871             if 'return-ctype' in node:
1872                 return_ctype_prop = 'return-ctype'
1873                 return_ctype_node = node[return_ctype_prop]
1874
1875         if self._version >= 201:
1876             if '$return-ctype' in node:
1877                 if return_ctype_node is not None:
1878                     raise ConfigError('cannot specify both "return-ctype" and "$return-ctype" properties of clock object: prefer "$return-ctype"')
1879
1880                 return_ctype_prop = '$return-ctype'
1881                 return_ctype_node = node[return_ctype_prop]
1882
1883         if return_ctype_node is not None:
1884             if not _is_str_prop(return_ctype_node):
1885                 raise ConfigError('"{}" property of clock object must be a string'.format(return_ctype_prop))
1886
1887             clock.return_ctype = return_ctype_node
1888
1889         return clock
1890
1891     def _register_clocks(self, metadata_node):
1892         self._clocks = collections.OrderedDict()
1893
1894         if 'clocks' not in metadata_node:
1895             return
1896
1897         clocks_node = metadata_node['clocks']
1898
1899         if not _is_assoc_array_prop(clocks_node):
1900             raise ConfigError('"clocks" property (metadata) must be an associative array')
1901
1902         for clock_name, clock_node in clocks_node.items():
1903             if not is_valid_identifier(clock_name):
1904                 raise ConfigError('invalid clock name: "{}"'.format(clock_name))
1905
1906             if clock_name in self._clocks:
1907                 raise ConfigError('duplicate clock "{}"'.format(clock_name))
1908
1909             try:
1910                 clock = self._create_clock(clock_node)
1911             except Exception as e:
1912                 raise ConfigError('cannot create clock "{}"'.format(clock_name), e)
1913
1914             clock.name = clock_name
1915             self._clocks[clock_name] = clock
1916
1917     def _create_env(self, metadata_node):
1918         env = collections.OrderedDict()
1919
1920         if 'env' not in metadata_node:
1921             return env
1922
1923         env_node = metadata_node['env']
1924
1925         if not _is_assoc_array_prop(env_node):
1926             raise ConfigError('"env" property (metadata) must be an associative array')
1927
1928         for env_name, env_value in env_node.items():
1929             if env_name in env:
1930                 raise ConfigError('duplicate environment variable "{}"'.format(env_name))
1931
1932             if not is_valid_identifier(env_name):
1933                 raise ConfigError('invalid environment variable name: "{}"'.format(env_name))
1934
1935             if not _is_int_prop(env_value) and not _is_str_prop(env_value):
1936                 raise ConfigError('invalid environment variable value ("{}"): expecting integer or string'.format(env_name))
1937
1938             env[env_name] = env_value
1939
1940         return env
1941
1942     def _register_log_levels(self, metadata_node):
1943         self._log_levels = dict()
1944
1945         # log levels:
1946         #   v2.0:  "log-levels"
1947         #   v2.1+: "$log-levels"
1948         log_levels_node = None
1949
1950         if self._version >= 200:
1951             if 'log-levels' in metadata_node:
1952                 log_levels_prop = 'log-levels'
1953                 log_levels_node = metadata_node[log_levels_prop]
1954
1955         if self._version >= 201:
1956             if '$log-levels' in metadata_node:
1957                 if log_levels_node is not None:
1958                     raise ConfigError('cannot specify both "log-levels" and "$log-levels" properties of metadata object: prefer "$log-levels"')
1959
1960                 log_levels_prop = '$log-levels'
1961                 log_levels_node = metadata_node[log_levels_prop]
1962
1963         if log_levels_node is None:
1964             return
1965
1966         if not _is_assoc_array_prop(log_levels_node):
1967             raise ConfigError('"{}" property (metadata) must be an associative array'.format(log_levels_prop))
1968
1969         for ll_name, ll_value in log_levels_node.items():
1970             if ll_name in self._log_levels:
1971                 raise ConfigError('duplicate log level entry "{}"'.format(ll_name))
1972
1973             if not _is_int_prop(ll_value):
1974                 raise ConfigError('invalid log level entry ("{}"): expecting an integer'.format(ll_name))
1975
1976             if ll_value < 0:
1977                 raise ConfigError('invalid log level entry ("{}"): log level value must be positive'.format(ll_name))
1978
1979             self._log_levels[ll_name] = ll_value
1980
1981     def _create_trace(self, metadata_node):
1982         # create trace object
1983         trace = metadata.Trace()
1984
1985         if 'trace' not in metadata_node:
1986             raise ConfigError('missing "trace" property (metadata)')
1987
1988         trace_node = metadata_node['trace']
1989
1990         if not _is_assoc_array_prop(trace_node):
1991             raise ConfigError('"trace" property (metadata) must be an associative array')
1992
1993         unk_prop = _get_first_unknown_prop(trace_node, [
1994             'byte-order',
1995             'uuid',
1996             'packet-header-type',
1997         ])
1998
1999         if unk_prop:
2000             raise ConfigError('unknown trace object property: "{}"'.format(unk_prop))
2001
2002         # set byte order (already parsed)
2003         trace.byte_order = self._bo
2004
2005         # UUID
2006         if 'uuid' in trace_node:
2007             uuidp = trace_node['uuid']
2008
2009             if not _is_str_prop(uuidp):
2010                 raise ConfigError('"uuid" property of trace object must be a string')
2011
2012             if uuidp == 'auto':
2013                 uuidp = uuid.uuid1()
2014             else:
2015                 try:
2016                     uuidp = uuid.UUID(uuidp)
2017                 except:
2018                     raise ConfigError('malformed UUID (trace object): "{}"'.format(uuidp))
2019
2020             trace.uuid = uuidp
2021
2022         # packet header type
2023         if 'packet-header-type' in trace_node:
2024             try:
2025                 ph_type = self._create_type(trace_node['packet-header-type'])
2026             except Exception as e:
2027                 raise ConfigError('cannot create packet header type (trace)', e)
2028
2029             trace.packet_header_type = ph_type
2030
2031         return trace
2032
2033     def _lookup_log_level(self, ll):
2034         if _is_int_prop(ll):
2035             return ll
2036         elif _is_str_prop(ll) and ll in self._log_levels:
2037             return self._log_levels[ll]
2038
2039     def _create_event(self, event_node):
2040         event = metadata.Event()
2041
2042         if not _is_assoc_array_prop(event_node):
2043             raise ConfigError('event objects must be associative arrays')
2044
2045         unk_prop = _get_first_unknown_prop(event_node, [
2046             'log-level',
2047             'context-type',
2048             'payload-type',
2049         ])
2050
2051         if unk_prop:
2052             raise ConfigError('unknown event object property: "{}"'.format(unk_prop))
2053
2054         if 'log-level' in event_node:
2055             ll_node = event_node['log-level']
2056
2057             if _is_str_prop(ll_node):
2058                 ll = self._lookup_log_level(event_node['log-level'])
2059
2060                 if ll is None:
2061                     raise ConfigError('cannot find log level "{}"'.format(ll_node))
2062             elif _is_int_prop(ll_node):
2063                 if ll_node < 0:
2064                     raise ConfigError('invalid log level value {}: value must be positive'.format(ll_node))
2065
2066                 ll = ll_node
2067             else:
2068                 raise ConfigError('"log-level" property must be either a string or an integer')
2069
2070             event.log_level = ll
2071
2072         if 'context-type' in event_node:
2073             try:
2074                 t = self._create_type(event_node['context-type'])
2075             except Exception as e:
2076                 raise ConfigError('cannot create event\'s context type object', e)
2077
2078             event.context_type = t
2079
2080         if 'payload-type' not in event_node:
2081             raise ConfigError('missing "payload-type" property in event object')
2082
2083         try:
2084             t = self._create_type(event_node['payload-type'])
2085         except Exception as e:
2086             raise ConfigError('cannot create event\'s payload type object', e)
2087
2088         event.payload_type = t
2089
2090         return event
2091
2092     def _create_stream(self, stream_node):
2093         stream = metadata.Stream()
2094
2095         if not _is_assoc_array_prop(stream_node):
2096             raise ConfigError('stream objects must be associative arrays')
2097
2098         unk_prop = _get_first_unknown_prop(stream_node, [
2099             'packet-context-type',
2100             'event-header-type',
2101             'event-context-type',
2102             'events',
2103         ])
2104
2105         if unk_prop:
2106             raise ConfigError('unknown stream object property: "{}"'.format(unk_prop))
2107
2108         if 'packet-context-type' in stream_node:
2109             try:
2110                 t = self._create_type(stream_node['packet-context-type'])
2111             except Exception as e:
2112                 raise ConfigError('cannot create stream\'s packet context type object', e)
2113
2114             stream.packet_context_type = t
2115
2116         if 'event-header-type' in stream_node:
2117             try:
2118                 t = self._create_type(stream_node['event-header-type'])
2119             except Exception as e:
2120                 raise ConfigError('cannot create stream\'s event header type object', e)
2121
2122             stream.event_header_type = t
2123
2124         if 'event-context-type' in stream_node:
2125             try:
2126                 t = self._create_type(stream_node['event-context-type'])
2127             except Exception as e:
2128                 raise ConfigError('cannot create stream\'s event context type object', e)
2129
2130             stream.event_context_type = t
2131
2132         if 'events' not in stream_node:
2133             raise ConfigError('missing "events" property in stream object')
2134
2135         events = stream_node['events']
2136
2137         if not _is_assoc_array_prop(events):
2138             raise ConfigError('"events" property of stream object must be an associative array')
2139
2140         if not events:
2141             raise ConfigError('at least one event is needed within a stream object')
2142
2143         cur_id = 0
2144
2145         for ev_name, ev_node in events.items():
2146             try:
2147                 ev = self._create_event(ev_node)
2148             except Exception as e:
2149                 raise ConfigError('cannot create event "{}"'.format(ev_name), e)
2150
2151             ev.id = cur_id
2152             ev.name = ev_name
2153             stream.events[ev_name] = ev
2154             cur_id += 1
2155
2156         return stream
2157
2158     def _create_streams(self, metadata_node):
2159         streams = collections.OrderedDict()
2160
2161         if 'streams' not in metadata_node:
2162             raise ConfigError('missing "streams" property (metadata)')
2163
2164         streams_node = metadata_node['streams']
2165
2166         if not _is_assoc_array_prop(streams_node):
2167             raise ConfigError('"streams" property (metadata) must be an associative array')
2168
2169         if not streams_node:
2170             raise ConfigError('at least one stream is needed (metadata)')
2171
2172         cur_id = 0
2173
2174         for stream_name, stream_node in streams_node.items():
2175             try:
2176                 stream = self._create_stream(stream_node)
2177             except Exception as e:
2178                 raise ConfigError('cannot create stream "{}"'.format(stream_name), e)
2179
2180             stream.id = cur_id
2181             stream.name = str(stream_name)
2182             streams[stream_name] = stream
2183             cur_id += 1
2184
2185         return streams
2186
2187     def _create_metadata(self, root):
2188         meta = metadata.Metadata()
2189
2190         if 'metadata' not in root:
2191             raise ConfigError('missing "metadata" property (configuration)')
2192
2193         metadata_node = root['metadata']
2194
2195         if not _is_assoc_array_prop(metadata_node):
2196             raise ConfigError('"metadata" property (configuration) must be an associative array')
2197
2198         known_props = [
2199             'type-aliases',
2200             'log-levels',
2201             'trace',
2202             'env',
2203             'clocks',
2204             'streams',
2205         ]
2206
2207         if self._version >= 201:
2208             known_props.append('$log-levels')
2209
2210         unk_prop = _get_first_unknown_prop(metadata_node, known_props)
2211
2212         if unk_prop:
2213             add = ''
2214
2215             if unk_prop == '$include':
2216                 add = ' (use version 2.1 or greater)'
2217
2218             raise ConfigError('unknown metadata property{}: "{}"'.format(add, unk_prop))
2219
2220         self._set_byte_order(metadata_node)
2221         self._register_clocks(metadata_node)
2222         meta.clocks = self._clocks
2223         self._register_type_aliases(metadata_node)
2224         meta.env = self._create_env(metadata_node)
2225         meta.trace = self._create_trace(metadata_node)
2226         self._register_log_levels(metadata_node)
2227         meta.streams = self._create_streams(metadata_node)
2228
2229         return meta
2230
2231     def _get_version(self, root):
2232         if 'version' not in root:
2233             raise ConfigError('missing "version" property (configuration)')
2234
2235         version_node = root['version']
2236
2237         if not _is_str_prop(version_node):
2238             raise ConfigError('"version" property (configuration) must be a string')
2239
2240         version_node = version_node.strip()
2241
2242         if version_node not in ['2.0', '2.1']:
2243             raise ConfigError('unsupported version ({}): versions 2.0 and 2.1 are supported'.format(version_node))
2244
2245         # convert version string to comparable version integer
2246         parts = version_node.split('.')
2247         version = int(parts[0]) * 100 + int(parts[1])
2248
2249         return version
2250
2251     def _get_prefix(self, root):
2252         if 'prefix' not in root:
2253             return 'barectf_'
2254
2255         prefix_node = root['prefix']
2256
2257         if not _is_str_prop(prefix_node):
2258             raise ConfigError('"prefix" property (configuration) must be a string')
2259
2260         if not is_valid_identifier(prefix_node):
2261             raise ConfigError('"prefix" property (configuration) must be a valid C identifier')
2262
2263         return prefix_node
2264
2265     def _get_last_include_file(self):
2266         if self._include_stack:
2267             return self._include_stack[-1]
2268
2269         return self._root_yaml_path
2270
2271     def _load_include(self, yaml_path):
2272         for inc_dir in self._include_dirs:
2273             # current include dir + file name path
2274             # note: os.path.join() only takes the last arg if it's absolute
2275             inc_path = os.path.join(inc_dir, yaml_path)
2276
2277             # real path (symbolic links resolved)
2278             real_path = os.path.realpath(inc_path)
2279
2280             # normalized path (weird stuff removed!)
2281             norm_path = os.path.normpath(real_path)
2282
2283             if not os.path.isfile(norm_path):
2284                 # file does not exist: skip
2285                 continue
2286
2287             if norm_path in self._include_stack:
2288                 base_path = self._get_last_include_file()
2289                 raise ConfigError('in "{}": cannot recursively include file "{}"'.format(base_path, norm_path))
2290
2291             self._include_stack.append(norm_path)
2292
2293             # load raw content
2294             return self._yaml_ordered_load(norm_path)
2295
2296         if not self._ignore_include_not_found:
2297             base_path = self._get_last_include_file()
2298             raise ConfigError('in "{}": cannot include file "{}": file not found in include directories'.format(base_path, yaml_path))
2299
2300         return None
2301
2302     def _get_include_paths(self, include_node):
2303         if _is_str_prop(include_node):
2304             return [include_node]
2305         elif _is_array_prop(include_node):
2306             for include_path in include_node:
2307                 if not _is_str_prop(include_path):
2308                     raise ConfigError('invalid include property: expecting array of strings')
2309
2310             return include_node
2311
2312         raise ConfigError('invalid include property: expecting string or array of strings')
2313
2314     def _update_node(self, base_node, overlay_node):
2315         for olay_key, olay_value in overlay_node.items():
2316             if olay_key in base_node:
2317                 base_value = base_node[olay_key]
2318
2319                 if _is_assoc_array_prop(olay_value) and _is_assoc_array_prop(base_value):
2320                     # merge dictionaries
2321                     self._update_node(base_value, olay_value)
2322                 elif _is_array_prop(olay_value) and _is_array_prop(base_value):
2323                     # append extension array items to base items
2324                     base_value += olay_value
2325                 else:
2326                     # fall back to replacing
2327                     base_node[olay_key] = olay_value
2328             else:
2329                 base_node[olay_key] = olay_value
2330
2331     def _process_node_include(self, last_overlay_node, name,
2332                               process_base_include_cb,
2333                               process_children_include_cb=None):
2334         if not _is_assoc_array_prop(last_overlay_node):
2335             raise ConfigError('{} objects must be associative arrays'.format(name))
2336
2337         # process children inclusions first
2338         if process_children_include_cb:
2339             process_children_include_cb(last_overlay_node)
2340
2341         if '$include' in last_overlay_node:
2342             include_node = last_overlay_node['$include']
2343         else:
2344             # no includes!
2345             return last_overlay_node
2346
2347         include_paths = self._get_include_paths(include_node)
2348         cur_base_path = self._get_last_include_file()
2349         base_node = None
2350
2351         # keep the include paths and remove the include property
2352         include_paths = copy.deepcopy(include_paths)
2353         del last_overlay_node['$include']
2354
2355         for include_path in include_paths:
2356             # load raw YAML from included file
2357             overlay_node = self._load_include(include_path)
2358
2359             if overlay_node is None:
2360                 # cannot find include file, but we're ignoring those
2361                 # errors, otherwise _load_include() itself raises
2362                 # a config error
2363                 continue
2364
2365             # recursively process includes
2366             try:
2367                 overlay_node = process_base_include_cb(overlay_node)
2368             except Exception as e:
2369                 raise ConfigError('in "{}"'.format(cur_base_path), e)
2370
2371             # pop include stack now that we're done including
2372             del self._include_stack[-1]
2373
2374             # at this point, base_node is fully resolved (does not
2375             # contain any include property)
2376             if base_node is None:
2377                 base_node = overlay_node
2378             else:
2379                 self._update_node(base_node, overlay_node)
2380
2381         # finally, we update the latest base node with our last overlay
2382         # node
2383         if base_node is None:
2384             # nothing was included, which is possible when we're
2385             # ignoring include errors
2386             return last_overlay_node
2387
2388         self._update_node(base_node, last_overlay_node)
2389
2390         return base_node
2391
2392     def _process_event_include(self, event_node):
2393         return self._process_node_include(event_node, 'event',
2394                                           self._process_event_include)
2395
2396     def _process_stream_include(self, stream_node):
2397         def process_children_include(stream_node):
2398             if 'events' in stream_node:
2399                 events_node = stream_node['events']
2400
2401                 if not _is_assoc_array_prop(events_node):
2402                     raise ConfigError('"events" property must be an associative array')
2403
2404                 events_node_keys = list(events_node.keys())
2405
2406                 for key in events_node_keys:
2407                     event_node = events_node[key]
2408
2409                     try:
2410                         events_node[key] = self._process_event_include(event_node)
2411                     except Exception as e:
2412                         raise ConfigError('cannot process includes of event object "{}"'.format(key), e)
2413
2414         return self._process_node_include(stream_node, 'stream',
2415                                           self._process_stream_include,
2416                                           process_children_include)
2417
2418     def _process_trace_include(self, trace_node):
2419         return self._process_node_include(trace_node, 'trace',
2420                                           self._process_trace_include)
2421
2422     def _process_clock_include(self, clock_node):
2423         return self._process_node_include(clock_node, 'clock',
2424                                           self._process_clock_include)
2425
2426     def _process_metadata_include(self, metadata_node):
2427         def process_children_include(metadata_node):
2428             if 'trace' in metadata_node:
2429                 metadata_node['trace'] = self._process_trace_include(metadata_node['trace'])
2430
2431             if 'clocks' in metadata_node:
2432                 clocks_node = metadata_node['clocks']
2433
2434                 if not _is_assoc_array_prop(clocks_node):
2435                     raise ConfigError('"clocks" property (metadata) must be an associative array')
2436
2437                 clocks_node_keys = list(clocks_node.keys())
2438
2439                 for key in clocks_node_keys:
2440                     clock_node = clocks_node[key]
2441
2442                     try:
2443                         clocks_node[key] = self._process_clock_include(clock_node)
2444                     except Exception as e:
2445                         raise ConfigError('cannot process includes of clock object "{}"'.format(key), e)
2446
2447             if 'streams' in metadata_node:
2448                 streams_node = metadata_node['streams']
2449
2450                 if not _is_assoc_array_prop(streams_node):
2451                     raise ConfigError('"streams" property (metadata) must be an associative array')
2452
2453                 streams_node_keys = list(streams_node.keys())
2454
2455                 for key in streams_node_keys:
2456                     stream_node = streams_node[key]
2457
2458                     try:
2459                         streams_node[key] = self._process_stream_include(stream_node)
2460                     except Exception as e:
2461                         raise ConfigError('cannot process includes of stream object "{}"'.format(key), e)
2462
2463         return self._process_node_include(metadata_node, 'metadata',
2464                                           self._process_metadata_include,
2465                                           process_children_include)
2466
2467     def _process_root_includes(self, root):
2468         # The following config objects support includes:
2469         #
2470         #   * Metadata object
2471         #   * Trace object
2472         #   * Stream object
2473         #   * Event object
2474         #
2475         # We need to process the event includes first, then the stream
2476         # includes, then the trace includes, and finally the metadata
2477         # includes.
2478         #
2479         # In each object, only one of the $include and $include-replace
2480         # special properties is allowed.
2481         #
2482         # We keep a stack of absolute paths to included files to detect
2483         # recursion.
2484         if 'metadata' in root:
2485             root['metadata'] = self._process_metadata_include(root['metadata'])
2486
2487         return root
2488
2489     def _yaml_ordered_dump(self, node, **kwds):
2490         class ODumper(yaml.Dumper):
2491             pass
2492
2493         def dict_representer(dumper, node):
2494             return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2495                                             node.items())
2496
2497         ODumper.add_representer(collections.OrderedDict, dict_representer)
2498
2499         return yaml.dump(node, Dumper=ODumper, **kwds)
2500
2501     def _yaml_ordered_load(self, yaml_path):
2502         class OLoader(yaml.Loader):
2503             pass
2504
2505         def construct_mapping(loader, node):
2506             loader.flatten_mapping(node)
2507
2508             return collections.OrderedDict(loader.construct_pairs(node))
2509
2510         OLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2511                                 construct_mapping)
2512
2513         # YAML -> Python
2514         try:
2515             with open(yaml_path, 'r') as f:
2516                 node = yaml.load(f, OLoader)
2517         except (OSError, IOError) as e:
2518             raise ConfigError('cannot open file "{}"'.format(yaml_path))
2519         except Exception as e:
2520             raise ConfigError('unknown error while trying to load file "{}"'.format(yaml_path), e)
2521
2522         # loaded node must be an associate array
2523         if not _is_assoc_array_prop(node):
2524             raise ConfigError('root of YAML file "{}" must be an associative array'.format(yaml_path))
2525
2526         return node
2527
2528     def _reset(self):
2529         self._version = None
2530         self._include_stack = []
2531
2532     def parse(self, yaml_path):
2533         self._reset()
2534         self._root_yaml_path = yaml_path
2535
2536         try:
2537             root = self._yaml_ordered_load(yaml_path)
2538         except Exception as e:
2539             raise ConfigError('cannot parse YAML file "{}"'.format(yaml_path), e)
2540
2541         if not _is_assoc_array_prop(root):
2542             raise ConfigError('configuration must be an associative array')
2543
2544         unk_prop = _get_first_unknown_prop(root, [
2545             'version',
2546             'prefix',
2547             'metadata',
2548         ])
2549
2550         if unk_prop:
2551             raise ConfigError('unknown configuration property: "{}"'.format(unk_prop))
2552
2553         # get the config version
2554         self._version = self._get_version(root)
2555
2556         # process includes if supported
2557         if self._version >= 201:
2558             root = self._process_root_includes(root)
2559
2560         # dump config if required
2561         if self._dump_config:
2562             print(self._yaml_ordered_dump(root, indent=2,
2563                                           default_flow_style=False))
2564
2565         # get prefix and metadata
2566         prefix = self._get_prefix(root)
2567         meta = self._create_metadata(root)
2568
2569         return Config(self._version, prefix, meta)
2570
2571
2572 def from_yaml_file(path, include_dirs, ignore_include_not_found, dump_config):
2573     try:
2574         parser = _YamlConfigParser(include_dirs, ignore_include_not_found,
2575                                    dump_config)
2576         cfg = parser.parse(path)
2577
2578         return cfg
2579     except Exception as e:
2580         raise ConfigError('cannot create configuration from YAML file "{}"'.format(path), e)