barectf/config.py

   1 # The MIT License (MIT)
   2 #
   3 # Copyright (c) 2015 Philippe Proulx <pproulx@efficios.com>
   4 #
   5 # Permission is hereby granted, free of charge, to any person obtaining a copy
   6 # of this software and associated documentation files (the "Software"), to deal
   7 # in the Software without restriction, including without limitation the rights
   8 # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   9 # copies of the Software, and to permit persons to whom the Software is
  10 # furnished to do so, subject to the following conditions:
  11 #
  12 # The above copyright notice and this permission notice shall be included in
  13 # all copies or substantial portions of the Software.
  14 #
  15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  18 # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  21 # THE SOFTWARE.
  22
  23 from barectf import metadata
  24 import collections
  25 import datetime
  26 import barectf
  27 import enum
  28 import yaml
  29 import uuid
  30 import copy
  31 import re
  32 import os
  33
  34
  35 class ConfigError(RuntimeError):
  36     def __init__(self, msg, prev=None):
  37         super().__init__(msg)
  38         self._prev = prev
  39
  40     @property
  41     def prev(self):
  42         return self._prev
  43
  44
  45 class Config:
  46     def __init__(self, version, prefix, metadata):
  47         self.prefix = prefix
  48         self.version = version
  49         self.metadata = metadata
  50
  51     def _validate_metadata(self, meta):
  52         try:
  53             validator = _MetadataTypesHistologyValidator()
  54             validator.validate(meta)
  55             validator = _MetadataDynamicTypesValidator()
  56             validator.validate(meta)
  57             validator = _MetadataSpecialFieldsValidator()
  58             validator.validate(meta)
  59         except Exception as e:
  60             raise ConfigError('metadata error', e)
  61
  62         try:
  63             validator = _BarectfMetadataValidator()
  64             validator.validate(meta)
  65         except Exception as e:
  66             raise ConfigError('barectf metadata error', e)
  67
  68     def _augment_metadata_env(self, meta):
  69         env = meta.env
  70
  71         env['domain'] = 'bare'
  72         env['tracer_name'] = 'barectf'
  73         version_tuple = barectf.get_version_tuple()
  74         env['tracer_major'] = version_tuple[0]
  75         env['tracer_minor'] = version_tuple[1]
  76         env['tracer_patch'] = version_tuple[2]
  77         env['barectf_gen_date'] = str(datetime.datetime.now().isoformat())
  78
  79     @property
  80     def version(self):
  81         return self._version
  82
  83     @version.setter
  84     def version(self, value):
  85         self._version = value
  86
  87     @property
  88     def metadata(self):
  89         return self._metadata
  90
  91     @metadata.setter
  92     def metadata(self, value):
  93         self._validate_metadata(value)
  94         self._augment_metadata_env(value)
  95         self._metadata = value
  96
  97     @property
  98     def prefix(self):
  99         return self._prefix
 100
 101     @prefix.setter
 102     def prefix(self, value):
 103         if not is_valid_identifier(value):
 104             raise ConfigError('prefix must be a valid C identifier')
 105
 106         self._prefix = value
 107
 108
 109 def _is_assoc_array_prop(node):
 110     return isinstance(node, dict)
 111
 112
 113 def _is_array_prop(node):
 114     return isinstance(node, list)
 115
 116
 117 def _is_int_prop(node):
 118     return type(node) is int
 119
 120
 121 def _is_str_prop(node):
 122     return type(node) is str
 123
 124
 125 def _is_bool_prop(node):
 126     return type(node) is bool
 127
 128
 129 def _is_valid_alignment(align):
 130     return ((align & (align - 1)) == 0) and align > 0
 131
 132
 133 def _byte_order_str_to_bo(bo_str):
 134     bo_str = bo_str.lower()
 135
 136     if bo_str == 'le':
 137         return metadata.ByteOrder.LE
 138     elif bo_str == 'be':
 139         return metadata.ByteOrder.BE
 140
 141
 142 def _encoding_str_to_encoding(encoding_str):
 143     encoding_str = encoding_str.lower()
 144
 145     if encoding_str == 'utf-8' or encoding_str == 'utf8':
 146         return metadata.Encoding.UTF8
 147     elif encoding_str == 'ascii':
 148         return metadata.Encoding.ASCII
 149     elif encoding_str == 'none':
 150         return metadata.Encoding.NONE
 151
 152
 153 _re_iden = re.compile(r'^[a-zA-Z][a-zA-Z0-9_]*$')
 154 _ctf_keywords = set([
 155     'align',
 156     'callsite',
 157     'clock',
 158     'enum',
 159     'env',
 160     'event',
 161     'floating_point',
 162     'integer',
 163     'stream',
 164     'string',
 165     'struct',
 166     'trace',
 167     'typealias',
 168     'typedef',
 169     'variant',
 170 ])
 171
 172
 173 def is_valid_identifier(iden):
 174     if not _re_iden.match(iden):
 175         return False
 176
 177     if _re_iden in _ctf_keywords:
 178         return False
 179
 180     return True
 181
 182
 183 def _get_first_unknown_prop(node, known_props):
 184     for prop_name in node:
 185         if prop_name in known_props:
 186             continue
 187
 188         return prop_name
 189
 190
 191 # This validator validates the configured metadata for barectf specific
 192 # needs.
 193 #
 194 # barectf needs:
 195 #
 196 #   * all header/contexts are at least byte-aligned
 197 #   * all integer and floating point number sizes to be <= 64
 198 #   * no inner structures, arrays, or variants
 199 class _BarectfMetadataValidator:
 200     def __init__(self):
 201         self._type_to_validate_type_func = {
 202             metadata.Integer: self._validate_int_type,
 203             metadata.FloatingPoint: self._validate_float_type,
 204             metadata.Enum: self._validate_enum_type,
 205             metadata.String: self._validate_string_type,
 206             metadata.Struct: self._validate_struct_type,
 207             metadata.Array: self._validate_array_type,
 208             metadata.Variant: self._validate_variant_type,
 209         }
 210
 211     def _validate_int_type(self, t, entity_root):
 212         if t.size > 64:
 213             raise ConfigError('integer type\'s size must be lesser than or equal to 64 bits')
 214
 215     def _validate_float_type(self, t, entity_root):
 216         if t.size > 64:
 217             raise ConfigError('floating point number type\'s size must be lesser than or equal to 64 bits')
 218
 219     def _validate_enum_type(self, t, entity_root):
 220         if t.value_type.size > 64:
 221             raise ConfigError('enumeration type\'s integer type\'s size must be lesser than or equal to 64 bits')
 222
 223     def _validate_string_type(self, t, entity_root):
 224         pass
 225
 226     def _validate_struct_type(self, t, entity_root):
 227         if not entity_root:
 228             raise ConfigError('inner structure types are not supported as of this version')
 229
 230         for field_name, field_type in t.fields.items():
 231             if entity_root and self._cur_entity is _Entity.TRACE_PACKET_HEADER:
 232                 if field_name == 'uuid':
 233                     # allow
 234                     continue
 235
 236             try:
 237                 self._validate_type(field_type, False)
 238             except Exception as e:
 239                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 240
 241     def _validate_array_type(self, t, entity_root):
 242         raise ConfigError('array types are not supported as of this version')
 243
 244     def _validate_variant_type(self, t, entity_root):
 245         raise ConfigError('variant types are not supported as of this version')
 246
 247     def _validate_type(self, t, entity_root):
 248         self._type_to_validate_type_func[type(t)](t, entity_root)
 249
 250     def _validate_entity(self, t):
 251         if t is None:
 252             return
 253
 254         # make sure entity is byte-aligned
 255         if t.align < 8:
 256             raise ConfigError('type\'s alignment must be at least byte-aligned')
 257
 258         # make sure entity is a structure
 259         if type(t) is not metadata.Struct:
 260             raise ConfigError('expecting a structure type')
 261
 262         # validate types
 263         self._validate_type(t, True)
 264
 265     def _validate_entities_and_names(self, meta):
 266         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 267
 268         try:
 269             self._validate_entity(meta.trace.packet_header_type)
 270         except Exception as e:
 271             raise ConfigError('invalid trace packet header type', e)
 272
 273         for stream_name, stream in meta.streams.items():
 274             if not is_valid_identifier(stream_name):
 275                 raise ConfigError('stream name "{}" is not a valid C identifier'.format(stream_name))
 276
 277             self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 278
 279             try:
 280                 self._validate_entity(stream.packet_context_type)
 281             except Exception as e:
 282                 raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 283
 284             self._cur_entity = _Entity.STREAM_EVENT_HEADER
 285
 286             try:
 287                 self._validate_entity(stream.event_header_type)
 288             except Exception as e:
 289                 raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 290
 291             self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 292
 293             try:
 294                 self._validate_entity(stream.event_context_type)
 295             except Exception as e:
 296                 raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 297
 298             try:
 299                 for ev_name, ev in stream.events.items():
 300                     if not is_valid_identifier(ev_name):
 301                         raise ConfigError('event name "{}" is not a valid C identifier'.format(ev_name))
 302
 303                     self._cur_entity = _Entity.EVENT_CONTEXT
 304
 305                     try:
 306                         self._validate_entity(ev.context_type)
 307                     except Exception as e:
 308                         raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 309
 310                     self._cur_entity = _Entity.EVENT_PAYLOAD
 311
 312                     if ev.payload_type is None:
 313                         raise ConfigError('missing payload type in event "{}"'.format(ev_name), e)
 314
 315                     try:
 316                         self._validate_entity(ev.payload_type)
 317                     except Exception as e:
 318                         raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 319
 320                     if not ev.payload_type.fields:
 321                         raise ConfigError('empty payload type in event "{}"'.format(ev_name), e)
 322             except Exception as e:
 323                 raise ConfigError('invalid stream "{}"'.format(stream_name), e)
 324
 325     def validate(self, meta):
 326         self._validate_entities_and_names(meta)
 327
 328
 329 # This validator validates special fields of trace, stream, and event
 330 # types. For example, if checks that the "stream_id" field exists in the
 331 # trace packet header if there's more than one stream, and much more.
 332 class _MetadataSpecialFieldsValidator:
 333     def _validate_trace_packet_header_type(self, t):
 334         # needs "stream_id" field?
 335         if len(self._meta.streams) > 1:
 336             # yes
 337             if t is None:
 338                 raise ConfigError('need "stream_id" field in trace packet header type, but trace packet header type is missing')
 339
 340             if type(t) is not metadata.Struct:
 341                 raise ConfigError('need "stream_id" field in trace packet header type, but trace packet header type is not a structure type')
 342
 343             if 'stream_id' not in t.fields:
 344                 raise ConfigError('need "stream_id" field in trace packet header type')
 345
 346         # validate "magic" and "stream_id" types
 347         if type(t) is not metadata.Struct:
 348             return
 349
 350         for i, (field_name, field_type) in enumerate(t.fields.items()):
 351             if field_name == 'magic':
 352                 if type(field_type) is not metadata.Integer:
 353                     raise ConfigError('"magic" field in trace packet header type must be an integer type')
 354
 355                 if field_type.signed or field_type.size != 32:
 356                     raise ConfigError('"magic" field in trace packet header type must be a 32-bit unsigned integer type')
 357
 358                 if i != 0:
 359                     raise ConfigError('"magic" field must be the first trace packet header type\'s field')
 360             elif field_name == 'stream_id':
 361                 if type(field_type) is not metadata.Integer:
 362                     raise ConfigError('"stream_id" field in trace packet header type must be an integer type')
 363
 364                 if field_type.signed:
 365                     raise ConfigError('"stream_id" field in trace packet header type must be an unsigned integer type')
 366             elif field_name == 'uuid':
 367                 if self._meta.trace.uuid is None:
 368                     raise ConfigError('"uuid" field in trace packet header type specified, but no trace UUID provided')
 369
 370                 if type(field_type) is not metadata.Array:
 371                     raise ConfigError('"uuid" field in trace packet header type must be an array')
 372
 373                 if field_type.length != 16:
 374                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 bytes')
 375
 376                 element_type = field_type.element_type
 377
 378                 if type(element_type) is not metadata.Integer:
 379                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 380
 381                 if element_type.size != 8:
 382                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 383
 384                 if element_type.signed:
 385                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned bytes')
 386
 387                 if element_type.align != 8:
 388                     raise ConfigError('"uuid" field in trace packet header type must be an array of 16 unsigned, byte-aligned bytes')
 389
 390     def _validate_trace(self, meta):
 391         self._validate_trace_packet_header_type(meta.trace.packet_header_type)
 392
 393     def _validate_stream_packet_context(self, stream):
 394         t = stream.packet_context_type
 395
 396         if type(t) is None:
 397             return
 398
 399         if type(t) is not metadata.Struct:
 400             return
 401
 402         # "timestamp_begin", if exists, is an unsigned integer type,
 403         # mapped to a clock
 404         if 'timestamp_begin' in t.fields:
 405             ts_begin = t.fields['timestamp_begin']
 406
 407             if type(ts_begin) is not metadata.Integer:
 408                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an integer type')
 409
 410             if ts_begin.signed:
 411                 raise ConfigError('"timestamp_begin" field in stream packet context type must be an unsigned integer type')
 412
 413             if not ts_begin.property_mappings:
 414                 raise ConfigError('"timestamp_begin" field in stream packet context type must be mapped to a clock')
 415
 416         # "timestamp_end", if exists, is an unsigned integer type,
 417         # mapped to a clock
 418         if 'timestamp_end' in t.fields:
 419             ts_end = t.fields['timestamp_end']
 420
 421             if type(ts_end) is not metadata.Integer:
 422                 raise ConfigError('"timestamp_end" field in stream packet context type must be an integer type')
 423
 424             if ts_end.signed:
 425                 raise ConfigError('"timestamp_end" field in stream packet context type must be an unsigned integer type')
 426
 427             if not ts_end.property_mappings:
 428                 raise ConfigError('"timestamp_end" field in stream packet context type must be mapped to a clock')
 429
 430         # "timestamp_begin" and "timestamp_end" exist together
 431         if (('timestamp_begin' in t.fields) ^ ('timestamp_end' in t.fields)):
 432             raise ConfigError('"timestamp_begin" and "timestamp_end" fields must be defined together in stream packet context type')
 433
 434         # "events_discarded", if exists, is an unsigned integer type
 435         if 'events_discarded' in t.fields:
 436             events_discarded = t.fields['events_discarded']
 437
 438             if type(events_discarded) is not metadata.Integer:
 439                 raise ConfigError('"events_discarded" field in stream packet context type must be an integer type')
 440
 441             if events_discarded.signed:
 442                 raise ConfigError('"events_discarded" field in stream packet context type must be an unsigned integer type')
 443
 444         # "packet_size" and "content_size" must exist
 445         if 'packet_size' not in t.fields:
 446             raise ConfigError('missing "packet_size" field in stream packet context type')
 447
 448         packet_size = t.fields['packet_size']
 449
 450         # "content_size" and "content_size" must exist
 451         if 'content_size' not in t.fields:
 452             raise ConfigError('missing "content_size" field in stream packet context type')
 453
 454         content_size = t.fields['content_size']
 455
 456         # "packet_size" is an unsigned integer type
 457         if type(packet_size) is not metadata.Integer:
 458             raise ConfigError('"packet_size" field in stream packet context type must be an integer type')
 459
 460         if packet_size.signed:
 461             raise ConfigError('"packet_size" field in stream packet context type must be an unsigned integer type')
 462
 463         # "content_size" is an unsigned integer type
 464         if type(content_size) is not metadata.Integer:
 465             raise ConfigError('"content_size" field in stream packet context type must be an integer type')
 466
 467         if content_size.signed:
 468             raise ConfigError('"content_size" field in stream packet context type must be an unsigned integer type')
 469
 470     def _validate_stream_event_header(self, stream):
 471         t = stream.event_header_type
 472
 473         # needs "id" field?
 474         if len(stream.events) > 1:
 475             # yes
 476             if t is None:
 477                 raise ConfigError('need "id" field in stream event header type, but stream event header type is missing')
 478
 479             if type(t) is not metadata.Struct:
 480                 raise ConfigError('need "id" field in stream event header type, but stream event header type is not a structure type')
 481
 482             if 'id' not in t.fields:
 483                 raise ConfigError('need "id" field in stream event header type')
 484
 485         # validate "id" and "timestamp" types
 486         if type(t) is not metadata.Struct:
 487             return
 488
 489         # "timestamp", if exists, is an unsigned integer type,
 490         # mapped to a clock
 491         if 'timestamp' in t.fields:
 492             ts = t.fields['timestamp']
 493
 494             if type(ts) is not metadata.Integer:
 495                 raise ConfigError('"ts" field in stream event header type must be an integer type')
 496
 497             if ts.signed:
 498                 raise ConfigError('"ts" field in stream event header type must be an unsigned integer type')
 499
 500             if not ts.property_mappings:
 501                 raise ConfigError('"ts" field in stream event header type must be mapped to a clock')
 502
 503         # "id" is an unsigned integer type
 504         if 'id' in t.fields:
 505             eid = t.fields['id']
 506
 507             if type(eid) is not metadata.Integer:
 508                 raise ConfigError('"id" field in stream event header type must be an integer type')
 509
 510             if eid.signed:
 511                 raise ConfigError('"id" field in stream event header type must be an unsigned integer type')
 512
 513     def _validate_stream(self, stream):
 514         self._validate_stream_packet_context(stream)
 515         self._validate_stream_event_header(stream)
 516
 517     def validate(self, meta):
 518         self._meta = meta
 519         self._validate_trace(meta)
 520
 521         for stream in meta.streams.values():
 522             try:
 523                 self._validate_stream(stream)
 524             except Exception as e:
 525                 raise ConfigError('invalid stream "{}"'.format(stream.name), e)
 526
 527
 528 class _MetadataDynamicTypesValidatorStackEntry:
 529     def __init__(self, base_t):
 530         self._base_t = base_t
 531         self._index = 0
 532
 533     @property
 534     def index(self):
 535         return self._index
 536
 537     @index.setter
 538     def index(self, value):
 539         self._index = value
 540
 541     @property
 542     def base_t(self):
 543         return self._base_t
 544
 545     @base_t.setter
 546     def base_t(self, value):
 547         self._base_t = value
 548
 549
 550 # Entities. Order of values is important here.
 551 @enum.unique
 552 class _Entity(enum.IntEnum):
 553     TRACE_PACKET_HEADER = 0
 554     STREAM_PACKET_CONTEXT = 1
 555     STREAM_EVENT_HEADER = 2
 556     STREAM_EVENT_CONTEXT = 3
 557     EVENT_CONTEXT = 4
 558     EVENT_PAYLOAD = 5
 559
 560
 561 # This validator validates dynamic metadata types, that is, it ensures
 562 # variable-length array lengths and variant tags actually point to
 563 # something that exists. It also checks that variable-length array
 564 # lengths point to integer types and variant tags to enumeration types.
 565 class _MetadataDynamicTypesValidator:
 566     def __init__(self):
 567         self._type_to_visit_type_func = {
 568             metadata.Integer: None,
 569             metadata.FloatingPoint: None,
 570             metadata.Enum: None,
 571             metadata.String: None,
 572             metadata.Struct: self._visit_struct_type,
 573             metadata.Array: self._visit_array_type,
 574             metadata.Variant: self._visit_variant_type,
 575         }
 576
 577         self._cur_trace = None
 578         self._cur_stream = None
 579         self._cur_event = None
 580
 581     def _lookup_path_from_base(self, path, parts, base, start_index,
 582                                base_is_current, from_t):
 583         index = start_index
 584         cur_t = base
 585         found_path = []
 586
 587         while index < len(parts):
 588             part = parts[index]
 589             next_t = None
 590
 591             if type(cur_t) is metadata.Struct:
 592                 enumerated_items = enumerate(cur_t.fields.items())
 593
 594                 # lookup each field
 595                 for i, (field_name, field_type) in enumerated_items:
 596                     if field_name == part:
 597                         next_t = field_type
 598                         found_path.append((i, field_type))
 599
 600                 if next_t is None:
 601                     raise ConfigError('invalid path "{}": cannot find field "{}" in structure type'.format(path, part))
 602             elif type(cur_t) is metadata.Variant:
 603                 enumerated_items = enumerate(cur_t.types.items())
 604
 605                 # lookup each type
 606                 for i, (type_name, type_type) in enumerated_items:
 607                     if type_name == part:
 608                         next_t = type_type
 609                         found_path.append((i, type_type))
 610
 611                 if next_t is None:
 612                     raise ConfigError('invalid path "{}": cannot find type "{}" in variant type'.format(path, part))
 613             else:
 614                 raise ConfigError('invalid path "{}": requesting "{}" in a non-variant, non-structure type'.format(path, part))
 615
 616             cur_t = next_t
 617             index += 1
 618
 619         # make sure that the pointed type is not the pointing type
 620         if from_t is cur_t:
 621             raise ConfigError('invalid path "{}": pointing to self'.format(path))
 622
 623         # if we're here, we found the type; however, it could be located
 624         # _after_ the variant/VLA looking for it, if the pointing
 625         # and pointed types are in the same entity, so compare the
 626         # current stack entries indexes to our index path in that case
 627         if not base_is_current:
 628             return cur_t
 629
 630         for index, entry in enumerate(self._stack):
 631             if index == len(found_path):
 632                 # end of index path; valid so far
 633                 break
 634
 635             if found_path[index][0] > entry.index:
 636                 raise ConfigError('invalid path "{}": pointed type is after pointing type'.format(path))
 637
 638         # also make sure that both pointed and pointing types share
 639         # a common structure ancestor
 640         for index, entry in enumerate(self._stack):
 641             if index == len(found_path):
 642                 break
 643
 644             if entry.base_t is not found_path[index][1]:
 645                 # found common ancestor
 646                 if type(entry.base_t) is metadata.Variant:
 647                     raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 648
 649         return cur_t
 650
 651     def _lookup_path_from_top(self, path, parts):
 652         if len(parts) != 1:
 653             raise ConfigError('invalid path "{}": multipart relative path not supported'.format(path))
 654
 655         find_name = parts[0]
 656         index = len(self._stack) - 1
 657         got_struct = False
 658
 659         # check stack entries in reversed order
 660         for entry in reversed(self._stack):
 661             # structure base type
 662             if type(entry.base_t) is metadata.Struct:
 663                 got_struct = True
 664                 enumerated_items = enumerate(entry.base_t.fields.items())
 665
 666                 # lookup each field, until the current visiting index is met
 667                 for i, (field_name, field_type) in enumerated_items:
 668                     if i == entry.index:
 669                         break
 670
 671                     if field_name == find_name:
 672                         return field_type
 673
 674             # variant base type
 675             elif type(entry.base_t) is metadata.Variant:
 676                 enumerated_items = enumerate(entry.base_t.types.items())
 677
 678                 # lookup each type, until the current visiting index is met
 679                 for i, (type_name, type_type) in enumerated_items:
 680                     if i == entry.index:
 681                         break
 682
 683                     if type_name == find_name:
 684                         if not got_struct:
 685                             raise ConfigError('invalid path "{}": type cannot be reached because pointed and pointing types are in the same variant type'.format(path))
 686
 687                         return type_type
 688
 689         # nothing returned here: cannot find type
 690         raise ConfigError('invalid path "{}": cannot find type in current context'.format(path))
 691
 692     def _lookup_path(self, path, from_t):
 693         parts = path.lower().split('.')
 694         base = None
 695         base_is_current = False
 696
 697         if len(parts) >= 3:
 698             if parts[0] == 'trace':
 699                 if parts[1] == 'packet' and parts[2] == 'header':
 700                     # make sure packet header exists
 701                     if self._cur_trace.packet_header_type is None:
 702                         raise ConfigError('invalid path "{}": no defined trace packet header type'.format(path))
 703
 704                     base = self._cur_trace.packet_header_type
 705
 706                     if self._cur_entity == _Entity.TRACE_PACKET_HEADER:
 707                         base_is_current = True
 708                 else:
 709                     raise ConfigError('invalid path "{}": unknown names after "trace"'.format(path))
 710             elif parts[0] == 'stream':
 711                 if parts[1] == 'packet' and parts[2] == 'context':
 712                     if self._cur_entity < _Entity.STREAM_PACKET_CONTEXT:
 713                         raise ConfigError('invalid path "{}": cannot access stream packet context here'.format(path))
 714
 715                     if self._cur_stream.packet_context_type is None:
 716                         raise ConfigError('invalid path "{}": no defined stream packet context type'.format(path))
 717
 718                     base = self._cur_stream.packet_context_type
 719
 720                     if self._cur_entity == _Entity.STREAM_PACKET_CONTEXT:
 721                         base_is_current = True
 722                 elif parts[1] == 'event':
 723                     if parts[2] == 'header':
 724                         if self._cur_entity < _Entity.STREAM_EVENT_HEADER:
 725                             raise ConfigError('invalid path "{}": cannot access stream event header here'.format(path))
 726
 727                         if self._cur_stream.event_header_type is None:
 728                             raise ConfigError('invalid path "{}": no defined stream event header type'.format(path))
 729
 730                         base = self._cur_stream.event_header_type
 731
 732                         if self._cur_entity == _Entity.STREAM_EVENT_HEADER:
 733                             base_is_current = True
 734                     elif parts[2] == 'context':
 735                         if self._cur_entity < _Entity.STREAM_EVENT_CONTEXT:
 736                             raise ConfigError('invalid path "{}": cannot access stream event context here'.format(path))
 737
 738                         if self._cur_stream.event_context_type is None:
 739                             raise ConfigError('invalid path "{}": no defined stream event context type'.format(path))
 740
 741                         base = self._cur_stream.event_context_type
 742
 743                         if self._cur_entity == _Entity.STREAM_EVENT_CONTEXT:
 744                             base_is_current = True
 745                     else:
 746                         raise ConfigError('invalid path "{}": unknown names after "stream.event"'.format(path))
 747                 else:
 748                     raise ConfigError('invalid path "{}": unknown names after "stream"'.format(path))
 749
 750             if base is not None:
 751                 start_index = 3
 752
 753         if len(parts) >= 2 and base is None:
 754             if parts[0] == 'event':
 755                 if parts[1] == 'context':
 756                     if self._cur_entity < _Entity.EVENT_CONTEXT:
 757                         raise ConfigError('invalid path "{}": cannot access event context here'.format(path))
 758
 759                     if self._cur_event.context_type is None:
 760                         raise ConfigError('invalid path "{}": no defined event context type'.format(path))
 761
 762                     base = self._cur_event.context_type
 763
 764                     if self._cur_entity == _Entity.EVENT_CONTEXT:
 765                         base_is_current = True
 766                 elif parts[1] == 'payload' or parts[1] == 'fields':
 767                     if self._cur_entity < _Entity.EVENT_PAYLOAD:
 768                         raise ConfigError('invalid path "{}": cannot access event payload here'.format(path))
 769
 770                     if self._cur_event.payload_type is None:
 771                         raise ConfigError('invalid path "{}": no defined event payload type'.format(path))
 772
 773                     base = self._cur_event.payload_type
 774
 775                     if self._cur_entity == _Entity.EVENT_PAYLOAD:
 776                         base_is_current = True
 777                 else:
 778                     raise ConfigError('invalid path "{}": unknown names after "event"'.format(path))
 779
 780             if base is not None:
 781                 start_index = 2
 782
 783         if base is not None:
 784             return self._lookup_path_from_base(path, parts, base, start_index,
 785                                                base_is_current, from_t)
 786         else:
 787             return self._lookup_path_from_top(path, parts)
 788
 789     def _stack_reset(self):
 790         self._stack = []
 791
 792     def _stack_push(self, base_t):
 793         entry = _MetadataDynamicTypesValidatorStackEntry(base_t)
 794         self._stack.append(entry)
 795
 796     def _stack_pop(self):
 797         self._stack.pop()
 798
 799     def _stack_incr_index(self):
 800         self._stack[-1].index += 1
 801
 802     def _visit_struct_type(self, t):
 803         self._stack_push(t)
 804
 805         for field_name, field_type in t.fields.items():
 806             try:
 807                 self._visit_type(field_type)
 808             except Exception as e:
 809                 raise ConfigError('in structure type\'s field "{}"'.format(field_name), e)
 810
 811             self._stack_incr_index()
 812
 813         self._stack_pop()
 814
 815     def _visit_array_type(self, t):
 816         if not t.is_static:
 817             # find length type
 818             try:
 819                 length_type = self._lookup_path(t.length, t)
 820             except Exception as e:
 821                 raise ConfigError('invalid array type\'s length', e)
 822
 823             # make sure length type an unsigned integer
 824             if type(length_type) is not metadata.Integer:
 825                 raise ConfigError('array type\'s length does not point to an integer type')
 826
 827             if length_type.signed:
 828                 raise ConfigError('array type\'s length does not point to an unsigned integer type')
 829
 830         self._visit_type(t.element_type)
 831
 832     def _visit_variant_type(self, t):
 833         # find tag type
 834         try:
 835             tag_type = self._lookup_path(t.tag, t)
 836         except Exception as e:
 837             raise ConfigError('invalid variant type\'s tag', e)
 838
 839         # make sure tag type is an enumeration
 840         if type(tag_type) is not metadata.Enum:
 841             raise ConfigError('variant type\'s tag does not point to an enumeration type')
 842
 843         # verify that each variant type's type exists as an enumeration member
 844         for tag_name in t.types.keys():
 845             if tag_name not in tag_type.members:
 846                 raise ConfigError('cannot find variant type\'s type "{}" in pointed tag type'.format(tag_name))
 847
 848         self._stack_push(t)
 849
 850         for type_name, type_type in t.types.items():
 851             try:
 852                 self._visit_type(type_type)
 853             except Exception as e:
 854                 raise ConfigError('in variant type\'s type "{}"'.format(type_name), e)
 855
 856             self._stack_incr_index()
 857
 858         self._stack_pop()
 859
 860     def _visit_type(self, t):
 861         if t is None:
 862             return
 863
 864         if type(t) in self._type_to_visit_type_func:
 865             func = self._type_to_visit_type_func[type(t)]
 866
 867             if func is not None:
 868                 func(t)
 869
 870     def _visit_event(self, ev):
 871         ev_name = ev.name
 872
 873         # set current event
 874         self._cur_event = ev
 875
 876         # visit event context type
 877         self._stack_reset()
 878         self._cur_entity = _Entity.EVENT_CONTEXT
 879
 880         try:
 881             self._visit_type(ev.context_type)
 882         except Exception as e:
 883             raise ConfigError('invalid context type in event "{}"'.format(ev_name), e)
 884
 885         # visit event payload type
 886         self._stack_reset()
 887         self._cur_entity = _Entity.EVENT_PAYLOAD
 888
 889         try:
 890             self._visit_type(ev.payload_type)
 891         except Exception as e:
 892             raise ConfigError('invalid payload type in event "{}"'.format(ev_name), e)
 893
 894     def _visit_stream(self, stream):
 895         stream_name = stream.name
 896
 897         # set current stream
 898         self._cur_stream = stream
 899
 900         # reset current event
 901         self._cur_event = None
 902
 903         # visit stream packet context type
 904         self._stack_reset()
 905         self._cur_entity = _Entity.STREAM_PACKET_CONTEXT
 906
 907         try:
 908             self._visit_type(stream.packet_context_type)
 909         except Exception as e:
 910             raise ConfigError('invalid packet context type in stream "{}"'.format(stream_name), e)
 911
 912         # visit stream event header type
 913         self._stack_reset()
 914         self._cur_entity = _Entity.STREAM_EVENT_HEADER
 915
 916         try:
 917             self._visit_type(stream.event_header_type)
 918         except Exception as e:
 919             raise ConfigError('invalid event header type in stream "{}"'.format(stream_name), e)
 920
 921         # visit stream event context type
 922         self._stack_reset()
 923         self._cur_entity = _Entity.STREAM_EVENT_CONTEXT
 924
 925         try:
 926             self._visit_type(stream.event_context_type)
 927         except Exception as e:
 928             raise ConfigError('invalid event context type in stream "{}"'.format(stream_name), e)
 929
 930         # visit events
 931         for ev in stream.events.values():
 932             try:
 933                 self._visit_event(ev)
 934             except Exception as e:
 935                 raise ConfigError('invalid stream "{}"'.format(stream_name))
 936
 937     def validate(self, meta):
 938         # set current trace
 939         self._cur_trace = meta.trace
 940
 941         # visit trace packet header type
 942         self._stack_reset()
 943         self._cur_entity = _Entity.TRACE_PACKET_HEADER
 944
 945         try:
 946             self._visit_type(meta.trace.packet_header_type)
 947         except Exception as e:
 948             raise ConfigError('invalid packet header type in trace', e)
 949
 950         # visit streams
 951         for stream in meta.streams.values():
 952             self._visit_stream(stream)
 953
 954
 955 # Since type inheritance allows types to be only partially defined at
 956 # any place in the configuration, this validator validates that actual
 957 # trace, stream, and event types are all complete and valid.
 958 class _MetadataTypesHistologyValidator:
 959     def __init__(self):
 960         self._type_to_validate_type_histology_func = {
 961             metadata.Integer: self._validate_integer_histology,
 962             metadata.FloatingPoint: self._validate_float_histology,
 963             metadata.Enum: self._validate_enum_histology,
 964             metadata.String: self._validate_string_histology,
 965             metadata.Struct: self._validate_struct_histology,
 966             metadata.Array: self._validate_array_histology,
 967             metadata.Variant: self._validate_variant_histology,
 968         }
 969
 970     def _validate_integer_histology(self, t):
 971         # size is set
 972         if t.size is None:
 973             raise ConfigError('missing integer type\'s size')
 974
 975     def _validate_float_histology(self, t):
 976         # exponent digits is set
 977         if t.exp_size is None:
 978             raise ConfigError('missing floating point number type\'s exponent size')
 979
 980         # mantissa digits is set
 981         if t.mant_size is None:
 982             raise ConfigError('missing floating point number type\'s mantissa size')
 983
 984         # exponent and mantissa sum is a multiple of 8
 985         if (t.exp_size + t.mant_size) % 8 != 0:
 986             raise ConfigError('floating point number type\'s mantissa and exponent sizes sum must be a multiple of 8')
 987
 988     def _validate_enum_histology(self, t):
 989         # integer type is set
 990         if t.value_type is None:
 991             raise ConfigError('missing enumeration type\'s integer type')
 992
 993         # there's at least one member
 994         if not t.members:
 995             raise ConfigError('enumeration type needs at least one member')
 996
 997         # no overlapping values
 998         ranges = []
 999
1000         for label, value in t.members.items():
1001             for rg in ranges:
1002                 if value[0] <= rg[1] and rg[0] <= value[1]:
1003                     raise ConfigError('enumeration type\'s member "{}" overlaps another member'.format(label))
1004
1005             ranges.append(value)
1006
1007     def _validate_string_histology(self, t):
1008         # always valid
1009         pass
1010
1011     def _validate_struct_histology(self, t):
1012         # all fields are valid
1013         for field_name, field_type in t.fields.items():
1014             try:
1015                 self._validate_type_histology(field_type)
1016             except Exception as e:
1017                 raise ConfigError('invalid structure type\'s field "{}"'.format(field_name), e)
1018
1019     def _validate_array_histology(self, t):
1020         # length is set
1021         if t.length is None:
1022             raise ConfigError('missing array type\'s length')
1023
1024         # element type is set
1025         if t.element_type is None:
1026             raise ConfigError('missing array type\'s element type')
1027
1028         # element type is valid
1029         try:
1030             self._validate_type_histology(t.element_type)
1031         except Exception as e:
1032             raise ConfigError('invalid array type\'s element type', e)
1033
1034     def _validate_variant_histology(self, t):
1035         # tag is set
1036         if t.tag is None:
1037             raise ConfigError('missing variant type\'s tag')
1038
1039         # there's at least one type
1040         if not t.types:
1041             raise ConfigError('variant type needs at least one type')
1042
1043         # all types are valid
1044         for type_name, type_t in t.types.items():
1045             try:
1046                 self._validate_type_histology(type_t)
1047             except Exception as e:
1048                 raise ConfigError('invalid variant type\'s type "{}"'.format(type_name), e)
1049
1050     def _validate_type_histology(self, t):
1051         if t is None:
1052             return
1053
1054         self._type_to_validate_type_histology_func[type(t)](t)
1055
1056     def _validate_entity_type_histology(self, t):
1057         if t is None:
1058             return
1059
1060         # entity cannot be an array
1061         if type(t) is metadata.Array:
1062             raise ConfigError('cannot use an array here')
1063
1064         self._validate_type_histology(t)
1065
1066     def _validate_event_types_histology(self, ev):
1067         ev_name = ev.name
1068
1069         # validate event context type
1070         try:
1071             self._validate_entity_type_histology(ev.context_type)
1072         except Exception as e:
1073             raise ConfigError('invalid event context type for event "{}"'.format(ev_name), e)
1074
1075         # validate event payload type
1076         if ev.payload_type is None:
1077             raise ConfigError('event payload type must exist in event "{}"'.format(ev_name))
1078
1079         # TODO: also check arrays, sequences, and variants
1080         if type(ev.payload_type) is metadata.Struct:
1081             if not ev.payload_type.fields:
1082                 raise ConfigError('event payload type must have at least one field for event "{}"'.format(ev_name))
1083
1084         try:
1085             self._validate_entity_type_histology(ev.payload_type)
1086         except Exception as e:
1087             raise ConfigError('invalid event payload type for event "{}"'.format(ev_name), e)
1088
1089     def _validate_stream_types_histology(self, stream):
1090         stream_name = stream.name
1091
1092         # validate stream packet context type
1093         try:
1094             self._validate_entity_type_histology(stream.packet_context_type)
1095         except Exception as e:
1096             raise ConfigError('invalid stream packet context type for stream "{}"'.format(stream_name), e)
1097
1098         # validate stream event header type
1099         try:
1100             self._validate_entity_type_histology(stream.event_header_type)
1101         except Exception as e:
1102             raise ConfigError('invalid stream event header type for stream "{}"'.format(stream_name), e)
1103
1104         # validate stream event context type
1105         try:
1106             self._validate_entity_type_histology(stream.event_context_type)
1107         except Exception as e:
1108             raise ConfigError('invalid stream event context type for stream "{}"'.format(stream_name), e)
1109
1110         # validate events
1111         for ev in stream.events.values():
1112             try:
1113                 self._validate_event_types_histology(ev)
1114             except Exception as e:
1115                 raise ConfigError('invalid event in stream "{}"'.format(stream_name), e)
1116
1117     def validate(self, meta):
1118         # validate trace packet header type
1119         try:
1120             self._validate_entity_type_histology(meta.trace.packet_header_type)
1121         except Exception as e:
1122             raise ConfigError('invalid trace packet header type', e)
1123
1124         # validate streams
1125         for stream in meta.streams.values():
1126             self._validate_stream_types_histology(stream)
1127
1128
1129 class _YamlConfigParser:
1130     def __init__(self, include_dirs, ignore_include_not_found, dump_config):
1131         self._class_name_to_create_type_func = {
1132             'int': self._create_integer,
1133             'integer': self._create_integer,
1134             'flt': self._create_float,
1135             'float': self._create_float,
1136             'floating-point': self._create_float,
1137             'enum': self._create_enum,
1138             'enumeration': self._create_enum,
1139             'str': self._create_string,
1140             'string': self._create_string,
1141             'struct': self._create_struct,
1142             'structure': self._create_struct,
1143             'array': self._create_array,
1144             'var': self._create_variant,
1145             'variant': self._create_variant,
1146         }
1147         self._type_to_create_type_func = {
1148             metadata.Integer: self._create_integer,
1149             metadata.FloatingPoint: self._create_float,
1150             metadata.Enum: self._create_enum,
1151             metadata.String: self._create_string,
1152             metadata.Struct: self._create_struct,
1153             metadata.Array: self._create_array,
1154             metadata.Variant: self._create_variant,
1155         }
1156         self._include_dirs = include_dirs
1157         self._include_dirs.append(os.getcwd())
1158         self._ignore_include_not_found = ignore_include_not_found
1159         self._dump_config = dump_config
1160
1161     def _set_byte_order(self, metadata_node):
1162         if 'trace' not in metadata_node:
1163             raise ConfigError('missing "trace" property (metadata)')
1164
1165         trace_node = metadata_node['trace']
1166
1167         if not _is_assoc_array_prop(trace_node):
1168             raise ConfigError('"trace" property (metadata) must be an associative array')
1169
1170         if 'byte-order' not in trace_node:
1171             raise ConfigError('missing "byte-order" property (trace)')
1172
1173         bo_node = trace_node['byte-order']
1174
1175         if not _is_str_prop(bo_node):
1176             raise ConfigError('"byte-order" property of trace object must be a string ("le" or "be")')
1177
1178         self._bo = _byte_order_str_to_bo(bo_node)
1179
1180         if self._bo is None:
1181             raise ConfigError('invalid "byte-order" property (trace): must be "le" or "be"')
1182
1183     def _lookup_type_alias(self, name):
1184         if name in self._tas:
1185             return copy.deepcopy(self._tas[name])
1186
1187     def _set_int_clock_prop_mapping(self, int_obj, prop_mapping_node):
1188         unk_prop = _get_first_unknown_prop(prop_mapping_node, ['type', 'name', 'property'])
1189
1190         if unk_prop:
1191             raise ConfigError('unknown property in integer type object\'s clock property mapping: "{}"'.format(unk_prop))
1192
1193         if 'name' not in prop_mapping_node:
1194             raise ConfigError('missing "name" property in integer type object\'s clock property mapping')
1195
1196         if 'property' not in prop_mapping_node:
1197             raise ConfigError('missing "property" property in integer type object\'s clock property mapping')
1198
1199         clock_name = prop_mapping_node['name']
1200         prop = prop_mapping_node['property']
1201
1202         if not _is_str_prop(clock_name):
1203             raise ConfigError('"name" property of integer type object\'s clock property mapping must be a string')
1204
1205         if not _is_str_prop(prop):
1206             raise ConfigError('"property" property of integer type object\'s clock property mapping must be a string')
1207
1208         if clock_name not in self._clocks:
1209             raise ConfigError('invalid clock name "{}" in integer type object\'s clock property mapping'.format(clock_name))
1210
1211         if prop != 'value':
1212             raise ConfigError('invalid "property" property in integer type object\'s clock property mapping: "{}"'.format(prop))
1213
1214         mapped_clock = self._clocks[clock_name]
1215         int_obj.property_mappings.append(metadata.PropertyMapping(mapped_clock, prop))
1216
1217     def _get_first_unknown_type_prop(self, type_node, known_props):
1218         kp = known_props + ['inherit', 'class']
1219
1220         if self._version >= 201:
1221             kp.append('$inherit')
1222
1223         return _get_first_unknown_prop(type_node, kp)
1224
1225     def _create_integer(self, obj, node):
1226         if obj is None:
1227             # create integer object
1228             obj = metadata.Integer()
1229
1230         unk_prop = self._get_first_unknown_type_prop(node, [
1231             'size',
1232             'align',
1233             'signed',
1234             'byte-order',
1235             'base',
1236             'encoding',
1237             'property-mappings',
1238         ])
1239
1240         if unk_prop:
1241             raise ConfigError('unknown integer type object property: "{}"'.format(unk_prop))
1242
1243         # size
1244         if 'size' in node:
1245             size = node['size']
1246
1247             if not _is_int_prop(size):
1248                 raise ConfigError('"size" property of integer type object must be an integer')
1249
1250             if size < 1:
1251                 raise ConfigError('invalid integer size: {}'.format(size))
1252
1253             obj.size = size
1254
1255         # align
1256         if 'align' in node:
1257             align = node['align']
1258
1259             if not _is_int_prop(align):
1260                 raise ConfigError('"align" property of integer type object must be an integer')
1261
1262             if not _is_valid_alignment(align):
1263                 raise ConfigError('invalid alignment: {}'.format(align))
1264
1265             obj.align = align
1266
1267         # signed
1268         if 'signed' in node:
1269             signed = node['signed']
1270
1271             if not _is_bool_prop(signed):
1272                 raise ConfigError('"signed" property of integer type object must be a boolean')
1273
1274             obj.signed = signed
1275
1276         # byte order
1277         if 'byte-order' in node:
1278             byte_order = node['byte-order']
1279
1280             if not _is_str_prop(byte_order):
1281                 raise ConfigError('"byte-order" property of integer type object must be a string ("le" or "be")')
1282
1283             byte_order = _byte_order_str_to_bo(byte_order)
1284
1285             if byte_order is None:
1286                 raise ConfigError('invalid "byte-order" property in integer type object')
1287         else:
1288             byte_order = self._bo
1289
1290         obj.byte_order = byte_order
1291
1292         # base
1293         if 'base' in node:
1294             base = node['base']
1295
1296             if not _is_str_prop(base):
1297                 raise ConfigError('"base" property of integer type object must be a string ("bin", "oct", "dec", or "hex")')
1298
1299             if base == 'bin':
1300                 base = 2
1301             elif base == 'oct':
1302                 base = 8
1303             elif base == 'dec':
1304                 base = 10
1305             elif base == 'hex':
1306                 base = 16
1307             else:
1308                 raise ConfigError('unknown "base" property value: "{}" ("bin", "oct", "dec", and "hex" are accepted)'.format(base))
1309
1310             obj.base = base
1311
1312         # encoding
1313         if 'encoding' in node:
1314             encoding = node['encoding']
1315
1316             if not _is_str_prop(encoding):
1317                 raise ConfigError('"encoding" property of integer type object must be a string ("none", "ascii", or "utf-8")')
1318
1319             encoding = _encoding_str_to_encoding(encoding)
1320
1321             if encoding is None:
1322                 raise ConfigError('invalid "encoding" property in integer type object')
1323
1324             obj.encoding = encoding
1325
1326         # property mappings
1327         if 'property-mappings' in node:
1328             prop_mappings = node['property-mappings']
1329
1330             if not _is_array_prop(prop_mappings):
1331                 raise ConfigError('"property-mappings" property of integer type object must be an array')
1332
1333             if len(prop_mappings) > 1:
1334                 raise ConfigError('length of "property-mappings" array in integer type object must be 1')
1335
1336             del obj.property_mappings[:]
1337
1338             for index, prop_mapping in enumerate(prop_mappings):
1339                 if not _is_assoc_array_prop(prop_mapping):
1340                     raise ConfigError('elements of "property-mappings" property of integer type object must be associative arrays')
1341
1342                 if 'type' not in prop_mapping:
1343                     raise ConfigError('missing "type" property in integer type object\'s "property-mappings" array\'s element #{}'.format(index))
1344
1345                 prop_type = prop_mapping['type']
1346
1347                 if not _is_str_prop(prop_type):
1348                     raise ConfigError('"type" property of integer type object\'s "property-mappings" array\'s element #{} must be a string'.format(index))
1349
1350                 if prop_type == 'clock':
1351                     self._set_int_clock_prop_mapping(obj, prop_mapping)
1352                 else:
1353                     raise ConfigError('unknown property mapping type "{}" in integer type object\'s "property-mappings" array\'s element #{}'.format(prop_type, index))
1354
1355         return obj
1356
1357     def _create_float(self, obj, node):
1358         if obj is None:
1359             # create floating point number object
1360             obj = metadata.FloatingPoint()
1361
1362         unk_prop = self._get_first_unknown_type_prop(node, [
1363             'size',
1364             'align',
1365             'byte-order',
1366         ])
1367
1368         if unk_prop:
1369             raise ConfigError('unknown floating point number type object property: "{}"'.format(unk_prop))
1370
1371         # size
1372         if 'size' in node:
1373             size = node['size']
1374
1375             if not _is_assoc_array_prop(size):
1376                 raise ConfigError('"size" property of floating point number type object must be an associative array')
1377
1378             unk_prop = _get_first_unknown_prop(size, ['exp', 'mant'])
1379
1380             if unk_prop:
1381                 raise ConfigError('unknown floating point number type object\'s "size" property: "{}"'.format(unk_prop))
1382
1383             if 'exp' in size:
1384                 exp = size['exp']
1385
1386                 if not _is_int_prop(exp):
1387                     raise ConfigError('"exp" property of floating point number type object\'s "size" property must be an integer')
1388
1389                 if exp < 1:
1390                     raise ConfigError('invalid floating point number exponent size: {}')
1391
1392                 obj.exp_size = exp
1393
1394             if 'mant' in size:
1395                 mant = size['mant']
1396
1397                 if not _is_int_prop(mant):
1398                     raise ConfigError('"mant" property of floating point number type object\'s "size" property must be an integer')
1399
1400                 if mant < 1:
1401                     raise ConfigError('invalid floating point number mantissa size: {}')
1402
1403                 obj.mant_size = mant
1404
1405         # align
1406         if 'align' in node:
1407             align = node['align']
1408
1409             if not _is_int_prop(align):
1410                 raise ConfigError('"align" property of floating point number type object must be an integer')
1411
1412             if not _is_valid_alignment(align):
1413                 raise ConfigError('invalid alignment: {}'.format(align))
1414
1415             obj.align = align
1416
1417         # byte order
1418         if 'byte-order' in node:
1419             byte_order = node['byte-order']
1420
1421             if not _is_str_prop(byte_order):
1422                 raise ConfigError('"byte-order" property of floating point number type object must be a string ("le" or "be")')
1423
1424             byte_order = _byte_order_str_to_bo(byte_order)
1425
1426             if byte_order is None:
1427                 raise ConfigError('invalid "byte-order" property in floating point number type object')
1428         else:
1429             byte_order = self._bo
1430
1431         obj.byte_order = byte_order
1432
1433         return obj
1434
1435     def _create_enum(self, obj, node):
1436         if obj is None:
1437             # create enumeration object
1438             obj = metadata.Enum()
1439
1440         unk_prop = self._get_first_unknown_type_prop(node, [
1441             'value-type',
1442             'members',
1443         ])
1444
1445         if unk_prop:
1446             raise ConfigError('unknown enumeration type object property: "{}"'.format(unk_prop))
1447
1448         # value type
1449         if 'value-type' in node:
1450             try:
1451                 obj.value_type = self._create_type(node['value-type'])
1452             except Exception as e:
1453                 raise ConfigError('cannot create enumeration type\'s integer type', e)
1454
1455         # members
1456         if 'members' in node:
1457             members_node = node['members']
1458
1459             if not _is_array_prop(members_node):
1460                 raise ConfigError('"members" property of enumeration type object must be an array')
1461
1462             cur = 0
1463
1464             for index, m_node in enumerate(members_node):
1465                 if not _is_str_prop(m_node) and not _is_assoc_array_prop(m_node):
1466                     raise ConfigError('invalid enumeration member #{}: expecting a string or an associative array'.format(index))
1467
1468                 if _is_str_prop(m_node):
1469                     label = m_node
1470                     value = (cur, cur)
1471                     cur += 1
1472                 else:
1473                     if 'label' not in m_node:
1474                         raise ConfigError('missing "label" property in enumeration member #{}'.format(index))
1475
1476                     label = m_node['label']
1477
1478                     if not _is_str_prop(label):
1479                         raise ConfigError('"label" property of enumeration member #{} must be a string'.format(index))
1480
1481                     if 'value' not in m_node:
1482                         raise ConfigError('missing "value" property in enumeration member ("{}")'.format(label))
1483
1484                     value = m_node['value']
1485
1486                     if not _is_int_prop(value) and not _is_array_prop(value):
1487                         raise ConfigError('invalid enumeration member ("{}"): expecting an integer or an array'.format(label))
1488
1489                     if _is_int_prop(value):
1490                         cur = value + 1
1491                         value = (value, value)
1492                     else:
1493                         if len(value) != 2:
1494                             raise ConfigError('invalid enumeration member ("{}"): range must have exactly two items'.format(label))
1495
1496                         mn = value[0]
1497                         mx = value[1]
1498
1499                         if mn > mx:
1500                             raise ConfigError('invalid enumeration member ("{}"): invalid range ({} > {})'.format(label, mn, mx))
1501
1502                         value = (mn, mx)
1503                         cur = mx + 1
1504
1505                 obj.members[label] = value
1506
1507         return obj
1508
1509     def _create_string(self, obj, node):
1510         if obj is None:
1511             # create string object
1512             obj = metadata.String()
1513
1514         unk_prop = self._get_first_unknown_type_prop(node, [
1515             'encoding',
1516         ])
1517
1518         if unk_prop:
1519             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1520
1521         # encoding
1522         if 'encoding' in node:
1523             encoding = node['encoding']
1524
1525             if not _is_str_prop(encoding):
1526                 raise ConfigError('"encoding" property of string type object must be a string ("none", "ascii", or "utf-8")')
1527
1528             encoding = _encoding_str_to_encoding(encoding)
1529
1530             if encoding is None:
1531                 raise ConfigError('invalid "encoding" property in string type object')
1532
1533             obj.encoding = encoding
1534
1535         return obj
1536
1537     def _create_struct(self, obj, node):
1538         if obj is None:
1539             # create structure object
1540             obj = metadata.Struct()
1541
1542         unk_prop = self._get_first_unknown_type_prop(node, [
1543             'min-align',
1544             'fields',
1545         ])
1546
1547         if unk_prop:
1548             raise ConfigError('unknown string type object property: "{}"'.format(unk_prop))
1549
1550         # minimum alignment
1551         if 'min-align' in node:
1552             min_align = node['min-align']
1553
1554             if not _is_int_prop(min_align):
1555                 raise ConfigError('"min-align" property of structure type object must be an integer')
1556
1557             if not _is_valid_alignment(min_align):
1558                 raise ConfigError('invalid minimum alignment: {}'.format(min_align))
1559
1560             obj.min_align = min_align
1561
1562         # fields
1563         if 'fields' in node:
1564             fields = node['fields']
1565
1566             if not _is_assoc_array_prop(fields):
1567                 raise ConfigError('"fields" property of structure type object must be an associative array')
1568
1569             for field_name, field_node in fields.items():
1570                 if not is_valid_identifier(field_name):
1571                     raise ConfigError('"{}" is not a valid field name for structure type'.format(field_name))
1572
1573                 try:
1574                     obj.fields[field_name] = self._create_type(field_node)
1575                 except Exception as e:
1576                     raise ConfigError('cannot create structure type\'s field "{}"'.format(field_name), e)
1577
1578         return obj
1579
1580     def _create_array(self, obj, node):
1581         if obj is None:
1582             # create array object
1583             obj = metadata.Array()
1584
1585         unk_prop = self._get_first_unknown_type_prop(node, [
1586             'length',
1587             'element-type',
1588         ])
1589
1590         if unk_prop:
1591             raise ConfigError('unknown array type object property: "{}"'.format(unk_prop))
1592
1593         # length
1594         if 'length' in node:
1595             length = node['length']
1596
1597             if not _is_int_prop(length) and not _is_str_prop(length):
1598                 raise ConfigError('"length" property of array type object must be an integer or a string')
1599
1600             if type(length) is int and length < 0:
1601                 raise ConfigError('invalid static array length: {}'.format(length))
1602
1603             obj.length = length
1604
1605         # element type
1606         if 'element-type' in node:
1607             try:
1608                 obj.element_type = self._create_type(node['element-type'])
1609             except Exception as e:
1610                 raise ConfigError('cannot create array type\'s element type', e)
1611
1612         return obj
1613
1614     def _create_variant(self, obj, node):
1615         if obj is None:
1616             # create variant object
1617             obj = metadata.Variant()
1618
1619         unk_prop = self._get_first_unknown_type_prop(node, [
1620             'tag',
1621             'types',
1622         ])
1623
1624         if unk_prop:
1625             raise ConfigError('unknown variant type object property: "{}"'.format(unk_prop))
1626
1627         # tag
1628         if 'tag' in node:
1629             tag = node['tag']
1630
1631             if not _is_str_prop(tag):
1632                 raise ConfigError('"tag" property of variant type object must be a string')
1633
1634             # do not validate variant tag for the moment; will be done in a
1635             # second phase
1636             obj.tag = tag
1637
1638         # element type
1639         if 'types' in node:
1640             types = node['types']
1641
1642             if not _is_assoc_array_prop(types):
1643                 raise ConfigError('"types" property of variant type object must be an associative array')
1644
1645             # do not validate type names for the moment; will be done in a
1646             # second phase
1647             for type_name, type_node in types.items():
1648                 if not is_valid_identifier(type_name):
1649                     raise ConfigError('"{}" is not a valid type name for variant type'.format(type_name))
1650
1651                 try:
1652                     obj.types[type_name] = self._create_type(type_node)
1653                 except Exception as e:
1654                     raise ConfigError('cannot create variant type\'s type "{}"'.format(type_name), e)
1655
1656         return obj
1657
1658     def _create_type(self, type_node):
1659         if type(type_node) is str:
1660             t = self._lookup_type_alias(type_node)
1661
1662             if t is None:
1663                 raise ConfigError('unknown type alias "{}"'.format(type_node))
1664
1665             return t
1666
1667         if not _is_assoc_array_prop(type_node):
1668             raise ConfigError('type objects must be associative arrays or strings (type alias name)')
1669
1670         # inherit:
1671         #   v2.0:  "inherit"
1672         #   v2.1+: "$inherit"
1673         inherit_node = None
1674
1675         if self._version >= 200:
1676             if 'inherit' in type_node:
1677                 inherit_prop = 'inherit'
1678                 inherit_node = type_node[inherit_prop]
1679
1680         if self._version >= 201:
1681             if '$inherit' in type_node:
1682                 if inherit_node is not None:
1683                     raise ConfigError('cannot specify both "inherit" and "$inherit" properties of type object: prefer "$inherit"')
1684
1685                 inherit_prop = '$inherit'
1686                 inherit_node = type_node[inherit_prop]
1687
1688         if inherit_node is not None and 'class' in type_node:
1689             raise ConfigError('cannot specify both "{}" and "class" properties in type object'.format(inherit_prop))
1690
1691         if inherit_node is not None:
1692             if not _is_str_prop(inherit_node):
1693                 raise ConfigError('"{}" property of type object must be a string'.format(inherit_prop))
1694
1695             base = self._lookup_type_alias(inherit_node)
1696
1697             if base is None:
1698                 raise ConfigError('cannot inherit from type alias "{}": type alias does not exist at this point'.format(inherit_node))
1699
1700             func = self._type_to_create_type_func[type(base)]
1701         else:
1702             if 'class' not in type_node:
1703                 raise ConfigError('type objects which do not inherit must have a "class" property')
1704
1705             class_name = type_node['class']
1706
1707             if type(class_name) is not str:
1708                 raise ConfigError('type objects\' "class" property must be a string')
1709
1710             if class_name not in self._class_name_to_create_type_func:
1711                 raise ConfigError('unknown type class "{}"'.format(class_name))
1712
1713             base = None
1714             func = self._class_name_to_create_type_func[class_name]
1715
1716         return func(base, type_node)
1717
1718     def _register_type_aliases(self, metadata_node):
1719         self._tas = dict()
1720
1721         if 'type-aliases' not in metadata_node:
1722             return
1723
1724         ta_node = metadata_node['type-aliases']
1725
1726         if not _is_assoc_array_prop(ta_node):
1727             raise ConfigError('"type-aliases" property (metadata) must be an associative array')
1728
1729         for ta_name, ta_type in ta_node.items():
1730             if ta_name in self._tas:
1731                 raise ConfigError('duplicate type alias "{}"'.format(ta_name))
1732
1733             try:
1734                 t = self._create_type(ta_type)
1735             except Exception as e:
1736                 raise ConfigError('cannot create type alias "{}"'.format(ta_name), e)
1737
1738             self._tas[ta_name] = t
1739
1740     def _create_clock(self, node):
1741         # create clock object
1742         clock = metadata.Clock()
1743
1744         if not _is_assoc_array_prop(node):
1745             raise ConfigError('clock objects must be associative arrays')
1746
1747         known_props = [
1748             'uuid',
1749             'description',
1750             'freq',
1751             'error-cycles',
1752             'offset',
1753             'absolute',
1754             'return-ctype',
1755         ]
1756
1757         if self._version >= 201:
1758             known_props.append('$return-ctype')
1759
1760         unk_prop = _get_first_unknown_prop(node, known_props)
1761
1762         if unk_prop:
1763             raise ConfigError('unknown clock object property: "{}"'.format(unk_prop))
1764
1765         # UUID
1766         if 'uuid' in node:
1767             uuidp = node['uuid']
1768
1769             if not _is_str_prop(uuidp):
1770                 raise ConfigError('"uuid" property of clock object must be a string')
1771
1772             try:
1773                 uuidp = uuid.UUID(uuidp)
1774             except:
1775                 raise ConfigError('malformed UUID (clock object): "{}"'.format(uuidp))
1776
1777             clock.uuid = uuidp
1778
1779         # description
1780         if 'description' in node:
1781             desc = node['description']
1782
1783             if not _is_str_prop(desc):
1784                 raise ConfigError('"description" property of clock object must be a string')
1785
1786             clock.description = desc
1787
1788         # frequency
1789         if 'freq' in node:
1790             freq = node['freq']
1791
1792             if not _is_int_prop(freq):
1793                 raise ConfigError('"freq" property of clock object must be an integer')
1794
1795             if freq < 1:
1796                 raise ConfigError('invalid clock frequency: {}'.format(freq))
1797
1798             clock.freq = freq
1799
1800         # error cycles
1801         if 'error-cycles' in node:
1802             error_cycles = node['error-cycles']
1803
1804             if not _is_int_prop(error_cycles):
1805                 raise ConfigError('"error-cycles" property of clock object must be an integer')
1806
1807             if error_cycles < 0:
1808                 raise ConfigError('invalid clock error cycles: {}'.format(error_cycles))
1809
1810             clock.error_cycles = error_cycles
1811
1812         # offset
1813         if 'offset' in node:
1814             offset = node['offset']
1815
1816             if not _is_assoc_array_prop(offset):
1817                 raise ConfigError('"offset" property of clock object must be an associative array')
1818
1819             unk_prop = _get_first_unknown_prop(offset, ['cycles', 'seconds'])
1820
1821             if unk_prop:
1822                 raise ConfigError('unknown clock object\'s offset property: "{}"'.format(unk_prop))
1823
1824             # cycles
1825             if 'cycles' in offset:
1826                 offset_cycles = offset['cycles']
1827
1828                 if not _is_int_prop(offset_cycles):
1829                     raise ConfigError('"cycles" property of clock object\'s offset property must be an integer')
1830
1831                 if offset_cycles < 0:
1832                     raise ConfigError('invalid clock offset cycles: {}'.format(offset_cycles))
1833
1834                 clock.offset_cycles = offset_cycles
1835
1836             # seconds
1837             if 'seconds' in offset:
1838                 offset_seconds = offset['seconds']
1839
1840                 if not _is_int_prop(offset_seconds):
1841                     raise ConfigError('"seconds" property of clock object\'s offset property must be an integer')
1842
1843                 if offset_seconds < 0:
1844                     raise ConfigError('invalid clock offset seconds: {}'.format(offset_seconds))
1845
1846                 clock.offset_seconds = offset_seconds
1847
1848         # absolute
1849         if 'absolute' in node:
1850             absolute = node['absolute']
1851
1852             if not _is_bool_prop(absolute):
1853                 raise ConfigError('"absolute" property of clock object must be a boolean')
1854
1855             clock.absolute = absolute
1856
1857         # return C type:
1858         #   v2.0:  "return-ctype"
1859         #   v2.1+: "$return-ctype"
1860         return_ctype_node = None
1861
1862         if self._version >= 200:
1863             if 'return-ctype' in node:
1864                 return_ctype_prop = 'return-ctype'
1865                 return_ctype_node = node[return_ctype_prop]
1866
1867         if self._version >= 201:
1868             if '$return-ctype' in node:
1869                 if return_ctype_node is not None:
1870                     raise ConfigError('cannot specify both "return-ctype" and "$return-ctype" properties of clock object: prefer "$return-ctype"')
1871
1872                 return_ctype_prop = '$return-ctype'
1873                 return_ctype_node = node[return_ctype_prop]
1874
1875         if return_ctype_node is not None:
1876             if not _is_str_prop(return_ctype_node):
1877                 raise ConfigError('"{}" property of clock object must be a string'.format(return_ctype_prop))
1878
1879             clock.return_ctype = return_ctype_node
1880
1881         return clock
1882
1883     def _register_clocks(self, metadata_node):
1884         self._clocks = collections.OrderedDict()
1885
1886         if 'clocks' not in metadata_node:
1887             return
1888
1889         clocks_node = metadata_node['clocks']
1890
1891         if not _is_assoc_array_prop(clocks_node):
1892             raise ConfigError('"clocks" property (metadata) must be an associative array')
1893
1894         for clock_name, clock_node in clocks_node.items():
1895             if not is_valid_identifier(clock_name):
1896                 raise ConfigError('invalid clock name: "{}"'.format(clock_name))
1897
1898             if clock_name in self._clocks:
1899                 raise ConfigError('duplicate clock "{}"'.format(clock_name))
1900
1901             try:
1902                 clock = self._create_clock(clock_node)
1903             except Exception as e:
1904                 raise ConfigError('cannot create clock "{}"'.format(clock_name), e)
1905
1906             clock.name = clock_name
1907             self._clocks[clock_name] = clock
1908
1909     def _create_env(self, metadata_node):
1910         env = collections.OrderedDict()
1911
1912         if 'env' not in metadata_node:
1913             return env
1914
1915         env_node = metadata_node['env']
1916
1917         if not _is_assoc_array_prop(env_node):
1918             raise ConfigError('"env" property (metadata) must be an associative array')
1919
1920         for env_name, env_value in env_node.items():
1921             if env_name in env:
1922                 raise ConfigError('duplicate environment variable "{}"'.format(env_name))
1923
1924             if not is_valid_identifier(env_name):
1925                 raise ConfigError('invalid environment variable name: "{}"'.format(env_name))
1926
1927             if not _is_int_prop(env_value) and not _is_str_prop(env_value):
1928                 raise ConfigError('invalid environment variable value ("{}"): expecting integer or string'.format(env_name))
1929
1930             env[env_name] = env_value
1931
1932         return env
1933
1934     def _register_log_levels(self, metadata_node):
1935         self._log_levels = dict()
1936
1937         # log levels:
1938         #   v2.0:  "log-levels"
1939         #   v2.1+: "$log-levels"
1940         log_levels_node = None
1941
1942         if self._version >= 200:
1943             if 'log-levels' in metadata_node:
1944                 log_levels_prop = 'log-levels'
1945                 log_levels_node = metadata_node[log_levels_prop]
1946
1947         if self._version >= 201:
1948             if '$log-levels' in metadata_node:
1949                 if log_levels_node is not None:
1950                     raise ConfigError('cannot specify both "log-levels" and "$log-levels" properties of metadata object: prefer "$log-levels"')
1951
1952                 log_levels_prop = '$log-levels'
1953                 log_levels_node = metadata_node[log_levels_prop]
1954
1955         if log_levels_node is None:
1956             return
1957
1958         if not _is_assoc_array_prop(log_levels_node):
1959             raise ConfigError('"{}" property (metadata) must be an associative array'.format(log_levels_prop))
1960
1961         for ll_name, ll_value in log_levels_node.items():
1962             if ll_name in self._log_levels:
1963                 raise ConfigError('duplicate log level entry "{}"'.format(ll_name))
1964
1965             if not _is_int_prop(ll_value):
1966                 raise ConfigError('invalid log level entry ("{}"): expecting an integer'.format(ll_name))
1967
1968             if ll_value < 0:
1969                 raise ConfigError('invalid log level entry ("{}"): log level value must be positive'.format(ll_name))
1970
1971             self._log_levels[ll_name] = ll_value
1972
1973     def _create_trace(self, metadata_node):
1974         # create trace object
1975         trace = metadata.Trace()
1976
1977         if 'trace' not in metadata_node:
1978             raise ConfigError('missing "trace" property (metadata)')
1979
1980         trace_node = metadata_node['trace']
1981
1982         if not _is_assoc_array_prop(trace_node):
1983             raise ConfigError('"trace" property (metadata) must be an associative array')
1984
1985         unk_prop = _get_first_unknown_prop(trace_node, [
1986             'byte-order',
1987             'uuid',
1988             'packet-header-type',
1989         ])
1990
1991         if unk_prop:
1992             raise ConfigError('unknown trace object property: "{}"'.format(unk_prop))
1993
1994         # set byte order (already parsed)
1995         trace.byte_order = self._bo
1996
1997         # UUID
1998         if 'uuid' in trace_node:
1999             uuidp = trace_node['uuid']
2000
2001             if not _is_str_prop(uuidp):
2002                 raise ConfigError('"uuid" property of trace object must be a string')
2003
2004             if uuidp == 'auto':
2005                 uuidp = uuid.uuid1()
2006             else:
2007                 try:
2008                     uuidp = uuid.UUID(uuidp)
2009                 except:
2010                     raise ConfigError('malformed UUID (trace object): "{}"'.format(uuidp))
2011
2012             trace.uuid = uuidp
2013
2014         # packet header type
2015         if 'packet-header-type' in trace_node:
2016             try:
2017                 ph_type = self._create_type(trace_node['packet-header-type'])
2018             except Exception as e:
2019                 raise ConfigError('cannot create packet header type (trace)', e)
2020
2021             trace.packet_header_type = ph_type
2022
2023         return trace
2024
2025     def _lookup_log_level(self, ll):
2026         if _is_int_prop(ll):
2027             return ll
2028         elif _is_str_prop(ll) and ll in self._log_levels:
2029             return self._log_levels[ll]
2030
2031     def _create_event(self, event_node):
2032         event = metadata.Event()
2033
2034         if not _is_assoc_array_prop(event_node):
2035             raise ConfigError('event objects must be associative arrays')
2036
2037         unk_prop = _get_first_unknown_prop(event_node, [
2038             'log-level',
2039             'context-type',
2040             'payload-type',
2041         ])
2042
2043         if unk_prop:
2044             raise ConfigError('unknown event object property: "{}"'.format(unk_prop))
2045
2046         if 'log-level' in event_node:
2047             ll_node = event_node['log-level']
2048
2049             if _is_str_prop(ll_node):
2050                 ll = self._lookup_log_level(event_node['log-level'])
2051
2052                 if ll is None:
2053                     raise ConfigError('cannot find log level "{}"'.format(ll_node))
2054             elif _is_int_prop(ll_node):
2055                 if ll_node < 0:
2056                     raise ConfigError('invalid log level value {}: value must be positive'.format(ll_node))
2057
2058                 ll = ll_node
2059             else:
2060                 raise ConfigError('"log-level" property must be either a string or an integer')
2061
2062             event.log_level = ll
2063
2064         if 'context-type' in event_node:
2065             try:
2066                 t = self._create_type(event_node['context-type'])
2067             except Exception as e:
2068                 raise ConfigError('cannot create event\'s context type object', e)
2069
2070             event.context_type = t
2071
2072         if 'payload-type' not in event_node:
2073             raise ConfigError('missing "payload-type" property in event object')
2074
2075         try:
2076             t = self._create_type(event_node['payload-type'])
2077         except Exception as e:
2078             raise ConfigError('cannot create event\'s payload type object', e)
2079
2080         event.payload_type = t
2081
2082         return event
2083
2084     def _create_stream(self, stream_node):
2085         stream = metadata.Stream()
2086
2087         if not _is_assoc_array_prop(stream_node):
2088             raise ConfigError('stream objects must be associative arrays')
2089
2090         unk_prop = _get_first_unknown_prop(stream_node, [
2091             'packet-context-type',
2092             'event-header-type',
2093             'event-context-type',
2094             'events',
2095         ])
2096
2097         if unk_prop:
2098             raise ConfigError('unknown stream object property: "{}"'.format(unk_prop))
2099
2100         if 'packet-context-type' in stream_node:
2101             try:
2102                 t = self._create_type(stream_node['packet-context-type'])
2103             except Exception as e:
2104                 raise ConfigError('cannot create stream\'s packet context type object', e)
2105
2106             stream.packet_context_type = t
2107
2108         if 'event-header-type' in stream_node:
2109             try:
2110                 t = self._create_type(stream_node['event-header-type'])
2111             except Exception as e:
2112                 raise ConfigError('cannot create stream\'s event header type object', e)
2113
2114             stream.event_header_type = t
2115
2116         if 'event-context-type' in stream_node:
2117             try:
2118                 t = self._create_type(stream_node['event-context-type'])
2119             except Exception as e:
2120                 raise ConfigError('cannot create stream\'s event context type object', e)
2121
2122             stream.event_context_type = t
2123
2124         if 'events' not in stream_node:
2125             raise ConfigError('missing "events" property in stream object')
2126
2127         events = stream_node['events']
2128
2129         if not _is_assoc_array_prop(events):
2130             raise ConfigError('"events" property of stream object must be an associative array')
2131
2132         if not events:
2133             raise ConfigError('at least one event is needed within a stream object')
2134
2135         cur_id = 0
2136
2137         for ev_name, ev_node in events.items():
2138             try:
2139                 ev = self._create_event(ev_node)
2140             except Exception as e:
2141                 raise ConfigError('cannot create event "{}"'.format(ev_name), e)
2142
2143             ev.id = cur_id
2144             ev.name = ev_name
2145             stream.events[ev_name] = ev
2146             cur_id += 1
2147
2148         return stream
2149
2150     def _create_streams(self, metadata_node):
2151         streams = collections.OrderedDict()
2152
2153         if 'streams' not in metadata_node:
2154             raise ConfigError('missing "streams" property (metadata)')
2155
2156         streams_node = metadata_node['streams']
2157
2158         if not _is_assoc_array_prop(streams_node):
2159             raise ConfigError('"streams" property (metadata) must be an associative array')
2160
2161         if not streams_node:
2162             raise ConfigError('at least one stream is needed (metadata)')
2163
2164         cur_id = 0
2165
2166         for stream_name, stream_node in streams_node.items():
2167             try:
2168                 stream = self._create_stream(stream_node)
2169             except Exception as e:
2170                 raise ConfigError('cannot create stream "{}"'.format(stream_name), e)
2171
2172             stream.id = cur_id
2173             stream.name = str(stream_name)
2174             streams[stream_name] = stream
2175             cur_id += 1
2176
2177         return streams
2178
2179     def _create_metadata(self, root):
2180         meta = metadata.Metadata()
2181
2182         if 'metadata' not in root:
2183             raise ConfigError('missing "metadata" property (configuration)')
2184
2185         metadata_node = root['metadata']
2186
2187         if not _is_assoc_array_prop(metadata_node):
2188             raise ConfigError('"metadata" property (configuration) must be an associative array')
2189
2190         known_props = [
2191             'type-aliases',
2192             'log-levels',
2193             'trace',
2194             'env',
2195             'clocks',
2196             'streams',
2197         ]
2198
2199         if self._version >= 201:
2200             known_props.append('$log-levels')
2201
2202         unk_prop = _get_first_unknown_prop(metadata_node, known_props)
2203
2204         if unk_prop:
2205             add = ''
2206
2207             if unk_prop == '$include':
2208                 add = ' (use version 2.1 or greater)'
2209
2210             raise ConfigError('unknown metadata property{}: "{}"'.format(add, unk_prop))
2211
2212         self._set_byte_order(metadata_node)
2213         self._register_clocks(metadata_node)
2214         meta.clocks = self._clocks
2215         self._register_type_aliases(metadata_node)
2216         meta.env = self._create_env(metadata_node)
2217         meta.trace = self._create_trace(metadata_node)
2218         self._register_log_levels(metadata_node)
2219         meta.streams = self._create_streams(metadata_node)
2220
2221         return meta
2222
2223     def _get_version(self, root):
2224         if 'version' not in root:
2225             raise ConfigError('missing "version" property (configuration)')
2226
2227         version_node = root['version']
2228
2229         if not _is_str_prop(version_node):
2230             raise ConfigError('"version" property (configuration) must be a string')
2231
2232         version_node = version_node.strip()
2233
2234         if version_node not in ['2.0', '2.1']:
2235             raise ConfigError('unsupported version ({}): versions 2.0 and 2.1 are supported'.format(version_node))
2236
2237         # convert version string to comparable version integer
2238         parts = version_node.split('.')
2239         version = int(parts[0]) * 100 + int(parts[1])
2240
2241         return version
2242
2243     def _get_prefix(self, root):
2244         if 'prefix' not in root:
2245             return 'barectf_'
2246
2247         prefix_node = root['prefix']
2248
2249         if not _is_str_prop(prefix_node):
2250             raise ConfigError('"prefix" property (configuration) must be a string')
2251
2252         if not is_valid_identifier(prefix_node):
2253             raise ConfigError('"prefix" property (configuration) must be a valid C identifier')
2254
2255         return prefix_node
2256
2257     def _get_last_include_file(self):
2258         if self._include_stack:
2259             return self._include_stack[-1]
2260
2261         return self._root_yaml_path
2262
2263     def _load_include(self, yaml_path):
2264         for inc_dir in self._include_dirs:
2265             # current include dir + file name path
2266             # note: os.path.join() only takes the last arg if it's absolute
2267             inc_path = os.path.join(inc_dir, yaml_path)
2268
2269             # real path (symbolic links resolved)
2270             real_path = os.path.realpath(inc_path)
2271
2272             # normalized path (weird stuff removed!)
2273             norm_path = os.path.normpath(real_path)
2274
2275             if not os.path.isfile(norm_path):
2276                 # file does not exist: skip
2277                 continue
2278
2279             if norm_path in self._include_stack:
2280                 base_path = self._get_last_include_file()
2281                 raise ConfigError('in "{}": cannot recursively include file "{}"'.format(base_path, norm_path))
2282
2283             self._include_stack.append(norm_path)
2284
2285             # load raw content
2286             return self._yaml_ordered_load(norm_path)
2287
2288         if not self._ignore_include_not_found:
2289             base_path = self._get_last_include_file()
2290             raise ConfigError('in "{}": cannot include file "{}": file not found in include directories'.format(base_path, yaml_path))
2291
2292         return None
2293
2294     def _get_include_paths(self, include_node):
2295         if _is_str_prop(include_node):
2296             return [include_node]
2297         elif _is_array_prop(include_node):
2298             for include_path in include_node:
2299                 if not _is_str_prop(include_path):
2300                     raise ConfigError('invalid include property: expecting array of strings')
2301
2302             return include_node
2303
2304         raise ConfigError('invalid include property: expecting string or array of strings')
2305
2306     def _update_node(self, base_node, overlay_node):
2307         for olay_key, olay_value in overlay_node.items():
2308             if olay_key in base_node:
2309                 base_value = base_node[olay_key]
2310
2311                 if _is_assoc_array_prop(olay_value) and _is_assoc_array_prop(base_value):
2312                     # merge dictionaries
2313                     self._update_node(base_value, olay_value)
2314                 elif _is_array_prop(olay_value) and _is_array_prop(base_value):
2315                     # append extension array items to base items
2316                     base_value += olay_value
2317                 else:
2318                     # fall back to replacing
2319                     base_node[olay_key] = olay_value
2320             else:
2321                 base_node[olay_key] = olay_value
2322
2323     def _process_node_include(self, last_overlay_node, name,
2324                               process_base_include_cb,
2325                               process_children_include_cb=None):
2326         if not _is_assoc_array_prop(last_overlay_node):
2327             raise ConfigError('{} objects must be associative arrays'.format(name))
2328
2329         # process children inclusions first
2330         if process_children_include_cb:
2331             process_children_include_cb(last_overlay_node)
2332
2333         if '$include' in last_overlay_node:
2334             include_node = last_overlay_node['$include']
2335         else:
2336             # no includes!
2337             return last_overlay_node
2338
2339         include_paths = self._get_include_paths(include_node)
2340         cur_base_path = self._get_last_include_file()
2341         base_node = None
2342
2343         # keep the include paths and remove the include property
2344         include_paths = copy.deepcopy(include_paths)
2345         del last_overlay_node['$include']
2346
2347         for include_path in include_paths:
2348             # load raw YAML from included file
2349             overlay_node = self._load_include(include_path)
2350
2351             if overlay_node is None:
2352                 # cannot find include file, but we're ignoring those
2353                 # errors, otherwise _load_include() itself raises
2354                 # a config error
2355                 continue
2356
2357             # recursively process includes
2358             try:
2359                 overlay_node = process_base_include_cb(overlay_node)
2360             except Exception as e:
2361                 raise ConfigError('in "{}"'.format(cur_base_path), e)
2362
2363             # pop include stack now that we're done including
2364             del self._include_stack[-1]
2365
2366             # at this point, base_node is fully resolved (does not
2367             # contain any include property)
2368             if base_node is None:
2369                 base_node = overlay_node
2370             else:
2371                 self._update_node(base_node, overlay_node)
2372
2373         # finally, we update the latest base node with our last overlay
2374         # node
2375         if base_node is None:
2376             # nothing was included, which is possible when we're
2377             # ignoring include errors
2378             return last_overlay_node
2379
2380         self._update_node(base_node, last_overlay_node)
2381
2382         return base_node
2383
2384     def _process_event_include(self, event_node):
2385         return self._process_node_include(event_node, 'event',
2386                                           self._process_event_include)
2387
2388     def _process_stream_include(self, stream_node):
2389         def process_children_include(stream_node):
2390             if 'events' in stream_node:
2391                 events_node = stream_node['events']
2392
2393                 if not _is_assoc_array_prop(events_node):
2394                     raise ConfigError('"events" property must be an associative array')
2395
2396                 events_node_keys = list(events_node.keys())
2397
2398                 for key in events_node_keys:
2399                     event_node = events_node[key]
2400
2401                     try:
2402                         events_node[key] = self._process_event_include(event_node)
2403                     except Exception as e:
2404                         raise ConfigError('cannot process includes of event object "{}"'.format(key), e)
2405
2406         return self._process_node_include(stream_node, 'stream',
2407                                           self._process_stream_include,
2408                                           process_children_include)
2409
2410     def _process_trace_include(self, trace_node):
2411         return self._process_node_include(trace_node, 'trace',
2412                                           self._process_trace_include)
2413
2414     def _process_clock_include(self, clock_node):
2415         return self._process_node_include(clock_node, 'clock',
2416                                           self._process_clock_include)
2417
2418     def _process_metadata_include(self, metadata_node):
2419         def process_children_include(metadata_node):
2420             if 'trace' in metadata_node:
2421                 metadata_node['trace'] = self._process_trace_include(metadata_node['trace'])
2422
2423             if 'clocks' in metadata_node:
2424                 clocks_node = metadata_node['clocks']
2425
2426                 if not _is_assoc_array_prop(clocks_node):
2427                     raise ConfigError('"clocks" property (metadata) must be an associative array')
2428
2429                 clocks_node_keys = list(clocks_node.keys())
2430
2431                 for key in clocks_node_keys:
2432                     clock_node = clocks_node[key]
2433
2434                     try:
2435                         clocks_node[key] = self._process_clock_include(clock_node)
2436                     except Exception as e:
2437                         raise ConfigError('cannot process includes of clock object "{}"'.format(key), e)
2438
2439             if 'streams' in metadata_node:
2440                 streams_node = metadata_node['streams']
2441
2442                 if not _is_assoc_array_prop(streams_node):
2443                     raise ConfigError('"streams" property (metadata) must be an associative array')
2444
2445                 streams_node_keys = list(streams_node.keys())
2446
2447                 for key in streams_node_keys:
2448                     stream_node = streams_node[key]
2449
2450                     try:
2451                         streams_node[key] = self._process_stream_include(stream_node)
2452                     except Exception as e:
2453                         raise ConfigError('cannot process includes of stream object "{}"'.format(key), e)
2454
2455         return self._process_node_include(metadata_node, 'metadata',
2456                                           self._process_metadata_include,
2457                                           process_children_include)
2458
2459     def _process_root_includes(self, root):
2460         # The following config objects support includes:
2461         #
2462         #   * Metadata object
2463         #   * Trace object
2464         #   * Stream object
2465         #   * Event object
2466         #
2467         # We need to process the event includes first, then the stream
2468         # includes, then the trace includes, and finally the metadata
2469         # includes.
2470         #
2471         # In each object, only one of the $include and $include-replace
2472         # special properties is allowed.
2473         #
2474         # We keep a stack of absolute paths to included files to detect
2475         # recursion.
2476         if 'metadata' in root:
2477             root['metadata'] = self._process_metadata_include(root['metadata'])
2478
2479         return root
2480
2481     def _yaml_ordered_dump(self, node, **kwds):
2482         class ODumper(yaml.Dumper):
2483             pass
2484
2485         def dict_representer(dumper, node):
2486             return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2487                                             node.items())
2488
2489         ODumper.add_representer(collections.OrderedDict, dict_representer)
2490
2491         return yaml.dump(node, Dumper=ODumper, **kwds)
2492
2493     def _yaml_ordered_load(self, yaml_path):
2494         class OLoader(yaml.Loader):
2495             pass
2496
2497         def construct_mapping(loader, node):
2498             loader.flatten_mapping(node)
2499
2500             return collections.OrderedDict(loader.construct_pairs(node))
2501
2502         OLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
2503                                 construct_mapping)
2504
2505         # YAML -> Python
2506         try:
2507             with open(yaml_path, 'r') as f:
2508                 node = yaml.load(f, OLoader)
2509         except (OSError, IOError) as e:
2510             raise ConfigError('cannot open file "{}"'.format(yaml_path))
2511         except Exception as e:
2512             raise ConfigError('unknown error while trying to load file "{}"'.format(yaml_path), e)
2513
2514         # loaded node must be an associate array
2515         if not _is_assoc_array_prop(node):
2516             raise ConfigError('root of YAML file "{}" must be an associative array'.format(yaml_path))
2517
2518         return node
2519
2520     def _reset(self):
2521         self._version = None
2522         self._include_stack = []
2523
2524     def parse(self, yaml_path):
2525         self._reset()
2526         self._root_yaml_path = yaml_path
2527
2528         try:
2529             root = self._yaml_ordered_load(yaml_path)
2530         except Exception as e:
2531             raise ConfigError('cannot parse YAML file "{}"'.format(yaml_path), e)
2532
2533         if not _is_assoc_array_prop(root):
2534             raise ConfigError('configuration must be an associative array')
2535
2536         unk_prop = _get_first_unknown_prop(root, [
2537             'version',
2538             'prefix',
2539             'metadata',
2540         ])
2541
2542         if unk_prop:
2543             raise ConfigError('unknown configuration property: "{}"'.format(unk_prop))
2544
2545         # get the config version
2546         self._version = self._get_version(root)
2547
2548         # process includes if supported
2549         if self._version >= 201:
2550             root = self._process_root_includes(root)
2551
2552         # dump config if required
2553         if self._dump_config:
2554             print(self._yaml_ordered_dump(root, indent=2,
2555                                           default_flow_style=False))
2556
2557         # get prefix and metadata
2558         prefix = self._get_prefix(root)
2559         meta = self._create_metadata(root)
2560
2561         return Config(self._version, prefix, meta)
2562
2563
2564 def from_yaml_file(path, include_dirs, ignore_include_not_found, dump_config):
2565     try:
2566         parser = _YamlConfigParser(include_dirs, ignore_include_not_found,
2567                                    dump_config)
2568         cfg = parser.parse(path)
2569
2570         return cfg
2571     except Exception as e:
2572         raise ConfigError('cannot create configuration from YAML file "{}"'.format(path), e)