1 # The MIT License (MIT)
3 # Copyright (c) 2015-2020 Philippe Proulx <pproulx@efficios.com>
5 # Permission is hereby granted, free of charge, to any person obtaining
6 # a copy of this software and associated documeneffective_filetation files (the
7 # "Software"), to deal in the Software without restriction, including
8 # without limitation the rights to use, copy, modify, merge, publish,
9 # distribute, sublicense, and/or sell copies of the Software, and to
10 # permit persons to whom the Software is furnished to do so, subject to
11 # the following conditions:
13 # The above copyright notice and this permission notice shall be
14 # included in all copies or substantial portions of the Software.
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19 # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20 # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
33 # The context of a configuration parsing error.
35 # Such a context object has a name and, optionally, a message.
36 class _ConfigurationParseErrorContext
:
37 def __init__(self
, name
, message
=None):
50 # Appends the context having the object name `obj_name` and the
51 # (optional) message `message` to the `_ConfigurationParseError`
52 # exception `exc` and then raises `exc` again.
53 def _append_error_ctx(exc
, obj_name
, message
=None):
54 exc
._append
_ctx
(obj_name
, message
)
58 # A configuration parsing error.
60 # Such an error object contains a list of contexts (`context` property).
62 # The first context of this list is the most specific context, while the
63 # last is the more general.
65 # Use _append_ctx() to append a context to an existing configuration
66 # parsing error when you catch it before raising it again. You can use
67 # _append_error_ctx() to do exactly this in a single call.
68 class _ConfigurationParseError(Exception):
69 def __init__(self
, init_ctx_obj_name
, init_ctx_msg
=None):
72 self
._append
_ctx
(init_ctx_obj_name
, init_ctx_msg
)
78 def _append_ctx(self
, name
, msg
=None):
79 self
._ctx
.append(_ConfigurationParseErrorContext(name
, msg
))
84 for ctx
in reversed(self
._ctx
):
87 if ctx
.message
is not None:
88 line
+= f
' {ctx.message}'
92 return '\n'.join(lines
)
95 _V3Prefixes
= collections
.namedtuple('_V3Prefixes', ['identifier', 'file_name'])
98 # Convers a v2 prefix to v3 prefixes.
99 def _v3_prefixes_from_v2_prefix(v2_prefix
):
100 return _V3Prefixes(v2_prefix
, v2_prefix
.rstrip('_'))
103 # This JSON schema reference resolver only serves to detect when it
104 # needs to resolve a remote URI.
106 # This must never happen in barectf because all our schemas are local;
107 # it would mean a programming or schema error.
108 class _RefResolver(jsonschema
.RefResolver
):
109 def resolve_remote(self
, uri
):
110 raise RuntimeError(f
'Missing local schema with URI `{uri}`')
113 # Schema validator which considers all the schemas found in the
114 # subdirectories `subdirs` (at build time) of the barectf package's
115 # `schemas` directory.
117 # The only public method is validate() which accepts an instance to
118 # validate as well as a schema short ID.
119 class _SchemaValidator
:
120 def __init__(self
, subdirs
):
121 schemas_dir
= pkg_resources
.resource_filename(__name__
, 'schemas')
124 for subdir
in subdirs
:
125 dir = os
.path
.join(schemas_dir
, subdir
)
127 for file_name
in os
.listdir(dir):
128 if not file_name
.endswith('.yaml'):
131 with
open(os
.path
.join(dir, file_name
)) as f
:
132 schema
= yaml
.load(f
, Loader
=yaml
.SafeLoader
)
134 assert '$id' in schema
135 schema_id
= schema
['$id']
136 assert schema_id
not in self
._store
137 self
._store
[schema_id
] = schema
140 def _dict_from_ordered_dict(obj
):
141 if type(obj
) is not collections
.OrderedDict
:
146 for k
, v
in obj
.items():
149 if type(v
) is collections
.OrderedDict
:
150 new_v
= _SchemaValidator
._dict
_from
_ordered
_dict
(v
)
151 elif type(v
) is list:
152 new_v
= [_SchemaValidator
._dict
_from
_ordered
_dict
(elem
) for elem
in v
]
158 def _validate(self
, instance
, schema_short_id
):
159 # retrieve full schema ID from short ID
160 schema_id
= f
'https://barectf.org/schemas/{schema_short_id}.json'
161 assert schema_id
in self
._store
163 # retrieve full schema
164 schema
= self
._store
[schema_id
]
166 # Create a reference resolver for this schema using this
167 # validator's schema store.
168 resolver
= _RefResolver(base_uri
=schema_id
, referrer
=schema
,
171 # create a JSON schema validator using this reference resolver
172 validator
= jsonschema
.Draft7Validator(schema
, resolver
=resolver
)
174 # Validate the instance, converting its
175 # `collections.OrderedDict` objects to `dict` objects so as to
176 # make any error message easier to read (because
177 # validator.validate() below uses str() for error messages, and
178 # collections.OrderedDict.__str__() returns a somewhat bulky
180 validator
.validate(self
._dict
_from
_ordered
_dict
(instance
))
182 # Validates `instance` using the schema having the short ID
185 # A schema short ID is the part between `schemas/` and `.json` in
188 # Raises a `_ConfigurationParseError` object, hiding any
189 # `jsonschema` exception, on validation failure.
190 def validate(self
, instance
, schema_short_id
):
192 self
._validate
(instance
, schema_short_id
)
193 except jsonschema
.ValidationError
as exc
:
194 # convert to barectf `_ConfigurationParseError` exception
195 contexts
= ['Configuration object']
197 # Each element of the instance's absolute path is either an
198 # integer (array element's index) or a string (object
200 for elem
in exc
.absolute_path
:
201 if type(elem
) is int:
202 ctx
= f
'Element #{elem + 1}'
204 ctx
= f
'`{elem}` property'
210 if len(exc
.context
) > 0:
211 # According to the documentation of
212 # jsonschema.ValidationError.context(), the method
215 # > list of errors from the subschemas
217 # This contains additional information about the
218 # validation failure which can help the user figure out
219 # what's wrong exactly.
221 # Join each message with `; ` and append this to our
222 # configuration parsing error's message.
223 msgs
= '; '.join([e
.message
for e
in exc
.context
])
224 schema_ctx
= f
': {msgs}'
226 new_exc
= _ConfigurationParseError(contexts
.pop(),
227 f
'{exc.message}{schema_ctx} (from schema `{schema_short_id}`)')
229 for ctx
in reversed(contexts
):
230 new_exc
._append
_ctx
(ctx
)
235 # barectf 3 YAML configuration node.
237 def __init__(self
, config_node
):
238 self
._config
_node
= config_node
241 def config_node(self
):
242 return self
._config
_node
245 _CONFIG_V3_YAML_TAG
= 'tag:barectf.org,2020/3/config'
248 # Loads the content of the YAML file-like object `file` as a Python
249 # object and returns it.
251 # If the file's object has the barectf 3 configuration tag, then this
252 # function returns a `_ConfigNodeV3` object. Otherwise, it returns a
253 # `collections.OrderedDict` object.
255 # All YAML maps are loaded as `collections.OrderedDict` objects.
256 def _yaml_load(file):
257 class Loader(yaml
.Loader
):
260 def config_ctor(loader
, node
):
261 if not isinstance(node
, yaml
.MappingNode
):
262 problem
= f
'Expecting a map for the tag `{node.tag}`'
263 raise yaml
.constructor
.ConstructorError(problem
=problem
)
265 loader
.flatten_mapping(node
)
266 return _ConfigNodeV3(collections
.OrderedDict(loader
.construct_pairs(node
)))
268 def mapping_ctor(loader
, node
):
269 loader
.flatten_mapping(node
)
270 return collections
.OrderedDict(loader
.construct_pairs(node
))
272 Loader
.add_constructor(_CONFIG_V3_YAML_TAG
, config_ctor
)
273 Loader
.add_constructor(yaml
.resolver
.BaseResolver
.DEFAULT_MAPPING_TAG
, mapping_ctor
)
277 return yaml
.load(file, Loader
=Loader
)
278 except (yaml
.YAMLError
, OSError, IOError) as exc
:
279 raise _ConfigurationParseError('YAML loader', f
'Cannot load file: {exc}')
282 def _yaml_load_path(path
):
283 with
open(path
) as f
:
287 # Dumps the content of the Python object `obj`
288 # (`collections.OrderedDict` or `_ConfigNodeV3`) as a YAML string and
290 def _yaml_dump(node
, **kwds
):
291 class Dumper(yaml
.Dumper
):
294 def config_repr(dumper
, node
):
295 return dumper
.represent_mapping(_CONFIG_V3_YAML_TAG
, node
.config_node
.items())
297 def mapping_repr(dumper
, node
):
298 return dumper
.represent_mapping(yaml
.resolver
.BaseResolver
.DEFAULT_MAPPING_TAG
,
301 Dumper
.add_representer(_ConfigNodeV3
, config_repr
)
302 Dumper
.add_representer(collections
.OrderedDict
, mapping_repr
)
305 return yaml
.dump(node
, Dumper
=Dumper
, version
=(1, 2), **kwds
)
308 # A common barectf YAML configuration parser.
310 # This is the base class of any barectf YAML configuration parser. It
311 # mostly contains helpers.
313 # Builds a base barectf YAML configuration parser to process the
314 # configuration node `node` (already loaded from the file having the
317 # For its _process_node_include() method, the parser considers the
318 # package inclusion directory as well as `include_dirs`, and ignores
319 # nonexistent inclusion files if `ignore_include_not_found` is
321 def __init__(self
, path
, node
, with_pkg_include_dir
, include_dirs
, ignore_include_not_found
,
323 self
._root
_path
= path
324 self
._root
_node
= node
325 self
._ft
_prop
_names
= [
335 'element-field-type',
338 self
._include
_dirs
= copy
.copy(include_dirs
)
340 if with_pkg_include_dir
:
341 self
._include
_dirs
.append(pkg_resources
.resource_filename(__name__
, f
'include/{major_version}'))
343 self
._ignore
_include
_not
_found
= ignore_include_not_found
344 self
._include
_stack
= []
345 self
._resolved
_ft
_aliases
= set()
346 self
._schema
_validator
= _SchemaValidator({'common/config', f
'{major_version}/config'})
347 self
._major
_version
= major_version
350 def _struct_ft_node_members_prop_name(self
):
351 if self
._major
_version
== 2:
356 # Returns the last included file name from the parser's inclusion
358 def _get_last_include_file(self
):
359 if self
._include
_stack
:
360 return self
._include
_stack
[-1]
362 return self
._root
_path
364 # Loads the inclusion file having the path `yaml_path` and returns
365 # its content as a `collections.OrderedDict` object.
366 def _load_include(self
, yaml_path
):
367 for inc_dir
in self
._include
_dirs
:
368 # Current inclusion dir + file name path.
370 # Note: os.path.join() only takes the last argument if it's
372 inc_path
= os
.path
.join(inc_dir
, yaml_path
)
374 # real path (symbolic links resolved)
375 real_path
= os
.path
.realpath(inc_path
)
377 # normalized path (weird stuff removed!)
378 norm_path
= os
.path
.normpath(real_path
)
380 if not os
.path
.isfile(norm_path
):
381 # file doesn't exist: skip
384 if norm_path
in self
._include
_stack
:
385 base_path
= self
._get
_last
_include
_file
()
386 raise _ConfigurationParseError(f
'File `{base_path}`',
387 f
'Cannot recursively include file `{norm_path}`')
389 self
._include
_stack
.append(norm_path
)
392 return _yaml_load_path(norm_path
)
394 if not self
._ignore
_include
_not
_found
:
395 base_path
= self
._get
_last
_include
_file
()
396 raise _ConfigurationParseError(f
'File `{base_path}`',
397 f
'Cannot include file `{yaml_path}`: file not found in inclusion directories')
399 # Returns a list of all the inclusion file paths as found in the
400 # inclusion node `include_node`.
401 def _get_include_paths(self
, include_node
):
402 if include_node
is None:
406 if type(include_node
) is str:
408 return [include_node
]
411 assert type(include_node
) is list
414 # Updates the node `base_node` with an overlay node `overlay_node`.
416 # Both the inclusion and field type node inheritance features use
417 # this update mechanism.
418 def _update_node(self
, base_node
, overlay_node
):
419 # see the comment about the `members` property below
420 def update_members_node(base_value
, olay_value
):
421 assert type(olay_value
) is list
422 assert type(base_value
) is list
424 for olay_item
in olay_value
:
425 # assume we append `olay_item` to `base_value` initially
426 append_olay_item
= True
428 if type(olay_item
) is collections
.OrderedDict
:
429 # overlay item is an object
430 if len(olay_item
) == 1:
431 # overlay object item contains a single property
432 olay_name
= list(olay_item
)[0]
434 # find corresponding base item
435 for base_item
in base_value
:
436 if type(base_item
) is collections
.OrderedDict
:
437 if len(olay_item
) == 1:
438 base_name
= list(base_item
)[0]
440 if olay_name
== base_name
:
441 # Names match: update with usual
443 self
._update
_node
(base_item
, olay_item
)
445 # Do _not_ append `olay_item` to
446 # `base_value`: we just updated
448 append_olay_item
= False
452 base_value
.append(copy
.deepcopy(olay_item
))
454 for olay_key
, olay_value
in overlay_node
.items():
455 if olay_key
in base_node
:
456 base_value
= base_node
[olay_key
]
458 if type(olay_value
) is collections
.OrderedDict
and type(base_value
) is collections
.OrderedDict
:
460 self
._update
_node
(base_value
, olay_value
)
461 elif type(olay_value
) is list and type(base_value
) is list:
462 if olay_key
== 'members' and self
._major
_version
== 3:
463 # This is a "temporary" hack.
465 # In barectf 2, a structure field type node
473 # Having an overlay such as
485 # because the `fields` property is a map.
487 # In barectf 3, this is fixed (a YAML map is not
488 # ordered), so that the same initial structure
489 # field type node looks like this:
498 # Although the `members` property is
499 # syntaxically an array, it's semantically an
500 # ordered map, where an entry's key is the array
501 # item's map's first key (like YAML's `!!omap`).
503 # Having an overlay such as
518 # with the naive strategy, while what we really
528 # As of this version of barectf, the _only_
529 # property with a list value which acts as an
530 # ordered map is named `members`. This is why we
531 # can only check the value of `olay_key`,
532 # whatever our context.
534 # update_members_node() attempts to perform
535 # this below. For a given item of `olay_value`,
538 # * It's not an object.
540 # * It contains more than one property.
542 # * Its single property's name does not match
543 # the name of the single property of any
544 # object item of `base_value`.
546 # then we append the item to `base_value` as
548 update_members_node(base_value
, olay_value
)
550 # append extension array items to base items
551 base_value
+= copy
.deepcopy(olay_value
)
553 # fall back to replacing base property
554 base_node
[olay_key
] = copy
.deepcopy(olay_value
)
556 # set base property from overlay property
557 base_node
[olay_key
] = copy
.deepcopy(olay_value
)
559 # Processes inclusions using `last_overlay_node` as the last overlay
560 # node to use to "patch" the node.
562 # If `last_overlay_node` contains an `$include` property, then this
563 # method patches the current base node (initially empty) in order
564 # using the content of the inclusion files (recursively).
566 # At the end, this method removes the `$include` property of
567 # `last_overlay_node` and then patches the current base node with
568 # its other properties before returning the result (always a deep
570 def _process_node_include(self
, last_overlay_node
,
571 process_base_include_cb
,
572 process_children_include_cb
=None):
573 # process children inclusions first
574 if process_children_include_cb
is not None:
575 process_children_include_cb(last_overlay_node
)
577 incl_prop_name
= '$include'
579 if incl_prop_name
in last_overlay_node
:
580 include_node
= last_overlay_node
[incl_prop_name
]
583 return last_overlay_node
585 include_paths
= self
._get
_include
_paths
(include_node
)
586 cur_base_path
= self
._get
_last
_include
_file
()
589 # keep the inclusion paths and remove the `$include` property
590 include_paths
= copy
.deepcopy(include_paths
)
591 del last_overlay_node
[incl_prop_name
]
593 for include_path
in include_paths
:
594 # load raw YAML from included file
595 overlay_node
= self
._load
_include
(include_path
)
597 if overlay_node
is None:
598 # Cannot find inclusion file, but we're ignoring those
599 # errors, otherwise _load_include() itself raises a
603 # recursively process inclusions
605 overlay_node
= process_base_include_cb(overlay_node
)
606 except _ConfigurationParseError
as exc
:
607 _append_error_ctx(exc
, f
'File `{cur_base_path}`')
609 # pop inclusion stack now that we're done including
610 del self
._include
_stack
[-1]
612 # At this point, `base_node` is fully resolved (does not
613 # contain any `$include` property).
614 if base_node
is None:
615 base_node
= overlay_node
617 self
._update
_node
(base_node
, overlay_node
)
619 # Finally, update the latest base node with our last overlay
621 if base_node
is None:
622 # Nothing was included, which is possible when we're
623 # ignoring inclusion errors.
624 return last_overlay_node
626 self
._update
_node
(base_node
, last_overlay_node
)
629 # Generates pairs of member node and field type node property name
630 # (in the member node) for the structure field type node's members
632 def _struct_ft_member_fts_iter(self
, node
):
633 if type(node
) is list:
635 assert self
._major
_version
== 3
637 for member_node
in node
:
638 assert type(member_node
) is collections
.OrderedDict
639 name
, val
= list(member_node
.items())[0]
641 if type(val
) is collections
.OrderedDict
:
645 yield member_node
, name
648 assert self
._major
_version
== 2
649 assert type(node
) is collections
.OrderedDict
654 # Resolves the field type alias `key` in the node `parent_node`, as
655 # well as any nested field type aliases, using the aliases of the
656 # `ft_aliases_node` node.
658 # If `key` is not in `parent_node`, this method returns.
660 # This method can modify `ft_aliases_node` and `parent_node[key]`.
662 # `ctx_obj_name` is the context's object name when this method
663 # raises a `_ConfigurationParseError` exception.
664 def _resolve_ft_alias(self
, ft_aliases_node
, parent_node
, key
, ctx_obj_name
, alias_set
=None):
665 if key
not in parent_node
:
668 node
= parent_node
[key
]
671 # some nodes can be null to use their default value
674 # This set holds all the field type aliases to be expanded,
675 # recursively. This is used to detect cycles.
676 if alias_set
is None:
679 if type(node
) is str:
682 # Make sure this alias names an existing field type node, at
684 if alias
not in ft_aliases_node
:
685 raise _ConfigurationParseError(ctx_obj_name
,
686 f
'Field type alias `{alias}` does not exist')
688 if alias
not in self
._resolved
_ft
_aliases
:
689 # Only check for a field type alias cycle when we didn't
690 # resolve the alias yet, as a given node can refer to
691 # the same field type alias more than once.
692 if alias
in alias_set
:
693 msg
= f
'Cycle detected during the `{alias}` field type alias resolution'
694 raise _ConfigurationParseError(ctx_obj_name
, msg
)
698 # Add `alias` to the set of encountered field type
699 # aliases before calling self._resolve_ft_alias() to
702 self
._resolve
_ft
_alias
(ft_aliases_node
, ft_aliases_node
, alias
, ctx_obj_name
,
704 self
._resolved
_ft
_aliases
.add(alias
)
706 # replace alias with field type node copy
707 parent_node
[key
] = copy
.deepcopy(ft_aliases_node
[alias
])
710 # resolve nested field type aliases
711 for pkey
in self
._ft
_prop
_names
:
712 self
._resolve
_ft
_alias
(ft_aliases_node
, node
, pkey
, ctx_obj_name
, alias_set
)
714 # Resolve field type aliases of structure field type node member
716 pkey
= self
._struct
_ft
_node
_members
_prop
_name
719 for member_node
, ft_prop_name
in self
._struct
_ft
_member
_fts
_iter
(node
[pkey
]):
720 self
._resolve
_ft
_alias
(ft_aliases_node
, member_node
, ft_prop_name
,
721 ctx_obj_name
, alias_set
)
723 # Like _resolve_ft_alias(), but builds a context object name for any
724 # `ctx_obj_name` exception.
725 def _resolve_ft_alias_from(self
, ft_aliases_node
, parent_node
, key
):
726 self
._resolve
_ft
_alias
(ft_aliases_node
, parent_node
, key
, f
'`{key}` property')
728 # Applies field type node inheritance to the property `key` of
731 # `parent_node[key]`, if it exists, must not contain any field type
732 # alias (all field type objects are complete).
734 # This method can modify `parent[key]`.
736 # When this method returns, no field type node has an `$inherit` or
737 # `inherit` property.
738 def _apply_ft_inheritance(self
, parent_node
, key
):
739 if key
not in parent_node
:
742 node
= parent_node
[key
]
747 # process children first
748 for pkey
in self
._ft
_prop
_names
:
749 self
._apply
_ft
_inheritance
(node
, pkey
)
751 # Process the field types of structure field type node member
753 pkey
= self
._struct
_ft
_node
_members
_prop
_name
756 for member_node
, ft_prop_name
in self
._struct
_ft
_member
_fts
_iter
(node
[pkey
]):
757 self
._apply
_ft
_inheritance
(member_node
, ft_prop_name
)
759 # apply inheritance for this node
760 if 'inherit' in node
:
761 # barectf 2.1: `inherit` property was renamed to `$inherit`
762 assert '$inherit' not in node
763 node
['$inherit'] = node
['inherit']
766 inherit_key
= '$inherit'
768 if inherit_key
in node
:
769 assert type(node
[inherit_key
]) is collections
.OrderedDict
771 # apply inheritance below
772 self
._apply
_ft
_inheritance
(node
, inherit_key
)
774 # `node` is an overlay on the `$inherit` node
775 base_node
= node
[inherit_key
]
776 del node
[inherit_key
]
777 self
._update
_node
(base_node
, node
)
779 # set updated base node as this node
780 parent_node
[key
] = base_node