Commit | Line | Data |
---|---|---|
4810b707 PP |
1 | # The MIT License (MIT) |
2 | # | |
3 | # Copyright (c) 2015-2020 Philippe Proulx <pproulx@efficios.com> | |
4 | # | |
5 | # Permission is hereby granted, free of charge, to any person obtaining | |
6 | # a copy of this software and associated documeneffective_filetation files (the | |
7 | # "Software"), to deal in the Software without restriction, including | |
8 | # without limitation the rights to use, copy, modify, merge, publish, | |
9 | # distribute, sublicense, and/or sell copies of the Software, and to | |
10 | # permit persons to whom the Software is furnished to do so, subject to | |
11 | # the following conditions: | |
12 | # | |
13 | # The above copyright notice and this permission notice shall be | |
14 | # included in all copies or substantial portions of the Software. | |
15 | # | |
16 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
17 | # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
18 | # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. | |
19 | # IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY | |
20 | # CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, | |
21 | # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE | |
22 | # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. | |
23 | ||
24 | import pkg_resources | |
25 | import collections | |
2d55dc7d | 26 | import jsonschema # type: ignore |
4810b707 PP |
27 | import os.path |
28 | import yaml | |
29 | import copy | |
30 | import os | |
2d55dc7d PP |
31 | from barectf.typing import VersionNumber, _OptStr |
32 | from typing import Optional, List, Dict, Any, TextIO, MutableMapping, Union, Set, Iterable, Callable, Tuple | |
33 | import typing | |
4810b707 PP |
34 | |
35 | ||
36 | # The context of a configuration parsing error. | |
37 | # | |
38 | # Such a context object has a name and, optionally, a message. | |
39 | class _ConfigurationParseErrorContext: | |
2d55dc7d | 40 | def __init__(self, name: str, message: _OptStr = None): |
4810b707 PP |
41 | self._name = name |
42 | self._msg = message | |
43 | ||
44 | @property | |
2d55dc7d | 45 | def name(self) -> str: |
4810b707 PP |
46 | return self._name |
47 | ||
48 | @property | |
2d55dc7d | 49 | def message(self) -> _OptStr: |
4810b707 PP |
50 | return self._msg |
51 | ||
52 | ||
4810b707 PP |
53 | # A configuration parsing error. |
54 | # | |
55 | # Such an error object contains a list of contexts (`context` property). | |
56 | # | |
57 | # The first context of this list is the most specific context, while the | |
58 | # last is the more general. | |
59 | # | |
60 | # Use _append_ctx() to append a context to an existing configuration | |
61 | # parsing error when you catch it before raising it again. You can use | |
62 | # _append_error_ctx() to do exactly this in a single call. | |
63 | class _ConfigurationParseError(Exception): | |
64 | def __init__(self, init_ctx_obj_name, init_ctx_msg=None): | |
65 | super().__init__() | |
2d55dc7d | 66 | self._ctx: List[_ConfigurationParseErrorContext] = [] |
4810b707 PP |
67 | self._append_ctx(init_ctx_obj_name, init_ctx_msg) |
68 | ||
69 | @property | |
2d55dc7d | 70 | def context(self) -> List[_ConfigurationParseErrorContext]: |
4810b707 PP |
71 | return self._ctx |
72 | ||
2d55dc7d | 73 | def _append_ctx(self, name: str, msg: _OptStr = None): |
4810b707 PP |
74 | self._ctx.append(_ConfigurationParseErrorContext(name, msg)) |
75 | ||
76 | def __str__(self): | |
77 | lines = [] | |
78 | ||
79 | for ctx in reversed(self._ctx): | |
80 | line = f'{ctx.name}:' | |
81 | ||
82 | if ctx.message is not None: | |
83 | line += f' {ctx.message}' | |
84 | ||
85 | lines.append(line) | |
86 | ||
87 | return '\n'.join(lines) | |
88 | ||
89 | ||
2d55dc7d PP |
90 | # Appends the context having the object name `obj_name` and the |
91 | # (optional) message `message` to the `_ConfigurationParseError` | |
92 | # exception `exc` and then raises `exc` again. | |
93 | def _append_error_ctx(exc: _ConfigurationParseError, obj_name: str, message: _OptStr = None): | |
94 | exc._append_ctx(obj_name, message) | |
95 | raise exc | |
96 | ||
97 | ||
4810b707 PP |
98 | _V3Prefixes = collections.namedtuple('_V3Prefixes', ['identifier', 'file_name']) |
99 | ||
100 | ||
101 | # Convers a v2 prefix to v3 prefixes. | |
2d55dc7d | 102 | def _v3_prefixes_from_v2_prefix(v2_prefix: str) -> _V3Prefixes: |
4810b707 PP |
103 | return _V3Prefixes(v2_prefix, v2_prefix.rstrip('_')) |
104 | ||
105 | ||
106 | # This JSON schema reference resolver only serves to detect when it | |
107 | # needs to resolve a remote URI. | |
108 | # | |
109 | # This must never happen in barectf because all our schemas are local; | |
110 | # it would mean a programming or schema error. | |
111 | class _RefResolver(jsonschema.RefResolver): | |
2d55dc7d | 112 | def resolve_remote(self, uri: str): |
4810b707 PP |
113 | raise RuntimeError(f'Missing local schema with URI `{uri}`') |
114 | ||
115 | ||
2d55dc7d PP |
116 | # Not all static type checkers support type recursion, so let's just use |
117 | # `Any` as a map node's value's type. | |
118 | _MapNode = MutableMapping[str, Any] | |
119 | ||
120 | ||
4810b707 PP |
121 | # Schema validator which considers all the schemas found in the |
122 | # subdirectories `subdirs` (at build time) of the barectf package's | |
123 | # `schemas` directory. | |
124 | # | |
125 | # The only public method is validate() which accepts an instance to | |
126 | # validate as well as a schema short ID. | |
127 | class _SchemaValidator: | |
2d55dc7d | 128 | def __init__(self, subdirs: Iterable[str]): |
4810b707 | 129 | schemas_dir = pkg_resources.resource_filename(__name__, 'schemas') |
2d55dc7d | 130 | self._store: Dict[str, str] = {} |
4810b707 PP |
131 | |
132 | for subdir in subdirs: | |
133 | dir = os.path.join(schemas_dir, subdir) | |
134 | ||
135 | for file_name in os.listdir(dir): | |
136 | if not file_name.endswith('.yaml'): | |
137 | continue | |
138 | ||
139 | with open(os.path.join(dir, file_name)) as f: | |
140 | schema = yaml.load(f, Loader=yaml.SafeLoader) | |
141 | ||
142 | assert '$id' in schema | |
143 | schema_id = schema['$id'] | |
144 | assert schema_id not in self._store | |
145 | self._store[schema_id] = schema | |
146 | ||
147 | @staticmethod | |
148 | def _dict_from_ordered_dict(obj): | |
149 | if type(obj) is not collections.OrderedDict: | |
150 | return obj | |
151 | ||
152 | dct = {} | |
153 | ||
154 | for k, v in obj.items(): | |
155 | new_v = v | |
156 | ||
157 | if type(v) is collections.OrderedDict: | |
158 | new_v = _SchemaValidator._dict_from_ordered_dict(v) | |
159 | elif type(v) is list: | |
160 | new_v = [_SchemaValidator._dict_from_ordered_dict(elem) for elem in v] | |
161 | ||
162 | dct[k] = new_v | |
163 | ||
164 | return dct | |
165 | ||
2d55dc7d | 166 | def _validate(self, instance: _MapNode, schema_short_id: str): |
4810b707 PP |
167 | # retrieve full schema ID from short ID |
168 | schema_id = f'https://barectf.org/schemas/{schema_short_id}.json' | |
169 | assert schema_id in self._store | |
170 | ||
171 | # retrieve full schema | |
172 | schema = self._store[schema_id] | |
173 | ||
174 | # Create a reference resolver for this schema using this | |
175 | # validator's schema store. | |
176 | resolver = _RefResolver(base_uri=schema_id, referrer=schema, | |
177 | store=self._store) | |
178 | ||
179 | # create a JSON schema validator using this reference resolver | |
180 | validator = jsonschema.Draft7Validator(schema, resolver=resolver) | |
181 | ||
182 | # Validate the instance, converting its | |
183 | # `collections.OrderedDict` objects to `dict` objects so as to | |
184 | # make any error message easier to read (because | |
185 | # validator.validate() below uses str() for error messages, and | |
186 | # collections.OrderedDict.__str__() returns a somewhat bulky | |
187 | # representation). | |
188 | validator.validate(self._dict_from_ordered_dict(instance)) | |
189 | ||
190 | # Validates `instance` using the schema having the short ID | |
191 | # `schema_short_id`. | |
192 | # | |
193 | # A schema short ID is the part between `schemas/` and `.json` in | |
194 | # its URI. | |
195 | # | |
196 | # Raises a `_ConfigurationParseError` object, hiding any | |
197 | # `jsonschema` exception, on validation failure. | |
2d55dc7d | 198 | def validate(self, instance: _MapNode, schema_short_id: str): |
4810b707 PP |
199 | try: |
200 | self._validate(instance, schema_short_id) | |
201 | except jsonschema.ValidationError as exc: | |
202 | # convert to barectf `_ConfigurationParseError` exception | |
203 | contexts = ['Configuration object'] | |
204 | ||
205 | # Each element of the instance's absolute path is either an | |
206 | # integer (array element's index) or a string (object | |
207 | # property's name). | |
208 | for elem in exc.absolute_path: | |
209 | if type(elem) is int: | |
210 | ctx = f'Element #{elem + 1}' | |
211 | else: | |
212 | ctx = f'`{elem}` property' | |
213 | ||
214 | contexts.append(ctx) | |
215 | ||
216 | schema_ctx = '' | |
217 | ||
218 | if len(exc.context) > 0: | |
219 | # According to the documentation of | |
220 | # jsonschema.ValidationError.context(), the method | |
221 | # returns a | |
222 | # | |
223 | # > list of errors from the subschemas | |
224 | # | |
225 | # This contains additional information about the | |
226 | # validation failure which can help the user figure out | |
227 | # what's wrong exactly. | |
228 | # | |
229 | # Join each message with `; ` and append this to our | |
230 | # configuration parsing error's message. | |
231 | msgs = '; '.join([e.message for e in exc.context]) | |
232 | schema_ctx = f': {msgs}' | |
233 | ||
234 | new_exc = _ConfigurationParseError(contexts.pop(), | |
235 | f'{exc.message}{schema_ctx} (from schema `{schema_short_id}`)') | |
236 | ||
237 | for ctx in reversed(contexts): | |
238 | new_exc._append_ctx(ctx) | |
239 | ||
240 | raise new_exc | |
241 | ||
242 | ||
243 | # barectf 3 YAML configuration node. | |
244 | class _ConfigNodeV3: | |
2d55dc7d | 245 | def __init__(self, config_node: _MapNode): |
4810b707 PP |
246 | self._config_node = config_node |
247 | ||
248 | @property | |
2d55dc7d | 249 | def config_node(self) -> _MapNode: |
4810b707 PP |
250 | return self._config_node |
251 | ||
252 | ||
253 | _CONFIG_V3_YAML_TAG = 'tag:barectf.org,2020/3/config' | |
254 | ||
255 | ||
256 | # Loads the content of the YAML file-like object `file` as a Python | |
257 | # object and returns it. | |
258 | # | |
259 | # If the file's object has the barectf 3 configuration tag, then this | |
260 | # function returns a `_ConfigNodeV3` object. Otherwise, it returns a | |
261 | # `collections.OrderedDict` object. | |
262 | # | |
263 | # All YAML maps are loaded as `collections.OrderedDict` objects. | |
2d55dc7d | 264 | def _yaml_load(file: TextIO) -> Union[_ConfigNodeV3, _MapNode]: |
4810b707 PP |
265 | class Loader(yaml.Loader): |
266 | pass | |
267 | ||
2d55dc7d | 268 | def config_ctor(loader, node) -> _ConfigNodeV3: |
4810b707 PP |
269 | if not isinstance(node, yaml.MappingNode): |
270 | problem = f'Expecting a map for the tag `{node.tag}`' | |
271 | raise yaml.constructor.ConstructorError(problem=problem) | |
272 | ||
273 | loader.flatten_mapping(node) | |
274 | return _ConfigNodeV3(collections.OrderedDict(loader.construct_pairs(node))) | |
275 | ||
2d55dc7d | 276 | def mapping_ctor(loader, node) -> _MapNode: |
4810b707 PP |
277 | loader.flatten_mapping(node) |
278 | return collections.OrderedDict(loader.construct_pairs(node)) | |
279 | ||
280 | Loader.add_constructor(_CONFIG_V3_YAML_TAG, config_ctor) | |
281 | Loader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, mapping_ctor) | |
282 | ||
283 | # YAML -> Python | |
284 | try: | |
285 | return yaml.load(file, Loader=Loader) | |
286 | except (yaml.YAMLError, OSError, IOError) as exc: | |
287 | raise _ConfigurationParseError('YAML loader', f'Cannot load file: {exc}') | |
288 | ||
289 | ||
2d55dc7d | 290 | def _yaml_load_path(path: str) -> Union[_ConfigNodeV3, _MapNode]: |
4810b707 PP |
291 | with open(path) as f: |
292 | return _yaml_load(f) | |
293 | ||
294 | ||
295 | # Dumps the content of the Python object `obj` | |
296 | # (`collections.OrderedDict` or `_ConfigNodeV3`) as a YAML string and | |
297 | # returns it. | |
2d55dc7d | 298 | def _yaml_dump(node: _MapNode, **kwds) -> str: |
4810b707 PP |
299 | class Dumper(yaml.Dumper): |
300 | pass | |
301 | ||
302 | def config_repr(dumper, node): | |
303 | return dumper.represent_mapping(_CONFIG_V3_YAML_TAG, node.config_node.items()) | |
304 | ||
305 | def mapping_repr(dumper, node): | |
306 | return dumper.represent_mapping(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, | |
307 | node.items()) | |
308 | ||
309 | Dumper.add_representer(_ConfigNodeV3, config_repr) | |
310 | Dumper.add_representer(collections.OrderedDict, mapping_repr) | |
311 | ||
312 | # Python -> YAML | |
313 | return yaml.dump(node, Dumper=Dumper, version=(1, 2), **kwds) | |
314 | ||
315 | ||
316 | # A common barectf YAML configuration parser. | |
317 | # | |
318 | # This is the base class of any barectf YAML configuration parser. It | |
319 | # mostly contains helpers. | |
320 | class _Parser: | |
321 | # Builds a base barectf YAML configuration parser to process the | |
2d55dc7d PP |
322 | # configuration node `node` (already loaded from the file-like |
323 | # object `file`). | |
4810b707 PP |
324 | # |
325 | # For its _process_node_include() method, the parser considers the | |
7cd4634e PP |
326 | # package inclusion directory as well as `include_dirs`, and ignores |
327 | # nonexistent inclusion files if `ignore_include_not_found` is | |
328 | # `True`. | |
2d55dc7d PP |
329 | def __init__(self, root_file: TextIO, node: Union[_MapNode, _ConfigNodeV3], |
330 | with_pkg_include_dir: bool, include_dirs: Optional[List[str]], | |
331 | ignore_include_not_found: bool, major_version: VersionNumber): | |
332 | self._root_file = root_file | |
4810b707 PP |
333 | self._root_node = node |
334 | self._ft_prop_names = [ | |
335 | # barectf 2.1+ | |
336 | '$inherit', | |
337 | ||
338 | # barectf 2 | |
339 | 'inherit', | |
340 | 'value-type', | |
341 | 'element-type', | |
342 | ||
343 | # barectf 3 | |
344 | 'element-field-type', | |
345 | ] | |
7cd4634e | 346 | |
2d55dc7d PP |
347 | if include_dirs is None: |
348 | include_dirs = [] | |
349 | ||
7cd4634e PP |
350 | self._include_dirs = copy.copy(include_dirs) |
351 | ||
352 | if with_pkg_include_dir: | |
353 | self._include_dirs.append(pkg_resources.resource_filename(__name__, f'include/{major_version}')) | |
354 | ||
4810b707 | 355 | self._ignore_include_not_found = ignore_include_not_found |
2d55dc7d PP |
356 | self._include_stack: List[str] = [] |
357 | self._resolved_ft_aliases: Set[str] = set() | |
c3fa1a14 | 358 | self._schema_validator = _SchemaValidator({'config/common', f'config/{major_version}'}) |
4810b707 PP |
359 | self._major_version = major_version |
360 | ||
361 | @property | |
2d55dc7d | 362 | def _struct_ft_node_members_prop_name(self) -> str: |
4810b707 PP |
363 | if self._major_version == 2: |
364 | return 'fields' | |
365 | else: | |
366 | return 'members' | |
367 | ||
368 | # Returns the last included file name from the parser's inclusion | |
2d55dc7d PP |
369 | # file name stack, or `N/A` if the root file does not have an |
370 | # associated path under the `name` property. | |
371 | def _get_last_include_file(self) -> str: | |
4810b707 PP |
372 | if self._include_stack: |
373 | return self._include_stack[-1] | |
374 | ||
2d55dc7d PP |
375 | if hasattr(self._root_file, 'name'): |
376 | return typing.cast(str, self._root_file.name) | |
377 | ||
378 | return 'N/A' | |
4810b707 PP |
379 | |
380 | # Loads the inclusion file having the path `yaml_path` and returns | |
381 | # its content as a `collections.OrderedDict` object. | |
2d55dc7d | 382 | def _load_include(self, yaml_path) -> Optional[_MapNode]: |
4810b707 PP |
383 | for inc_dir in self._include_dirs: |
384 | # Current inclusion dir + file name path. | |
385 | # | |
386 | # Note: os.path.join() only takes the last argument if it's | |
387 | # absolute. | |
388 | inc_path = os.path.join(inc_dir, yaml_path) | |
389 | ||
390 | # real path (symbolic links resolved) | |
391 | real_path = os.path.realpath(inc_path) | |
392 | ||
393 | # normalized path (weird stuff removed!) | |
394 | norm_path = os.path.normpath(real_path) | |
395 | ||
396 | if not os.path.isfile(norm_path): | |
397 | # file doesn't exist: skip | |
398 | continue | |
399 | ||
400 | if norm_path in self._include_stack: | |
401 | base_path = self._get_last_include_file() | |
402 | raise _ConfigurationParseError(f'File `{base_path}`', | |
403 | f'Cannot recursively include file `{norm_path}`') | |
404 | ||
405 | self._include_stack.append(norm_path) | |
406 | ||
407 | # load raw content | |
2d55dc7d | 408 | return typing.cast(_MapNode, _yaml_load_path(norm_path)) |
4810b707 PP |
409 | |
410 | if not self._ignore_include_not_found: | |
411 | base_path = self._get_last_include_file() | |
412 | raise _ConfigurationParseError(f'File `{base_path}`', | |
413 | f'Cannot include file `{yaml_path}`: file not found in inclusion directories') | |
414 | ||
2d55dc7d PP |
415 | return None |
416 | ||
4810b707 PP |
417 | # Returns a list of all the inclusion file paths as found in the |
418 | # inclusion node `include_node`. | |
2d55dc7d | 419 | def _get_include_paths(self, include_node: _MapNode) -> List[str]: |
4810b707 PP |
420 | if include_node is None: |
421 | # none | |
422 | return [] | |
423 | ||
424 | if type(include_node) is str: | |
425 | # wrap as array | |
2d55dc7d | 426 | return [typing.cast(str, include_node)] |
4810b707 PP |
427 | |
428 | # already an array | |
429 | assert type(include_node) is list | |
2d55dc7d | 430 | return typing.cast(List[str], include_node) |
4810b707 PP |
431 | |
432 | # Updates the node `base_node` with an overlay node `overlay_node`. | |
433 | # | |
434 | # Both the inclusion and field type node inheritance features use | |
435 | # this update mechanism. | |
2d55dc7d | 436 | def _update_node(self, base_node: _MapNode, overlay_node: _MapNode): |
4810b707 | 437 | # see the comment about the `members` property below |
2d55dc7d | 438 | def update_members_node(base_value: List[Any], olay_value: List[Any]): |
4810b707 PP |
439 | for olay_item in olay_value: |
440 | # assume we append `olay_item` to `base_value` initially | |
441 | append_olay_item = True | |
442 | ||
443 | if type(olay_item) is collections.OrderedDict: | |
444 | # overlay item is an object | |
445 | if len(olay_item) == 1: | |
446 | # overlay object item contains a single property | |
447 | olay_name = list(olay_item)[0] | |
448 | ||
449 | # find corresponding base item | |
450 | for base_item in base_value: | |
451 | if type(base_item) is collections.OrderedDict: | |
452 | if len(olay_item) == 1: | |
453 | base_name = list(base_item)[0] | |
454 | ||
455 | if olay_name == base_name: | |
456 | # Names match: update with usual | |
457 | # strategy. | |
458 | self._update_node(base_item, olay_item) | |
459 | ||
460 | # Do _not_ append `olay_item` to | |
461 | # `base_value`: we just updated | |
462 | # `base_item`. | |
463 | append_olay_item = False | |
464 | break | |
465 | ||
466 | if append_olay_item: | |
467 | base_value.append(copy.deepcopy(olay_item)) | |
468 | ||
469 | for olay_key, olay_value in overlay_node.items(): | |
470 | if olay_key in base_node: | |
471 | base_value = base_node[olay_key] | |
472 | ||
473 | if type(olay_value) is collections.OrderedDict and type(base_value) is collections.OrderedDict: | |
474 | # merge both objects | |
475 | self._update_node(base_value, olay_value) | |
476 | elif type(olay_value) is list and type(base_value) is list: | |
477 | if olay_key == 'members' and self._major_version == 3: | |
478 | # This is a "temporary" hack. | |
479 | # | |
480 | # In barectf 2, a structure field type node | |
481 | # looks like this: | |
482 | # | |
483 | # class: struct | |
484 | # fields: | |
485 | # hello: uint8 | |
486 | # world: string | |
487 | # | |
488 | # Having an overlay such as | |
489 | # | |
490 | # fields: | |
491 | # hello: float | |
492 | # | |
493 | # will result in | |
494 | # | |
495 | # class: struct | |
496 | # fields: | |
497 | # hello: float | |
498 | # world: string | |
499 | # | |
500 | # because the `fields` property is a map. | |
501 | # | |
502 | # In barectf 3, this is fixed (a YAML map is not | |
503 | # ordered), so that the same initial structure | |
504 | # field type node looks like this: | |
505 | # | |
506 | # class: struct | |
507 | # members: | |
508 | # - hello: uint8 | |
509 | # - world: | |
510 | # field-type: | |
511 | # class: str | |
512 | # | |
513 | # Although the `members` property is | |
514 | # syntaxically an array, it's semantically an | |
515 | # ordered map, where an entry's key is the array | |
516 | # item's map's first key (like YAML's `!!omap`). | |
517 | # | |
518 | # Having an overlay such as | |
519 | # | |
520 | # members: | |
521 | # - hello: float | |
522 | # | |
523 | # would result in | |
524 | # | |
525 | # class: struct | |
526 | # members: | |
527 | # - hello: uint8 | |
528 | # - world: | |
529 | # field-type: | |
530 | # class: str | |
531 | # - hello: float | |
532 | # | |
533 | # with the naive strategy, while what we really | |
534 | # want is: | |
535 | # | |
536 | # class: struct | |
537 | # members: | |
538 | # - hello: float | |
539 | # - world: | |
540 | # field-type: | |
541 | # class: str | |
542 | # | |
543 | # As of this version of barectf, the _only_ | |
544 | # property with a list value which acts as an | |
545 | # ordered map is named `members`. This is why we | |
546 | # can only check the value of `olay_key`, | |
547 | # whatever our context. | |
548 | # | |
549 | # update_members_node() attempts to perform | |
550 | # this below. For a given item of `olay_value`, | |
551 | # if | |
552 | # | |
553 | # * It's not an object. | |
554 | # | |
555 | # * It contains more than one property. | |
556 | # | |
557 | # * Its single property's name does not match | |
558 | # the name of the single property of any | |
559 | # object item of `base_value`. | |
560 | # | |
561 | # then we append the item to `base_value` as | |
562 | # usual. | |
563 | update_members_node(base_value, olay_value) | |
564 | else: | |
565 | # append extension array items to base items | |
566 | base_value += copy.deepcopy(olay_value) | |
567 | else: | |
568 | # fall back to replacing base property | |
569 | base_node[olay_key] = copy.deepcopy(olay_value) | |
570 | else: | |
571 | # set base property from overlay property | |
572 | base_node[olay_key] = copy.deepcopy(olay_value) | |
573 | ||
574 | # Processes inclusions using `last_overlay_node` as the last overlay | |
575 | # node to use to "patch" the node. | |
576 | # | |
577 | # If `last_overlay_node` contains an `$include` property, then this | |
578 | # method patches the current base node (initially empty) in order | |
579 | # using the content of the inclusion files (recursively). | |
580 | # | |
581 | # At the end, this method removes the `$include` property of | |
582 | # `last_overlay_node` and then patches the current base node with | |
583 | # its other properties before returning the result (always a deep | |
584 | # copy). | |
2d55dc7d PP |
585 | def _process_node_include(self, last_overlay_node: _MapNode, |
586 | process_base_include_cb: Callable[[_MapNode], _MapNode], | |
587 | process_children_include_cb: Optional[Callable[[_MapNode], None]] = None) -> _MapNode: | |
4810b707 PP |
588 | # process children inclusions first |
589 | if process_children_include_cb is not None: | |
590 | process_children_include_cb(last_overlay_node) | |
591 | ||
592 | incl_prop_name = '$include' | |
593 | ||
594 | if incl_prop_name in last_overlay_node: | |
595 | include_node = last_overlay_node[incl_prop_name] | |
596 | else: | |
597 | # no inclusions! | |
598 | return last_overlay_node | |
599 | ||
600 | include_paths = self._get_include_paths(include_node) | |
601 | cur_base_path = self._get_last_include_file() | |
602 | base_node = None | |
603 | ||
604 | # keep the inclusion paths and remove the `$include` property | |
605 | include_paths = copy.deepcopy(include_paths) | |
606 | del last_overlay_node[incl_prop_name] | |
607 | ||
608 | for include_path in include_paths: | |
609 | # load raw YAML from included file | |
610 | overlay_node = self._load_include(include_path) | |
611 | ||
612 | if overlay_node is None: | |
613 | # Cannot find inclusion file, but we're ignoring those | |
614 | # errors, otherwise _load_include() itself raises a | |
615 | # config error. | |
616 | continue | |
617 | ||
618 | # recursively process inclusions | |
619 | try: | |
620 | overlay_node = process_base_include_cb(overlay_node) | |
621 | except _ConfigurationParseError as exc: | |
622 | _append_error_ctx(exc, f'File `{cur_base_path}`') | |
623 | ||
624 | # pop inclusion stack now that we're done including | |
625 | del self._include_stack[-1] | |
626 | ||
627 | # At this point, `base_node` is fully resolved (does not | |
628 | # contain any `$include` property). | |
629 | if base_node is None: | |
630 | base_node = overlay_node | |
631 | else: | |
632 | self._update_node(base_node, overlay_node) | |
633 | ||
634 | # Finally, update the latest base node with our last overlay | |
635 | # node. | |
636 | if base_node is None: | |
637 | # Nothing was included, which is possible when we're | |
638 | # ignoring inclusion errors. | |
639 | return last_overlay_node | |
640 | ||
641 | self._update_node(base_node, last_overlay_node) | |
642 | return base_node | |
643 | ||
644 | # Generates pairs of member node and field type node property name | |
645 | # (in the member node) for the structure field type node's members | |
646 | # node `node`. | |
2d55dc7d PP |
647 | def _struct_ft_member_fts_iter(self, |
648 | node: Union[List[_MapNode], _MapNode]) -> Iterable[Tuple[_MapNode, str]]: | |
4810b707 PP |
649 | if type(node) is list: |
650 | # barectf 3 | |
651 | assert self._major_version == 3 | |
2d55dc7d | 652 | node = typing.cast(List[_MapNode], node) |
4810b707 PP |
653 | |
654 | for member_node in node: | |
655 | assert type(member_node) is collections.OrderedDict | |
2d55dc7d | 656 | member_node = typing.cast(_MapNode, member_node) |
4810b707 PP |
657 | name, val = list(member_node.items())[0] |
658 | ||
659 | if type(val) is collections.OrderedDict: | |
660 | member_node = val | |
661 | name = 'field-type' | |
662 | ||
663 | yield member_node, name | |
664 | else: | |
665 | # barectf 2 | |
666 | assert self._major_version == 2 | |
667 | assert type(node) is collections.OrderedDict | |
2d55dc7d | 668 | node = typing.cast(_MapNode, node) |
4810b707 PP |
669 | |
670 | for name in node: | |
671 | yield node, name | |
672 | ||
673 | # Resolves the field type alias `key` in the node `parent_node`, as | |
674 | # well as any nested field type aliases, using the aliases of the | |
675 | # `ft_aliases_node` node. | |
676 | # | |
677 | # If `key` is not in `parent_node`, this method returns. | |
678 | # | |
679 | # This method can modify `ft_aliases_node` and `parent_node[key]`. | |
680 | # | |
681 | # `ctx_obj_name` is the context's object name when this method | |
682 | # raises a `_ConfigurationParseError` exception. | |
2d55dc7d PP |
683 | def _resolve_ft_alias(self, ft_aliases_node: _MapNode, parent_node: _MapNode, key: str, |
684 | ctx_obj_name: str, alias_set: Optional[Set[str]] = None): | |
4810b707 PP |
685 | if key not in parent_node: |
686 | return | |
687 | ||
688 | node = parent_node[key] | |
689 | ||
690 | if node is None: | |
691 | # some nodes can be null to use their default value | |
692 | return | |
693 | ||
694 | # This set holds all the field type aliases to be expanded, | |
695 | # recursively. This is used to detect cycles. | |
696 | if alias_set is None: | |
697 | alias_set = set() | |
698 | ||
699 | if type(node) is str: | |
700 | alias = node | |
701 | ||
702 | # Make sure this alias names an existing field type node, at | |
703 | # least. | |
704 | if alias not in ft_aliases_node: | |
705 | raise _ConfigurationParseError(ctx_obj_name, | |
706 | f'Field type alias `{alias}` does not exist') | |
707 | ||
708 | if alias not in self._resolved_ft_aliases: | |
709 | # Only check for a field type alias cycle when we didn't | |
710 | # resolve the alias yet, as a given node can refer to | |
711 | # the same field type alias more than once. | |
712 | if alias in alias_set: | |
713 | msg = f'Cycle detected during the `{alias}` field type alias resolution' | |
714 | raise _ConfigurationParseError(ctx_obj_name, msg) | |
715 | ||
716 | # Resolve it. | |
717 | # | |
718 | # Add `alias` to the set of encountered field type | |
719 | # aliases before calling self._resolve_ft_alias() to | |
720 | # detect cycles. | |
721 | alias_set.add(alias) | |
722 | self._resolve_ft_alias(ft_aliases_node, ft_aliases_node, alias, ctx_obj_name, | |
723 | alias_set) | |
724 | self._resolved_ft_aliases.add(alias) | |
725 | ||
726 | # replace alias with field type node copy | |
727 | parent_node[key] = copy.deepcopy(ft_aliases_node[alias]) | |
728 | return | |
729 | ||
730 | # resolve nested field type aliases | |
731 | for pkey in self._ft_prop_names: | |
732 | self._resolve_ft_alias(ft_aliases_node, node, pkey, ctx_obj_name, alias_set) | |
733 | ||
734 | # Resolve field type aliases of structure field type node member | |
735 | # nodes. | |
736 | pkey = self._struct_ft_node_members_prop_name | |
737 | ||
738 | if pkey in node: | |
739 | for member_node, ft_prop_name in self._struct_ft_member_fts_iter(node[pkey]): | |
740 | self._resolve_ft_alias(ft_aliases_node, member_node, ft_prop_name, | |
741 | ctx_obj_name, alias_set) | |
742 | ||
743 | # Like _resolve_ft_alias(), but builds a context object name for any | |
744 | # `ctx_obj_name` exception. | |
2d55dc7d | 745 | def _resolve_ft_alias_from(self, ft_aliases_node: _MapNode, parent_node: _MapNode, key: str): |
4810b707 PP |
746 | self._resolve_ft_alias(ft_aliases_node, parent_node, key, f'`{key}` property') |
747 | ||
748 | # Applies field type node inheritance to the property `key` of | |
749 | # `parent_node`. | |
750 | # | |
751 | # `parent_node[key]`, if it exists, must not contain any field type | |
752 | # alias (all field type objects are complete). | |
753 | # | |
754 | # This method can modify `parent[key]`. | |
755 | # | |
756 | # When this method returns, no field type node has an `$inherit` or | |
757 | # `inherit` property. | |
2d55dc7d | 758 | def _apply_ft_inheritance(self, parent_node: _MapNode, key: str): |
4810b707 PP |
759 | if key not in parent_node: |
760 | return | |
761 | ||
762 | node = parent_node[key] | |
763 | ||
764 | if node is None: | |
765 | return | |
766 | ||
767 | # process children first | |
768 | for pkey in self._ft_prop_names: | |
769 | self._apply_ft_inheritance(node, pkey) | |
770 | ||
771 | # Process the field types of structure field type node member | |
772 | # nodes. | |
773 | pkey = self._struct_ft_node_members_prop_name | |
774 | ||
775 | if pkey in node: | |
776 | for member_node, ft_prop_name in self._struct_ft_member_fts_iter(node[pkey]): | |
777 | self._apply_ft_inheritance(member_node, ft_prop_name) | |
778 | ||
779 | # apply inheritance for this node | |
780 | if 'inherit' in node: | |
781 | # barectf 2.1: `inherit` property was renamed to `$inherit` | |
782 | assert '$inherit' not in node | |
783 | node['$inherit'] = node['inherit'] | |
784 | del node['inherit'] | |
785 | ||
786 | inherit_key = '$inherit' | |
787 | ||
788 | if inherit_key in node: | |
789 | assert type(node[inherit_key]) is collections.OrderedDict | |
790 | ||
791 | # apply inheritance below | |
792 | self._apply_ft_inheritance(node, inherit_key) | |
793 | ||
794 | # `node` is an overlay on the `$inherit` node | |
795 | base_node = node[inherit_key] | |
796 | del node[inherit_key] | |
797 | self._update_node(base_node, node) | |
798 | ||
799 | # set updated base node as this node | |
800 | parent_node[key] = base_node |