diff --git a/CHANGELOG.md b/CHANGELOG.md index 87ceca04..467f1756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## \[Unreleased\] +### Added + +- `SerializationOptions.metadata_sidecar`, which carries `__is_block__`, `__comments__` and `__inline_comments__` beside the mapping rather than among its keys. HCL reserves none of those names, so a document may declare an attribute called any of them -- and in-band one of the two has to lose: on read the marker overwrites the attribute, on write the deserializer drops it, and by then the dict holds a single value with no way to tell which happened. With the option set, `loads` returns an `HclDict`, a `dict` subclass whose `hcl_meta` holds the three, so the mapping contains attributes and nothing else. `dumps` accepts either form, including a hand-built dict using the old keys. Off by default: the keys are a documented part of the output shape, and JSON cannot carry a sidecar. `HclDict`, `HclMeta` and `meta_of` are exported from `hcl2`. Copying, merging with `|` and pickling carry the metadata; `dict(d)` and `{**d}` deliberately do not, since asking for a `dict` gives the mapping and nothing else. ([#331](https://github.com/amplify-education/python-hcl2/issues/331)) ### Fixed - Parse blocks whose type or unquoted label is an HCL keyword, such as the `in` block of the Snowflake provider's `snowflake_schemas` data source. HCL does not reserve its keywords, so `if`, `in`, `for`, `for_each`, `else`, `endif`, `endfor`, `true`, `false`, and `null` are now accepted in every block label position and normalized to identifiers — matching the existing behaviour for keyword attribute names. The block-side grammar gap was diagnosed independently in [#355](https://github.com/amplify-education/python-hcl2/pull/355). ([#357](https://github.com/amplify-education/python-hcl2/pull/357)) diff --git a/hcl2/__init__.py b/hcl2/__init__.py index 4bbdcd7e..14152251 100644 --- a/hcl2/__init__.py +++ b/hcl2/__init__.py @@ -24,27 +24,31 @@ from .builder import Builder from .deserializer import DeserializerOptions from .formatter import FormatterOptions +from .meta import HclDict, HclMeta, meta_of from .rules.base import StartRule from .utils import SerializationOptions __all__ = [ + "Builder", + "DeserializerOptions", "dump", "dumps", + "FormatterOptions", "from_dict", "from_json", + "HclDict", + "HclMeta", "load", "loads", + "meta_of", "parse", "parse_to_tree", "parses", "parses_to_tree", "query", "reconstruct", + "SerializationOptions", "serialize", - "transform", - "Builder", - "DeserializerOptions", - "FormatterOptions", "StartRule", - "SerializationOptions", + "transform", ] diff --git a/hcl2/deserializer.py b/hcl2/deserializer.py index 667e0b20..505bbf0c 100644 --- a/hcl2/deserializer.py +++ b/hcl2/deserializer.py @@ -10,6 +10,7 @@ from regex import regex from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK +from hcl2.meta import meta_of from hcl2.parser import parser as _get_parser from hcl2.rules.abstract import LarkElement, LarkRule from hcl2.rules.base import ( @@ -144,7 +145,7 @@ def _deserialize_block_elements(self, value: dict) -> List[LarkElement]: else: # otherwise it's just an attribute - if not self._is_reserved_key(key): + if not self._is_reserved_key(key, value): children.append(self._deserialize_attribute(key, val)) return children @@ -294,8 +295,8 @@ def _deserialize_block(self, first_label: str, value: dict) -> BlockRule: body = value # Keep peeling off single-key layers until we hit the body (dict with IS_BLOCK) - while isinstance(body, dict) and not body.get(IS_BLOCK): - non_block_keys = [k for k in body.keys() if not self._is_reserved_key(k)] + while isinstance(body, dict) and not self._is_marked_block(body): + non_block_keys = [k for k in body.keys() if not self._is_reserved_key(k, body)] if len(non_block_keys) == 1: # This is another label level label = non_block_keys[0] @@ -367,10 +368,23 @@ def _deserialize_object_elem(self, key: Any, value: Any) -> ObjectElemRule: return ObjectElemRule(result) - def _is_reserved_key(self, key: str) -> bool: - """Check if a key is a reserved metadata key that should be skipped during deserialization.""" + def _is_reserved_key(self, key: str, container: Optional[dict] = None) -> bool: + """Whether *key* in *container* is metadata rather than an attribute. + + A container carrying its metadata beside the mapping reserves nothing: + every key in it is an attribute the document declared, including one + spelled `__is_block__`. Only the in-band form has to reserve the names, + and only there can it lose an attribute to one. + """ + if container is not None and meta_of(container) is not None: + return False return key in (IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY) + def _is_marked_block(self, body: dict) -> bool: + """Whether *body* is itself a block, in whichever form marks it.""" + meta = meta_of(body) + return meta.is_block if meta is not None else bool(body.get(IS_BLOCK)) + def _is_expression(self, value: Any) -> bool: return isinstance(value, str) and value.startswith("${") and value.endswith("}") @@ -387,8 +401,12 @@ def _is_block(self, value: Any) -> bool: return False def _contains_block_marker(self, obj: dict) -> bool: - """Recursively check if a dict contains IS_BLOCK marker anywhere""" - if obj.get(IS_BLOCK): + """Recursively check whether a dict is marked as a block, in either form""" + meta = meta_of(obj) + if meta is not None: + if meta.is_block: + return True + elif obj.get(IS_BLOCK): return True for value in obj.values(): if isinstance(value, dict) and self._contains_block_marker(value): diff --git a/hcl2/meta.py b/hcl2/meta.py new file mode 100644 index 00000000..89c1ccee --- /dev/null +++ b/hcl2/meta.py @@ -0,0 +1,137 @@ +"""Out-of-band metadata for serialized bodies. + +The serializer has three things to say about a body that are not attributes of +it: that it is a block, what comments surround it, and which of those were +inline. They have always travelled as `__is_block__`, `__comments__` and +`__inline_comments__` keys in the same dict as the attributes, which works only +while no document declares an attribute by those names. HCL puts no such name +out of reach, so one that does loses either the attribute or the metadata, +silently and in both directions. + +`HclDict` carries them beside the mapping instead. It is a `dict`, so every +consumer that reads attributes keeps working unchanged, and `hcl_meta` holds +what used to sit among them. +""" + +import copy as copy_module +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + + +@dataclass +class HclMeta: + """What the serializer knows about a body that is not one of its attributes.""" + + is_block: bool = False + comments: List[dict] = field(default_factory=list) + inline_comments: List[dict] = field(default_factory=list) + + def is_empty(self) -> bool: + """Whether there is nothing here worth carrying.""" + return not (self.is_block or self.comments or self.inline_comments) + + +class HclDict(Dict[str, Any]): + """A dict whose HCL metadata lives on the object rather than among the keys. + + Equality, iteration, `json.dumps` and every other mapping operation behave + exactly as `dict` does -- the metadata is deliberately not part of the + mapping, so a document declaring an attribute called `__is_block__` gets + that attribute back and nothing else. + + JSON cannot carry the sidecar. Serializing an `HclDict` yields the + attributes alone, which is why the in-band keys remain the default. + """ + + __slots__ = ("hcl_meta",) + + def __init__(self, *args: Any, meta: Optional[HclMeta] = None) -> None: + """Build from a mapping, with the metadata passed separately. + + No `**kwargs`: this is the one class whose whole point is that no key + name is reserved, and taking keyword items would reserve `meta` -- + `HclDict(**{"meta": "prod"})` would swallow the attribute and store a + string where the metadata goes. `meta` is a real name in real configs. + Pass the mapping positionally, as `dict` also allows. + """ + super().__init__(*args) + if meta is not None and not isinstance(meta, HclMeta): + raise TypeError( + "HclDict(meta=...) takes an HclMeta; to store a key called " + f"'meta', pass the mapping positionally: HclDict({{'meta': {meta!r}}})" + ) + self.hcl_meta = meta if meta is not None else HclMeta() + + def __repr__(self) -> str: + """Show the metadata, so a debugging session does not have to guess.""" + if self.hcl_meta.is_empty(): + return super().__repr__() + return f"{super().__repr__()} + {self.hcl_meta!r}" + + def copy(self) -> "HclDict": + """Copy the mapping and the metadata together. + + `dict.copy` returns a plain `dict`, which would drop the sidecar -- + and `document = document.copy()` is ordinary enough that losing block + metadata to it would be a trap. The in-band form survives a copy + because its metadata is among the keys; this has to say so explicitly. + """ + return HclDict(self, meta=copy_module.copy(self.hcl_meta)) + + def __copy__(self) -> "HclDict": + """Same for `copy.copy`.""" + return self.copy() + + def __deepcopy__(self, memo: dict) -> "HclDict": + """Same for `copy.deepcopy`, metadata included. + + The duplicate is recorded in *memo* before anything inside it is + copied. A mapping may hold a reference back to itself, and copying + the children first means the recursion reaches this dict again with + nothing recorded, which does not terminate. `dict` registers its own + copy first for that reason; a subclass that did not would make a + cyclic document worse than the plain mapping it replaces. + """ + duplicate = HclDict() + memo[id(self)] = duplicate + duplicate.hcl_meta = copy_module.deepcopy(self.hcl_meta, memo) + for key, value in self.items(): + duplicate[copy_module.deepcopy(key, memo)] = copy_module.deepcopy(value, memo) + return duplicate + + def __reduce__(self) -> Tuple[Any, ...]: + """Carry the metadata through pickling, which `dict` would not.""" + return (_rebuild, (dict(self), self.hcl_meta)) + + def __or__(self, other: Any) -> "HclDict": # type: ignore[override] + """Merge, keeping this side's metadata. + + Narrower than `dict.__or__`, which is declared to return `dict` for any + mapping: this always returns an `HclDict`, so the ignore records a + deliberate narrowing rather than a mismatch. + + `dict.__or__` returns a plain `dict`, so `body | {"size": ...}` -- the + idiomatic non-mutating edit -- would drop the sidecar and the block + would then be written as an object. `{**body, ...}` cannot be helped: + unpacking always builds a plain `dict`, and there is no hook for it. + """ + merged = HclDict(self, meta=copy_module.copy(self.hcl_meta)) + merged.update(other) + return merged + + def __ror__(self, other: Any) -> "HclDict": # type: ignore[override] + """Same from the left, keeping this side's metadata.""" + merged = HclDict(other, meta=copy_module.copy(self.hcl_meta)) + merged.update(self) + return merged + + +def meta_of(value: Any) -> Optional[HclMeta]: + """Return the metadata carried beside *value*, or None if it carries none.""" + meta = getattr(value, "hcl_meta", None) + return meta if isinstance(meta, HclMeta) else None + + +def _rebuild(items: Dict[str, Any], meta: HclMeta) -> HclDict: + """Reconstruct an `HclDict` from its pickled parts.""" + return HclDict(items, meta=meta) diff --git a/hcl2/query/blocks.py b/hcl2/query/blocks.py index 269f2209..0142b234 100644 --- a/hcl2/query/blocks.py +++ b/hcl2/query/blocks.py @@ -3,6 +3,7 @@ from typing import Any, List, Optional from hcl2.const import COMMENTS_KEY +from hcl2.meta import meta_of from hcl2.query._base import NodeView, register_view from hcl2.rules.abstract import LarkElement from hcl2.rules.base import BlockRule @@ -72,8 +73,18 @@ def to_dict(self, options: Optional[SerializationOptions] = None) -> Any: ): # Place adjacent comments at the outer level of the block dict, # alongside the label keys — not drilled into the body dict. - existing = result.get(COMMENTS_KEY, []) - result[COMMENTS_KEY] = self._adjacent_comments + existing + # + # Whichever form the serializer used: writing the in-band key onto + # a dict carrying a sidecar would put it back among the attributes, + # where nothing reserves it any more, and `dumps` would emit it as + # real HCL. Reading `result.get(COMMENTS_KEY)` there would also + # find nothing, because the block's own comments are in the meta. + meta = meta_of(result) + if meta is not None: + meta.comments = self._adjacent_comments + meta.comments + else: + existing = result.get(COMMENTS_KEY, []) + result[COMMENTS_KEY] = self._adjacent_comments + existing return result def blocks(self, block_type: Optional[str] = None, *labels: str) -> List["NodeView"]: diff --git a/hcl2/rules/base.py b/hcl2/rules/base.py index 625bd835..367a83f1 100644 --- a/hcl2/rules/base.py +++ b/hcl2/rules/base.py @@ -5,7 +5,8 @@ from lark.tree import Meta -from hcl2.const import INLINE_COMMENTS_KEY, IS_BLOCK +from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK +from hcl2.meta import HclDict, HclMeta, meta_of from hcl2.rules.abstract import LarkRule, LarkToken from hcl2.rules.expressions import ExprTermRule from hcl2.rules.literal_rules import IdentifierRule @@ -87,9 +88,16 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext if child_comments: comments.extend(child_comments) + if options.metadata_sidecar: + meta = HclMeta() + if options.with_comments: + meta.comments = comments + meta.inline_comments = inline_comments + return HclDict(result.items(), meta=meta) + if options.with_comments: if comments: - result["__comments__"] = comments + result[COMMENTS_KEY] = comments if inline_comments: result[INLINE_COMMENTS_KEY] = inline_comments @@ -151,7 +159,11 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext """Serialize to a nested dict with labels as keys.""" result = self._body.serialize(options) if options.explicit_blocks: - result.update({IS_BLOCK: True}) + meta = meta_of(result) + if meta is not None: + meta.is_block = True + else: + result.update({IS_BLOCK: True}) labels = self._labels for label in reversed(labels[1:]): diff --git a/hcl2/rules/containers.py b/hcl2/rules/containers.py index 8b811ce8..25eb9d3d 100644 --- a/hcl2/rules/containers.py +++ b/hcl2/rules/containers.py @@ -2,6 +2,7 @@ from typing import Any, List, Optional, Tuple, Union +from hcl2.meta import HclDict from hcl2.rules.abstract import LarkRule from hcl2.rules.expressions import ExpressionRule from hcl2.rules.literal_rules import ( @@ -192,6 +193,13 @@ def serialize(self, options=SerializationOptions(), context=SerializationContext dict_result: dict = {} for element in self.elements: dict_result.update(element.serialize(options, context)) + if options.metadata_sidecar: + # An object literal has no metadata of its own, but it has to + # say so in the same form a body does. Left a plain dict, a key + # the document wrote as `__is_block__` reads back as the marker + # and the object is emitted as a block -- which is the very + # collision the option exists to remove. + return HclDict(dict_result) return dict_result with context.modify(inside_dollar_string=True): diff --git a/hcl2/utils.py b/hcl2/utils.py index 6e79f007..6d9a2a90 100644 --- a/hcl2/utils.py +++ b/hcl2/utils.py @@ -43,6 +43,16 @@ class SerializationOptions: # producing backwards-compatible output (e.g. "hello" instead of '"hello"'). # Note: round-trip through from_dict/dumps is NOT supported WITH this option. strip_string_quotes: bool = False + # Appended rather than grouped with the other block options on purpose: + # this dataclass is not `kw_only`, so inserting a field anywhere else + # silently changes what every positional argument after it means. + # + # Carry the metadata keys beside the mapping instead of among its keys, as + # `HclDict.hcl_meta`. The in-band keys collide with any attribute a document + # happens to name `__is_block__`, `__comments__` or `__inline_comments__`; + # the sidecar cannot. Off by default because the keys are a documented part + # of the output shape, and because JSON cannot carry the sidecar. + metadata_sidecar: bool = False _SIMPLE_ESCAPES = { diff --git a/test/unit/test_metadata_sidecar.py b/test/unit/test_metadata_sidecar.py new file mode 100644 index 00000000..adb12b4c --- /dev/null +++ b/test/unit/test_metadata_sidecar.py @@ -0,0 +1,343 @@ +# pylint: disable=C0103,C0114,C0115,C0116 +"""Metadata carried beside the mapping instead of among its keys (GH #331). + +`__is_block__`, `__comments__` and `__inline_comments__` are the serializer's, +but the names are not reserved in HCL: a document may declare an attribute +called any of them. In-band, one of the two has to lose -- on read the marker +overwrites the attribute, and on write `_is_reserved_key` drops it -- and the +caller cannot tell which happened, because by then the dict holds one value. + +`metadata_sidecar=True` puts the three on the object instead. The mapping then +holds attributes and nothing else, so there is nothing to collide with. +""" + +import copy +import dataclasses +import json +import pickle +from unittest import TestCase + +from hcl2.api import dumps, loads +from hcl2.const import COMMENTS_KEY, INLINE_COMMENTS_KEY, IS_BLOCK +from hcl2.meta import HclDict, HclMeta, meta_of +from hcl2.utils import SerializationOptions + +SIDECAR = SerializationOptions(metadata_sidecar=True) + +RESERVED = (IS_BLOCK, COMMENTS_KEY, INLINE_COMMENTS_KEY) + + +class TestAnAttributeNamedLikeMetadataSurvives(TestCase): + def test_it_is_read_as_an_attribute(self): + for key in RESERVED: + with self.subTest(key=key): + body = loads( + f'resource "a" "b" {{\n {key} = 99\n keep = 1\n}}\n', + serialization_options=SIDECAR, + )["resource"][0]['"a"']['"b"'] + self.assertEqual(body[key], 99) + self.assertEqual(body["keep"], 1) + + def test_it_survives_a_round_trip(self): + for key in RESERVED: + with self.subTest(key=key): + source = f'resource "a" "b" {{\n {key} = 99\n keep = 1\n}}\n' + written = dumps(loads(source, serialization_options=SIDECAR)) + self.assertIn(key, written) + self.assertIn("keep", written) + + def test_in_band_still_loses_it(self): + # The behaviour the option exists to avoid, pinned so the difference + # between the two modes stays visible. + written = dumps(loads('resource "a" "b" {\n __is_block__ = 99\n keep = 1\n}\n')) + self.assertNotIn("__is_block__", written) + self.assertIn("keep", written) + + +class TestTheMetadataItself(TestCase): + SOURCE = '# lead\nresource "a" "b" {\n x = 1 # trailing\n}\n' + + def test_a_block_is_marked_on_the_object(self): + body = loads(self.SOURCE, serialization_options=SIDECAR)["resource"][0]['"a"']['"b"'] + self.assertTrue(meta_of(body).is_block) + self.assertNotIn(IS_BLOCK, body) + + def test_comments_match_what_the_in_band_form_carries(self): + side = loads(self.SOURCE, serialization_options=SIDECAR) + in_band = loads(self.SOURCE) + self.assertEqual(meta_of(side).comments, in_band[COMMENTS_KEY]) + body = side["resource"][0]['"a"']['"b"'] + in_band_body = in_band["resource"][0]['"a"']['"b"'] + self.assertEqual(meta_of(body).comments, in_band_body[COMMENTS_KEY]) + + def test_a_document_without_metadata_carries_an_empty_one(self): + document = loads("x = 1\n", serialization_options=SIDECAR) + self.assertTrue(meta_of(document).is_empty()) + + +class TestItIsStillADict(TestCase): + """Anything that reads attributes must not notice the change.""" + + def setUp(self): + self.body = loads('resource "a" "b" {\n x = 1\n}\n', serialization_options=SIDECAR)["resource"][0][ + '"a"' + ]['"b"'] + + def test_equality_ignores_the_sidecar(self): + self.assertEqual(self.body, {"x": 1}) + + def test_json_serializes_the_attributes_alone(self): + # JSON cannot carry the sidecar, which is why the in-band keys remain + # the default rather than being replaced. + self.assertEqual(json.loads(json.dumps(self.body)), {"x": 1}) + + def test_it_is_a_dict(self): + self.assertIsInstance(self.body, dict) + self.assertEqual(list(self.body), ["x"]) + + +class TestBothFormsAreAccepted(TestCase): + """`dumps` reads whichever form it is handed, including a hand-built dict.""" + + IN_BAND = {"resource": [{'"aws_instance"': {'"web"': {IS_BLOCK: True, "ami": '"ami-1"'}}}]} + SIDECAR_DICT = { + "resource": [{'"aws_instance"': {'"web"': HclDict({"ami": '"ami-1"'}, meta=HclMeta(is_block=True))}}] + } + EXPECTED = 'resource "aws_instance" "web" {\n ami = "ami-1"\n}\n' + + def test_a_legacy_in_band_dict_still_writes(self): + self.assertEqual(dumps(self.IN_BAND), self.EXPECTED) + + def test_a_sidecar_dict_writes_the_same(self): + self.assertEqual(dumps(self.SIDECAR_DICT), dumps(self.IN_BAND)) + self.assertEqual(dumps(self.SIDECAR_DICT), self.EXPECTED) + + def test_a_nested_block_keeps_its_own_metadata(self): + source = 'resource "a" "b" {\n net {\n i = 0\n }\n}\n' + outer = loads(source, serialization_options=SIDECAR)["resource"][0]['"a"']['"b"'] + inner = outer["net"][0] + self.assertTrue(meta_of(outer).is_block) + self.assertTrue(meta_of(inner).is_block) + self.assertEqual(dumps(loads(source, serialization_options=SIDECAR)), source) + + +class TestTheDefaultIsUnchanged(TestCase): + def test_metadata_still_arrives_in_band(self): + body = loads('resource "a" "b" {\n x = 1\n}\n')["resource"][0]['"a"']['"b"'] + self.assertTrue(body[IS_BLOCK]) + self.assertIsNone(meta_of(body)) + + +class TestCopyingCarriesTheSidecar(TestCase): + """`document.copy()` is ordinary enough that losing metadata to it is a trap. + + `dict.copy` returns a plain `dict`, so an inherited copy would drop the + sidecar and the block would then be written as an object. The in-band form + survives a copy for free, because its metadata is among the keys; this has + to say so explicitly. + """ + + def setUp(self): + self.document = loads('resource "a" "b" {\n x = 1\n}\n', serialization_options=SIDECAR) + self.body = self.document["resource"][0]['"a"']['"b"'] + + def test_the_dict_method(self): + duplicate = self.body.copy() + self.assertIsInstance(duplicate, HclDict) + self.assertTrue(meta_of(duplicate).is_block) + + def test_copy_copy(self): + self.assertTrue(meta_of(copy.copy(self.body)).is_block) + + def test_copy_deepcopy(self): + duplicate = copy.deepcopy(self.body) + self.assertTrue(meta_of(duplicate).is_block) + self.assertIsNot(meta_of(duplicate), meta_of(self.body)) + + def test_pickle(self): + self.assertTrue(meta_of(pickle.loads(pickle.dumps(self.body))).is_block) + + def test_a_copied_document_still_writes_a_block(self): + copied = copy.deepcopy(self.document) + self.assertEqual(dumps(copied), dumps(self.document)) + + def test_dict_of_it_is_a_plain_dict(self): + # Deliberate: asking for a `dict` gives the mapping, nothing else. + plain = dict(self.body) + self.assertIsNone(meta_of(plain)) + self.assertEqual(plain, {"x": 1}) + + +class TestAnObjectLiteralIsCoveredToo(TestCase): + """The collision is not specific to block bodies. + + Only `BodyRule` was taught the sidecar at first, so `x = { __is_block__ = + true }` still tripped the in-band branch: the object was read as a block + and `dumps` emitted `x = keep = 1`, which is not HCL at all. An object + literal carries no metadata of its own, but it has to say so in the same + form a body does. + """ + + def _round_trip(self, source: str) -> str: + return dumps(loads(source, serialization_options=SIDECAR)) + + def test_each_reserved_name_survives_as_a_key(self): + for key in RESERVED: + with self.subTest(key=key): + written = self._round_trip(f"x = {{\n {key} = 99\n keep = 1\n}}\n") + self.assertIn(key, written) + self.assertIn("keep", written) + + def test_an_ordinary_object_is_unchanged(self): + self.assertEqual(self._round_trip("x = {\n a = 1\n}\n"), "x = {\n a = 1,\n}\n") + + +class TestTheOptionDidNotMoveTheOtherOnes(TestCase): + """`SerializationOptions` is not `kw_only`, so field order is a contract. + + Inserting the new field among the block options changed what every + positional argument after it meant -- silently, with no exception and no + test to catch it. It is appended instead. + """ + + def test_metadata_sidecar_is_last(self): + names = [f.name for f in dataclasses.fields(SerializationOptions)] + self.assertEqual(names[-1], "metadata_sidecar") + + def test_the_earlier_fields_keep_their_positions(self): + options = SerializationOptions(True, False, False, False, True, False, False, True, False) + self.assertFalse(options.force_operation_parentheses) + self.assertTrue(options.preserve_scientific_notation) + self.assertFalse(options.metadata_sidecar) + + +class TestTheQueryLayerWritesToTheSidecar(TestCase): + """`BlockView.to_dict` merges adjacent comments, and has to pick the form. + + Writing the in-band key onto a dict carrying a sidecar put it back among + the attributes, where nothing reserves it any more -- so `dumps` emitted + `__comments__ = [...]` as real HCL, which does not re-parse. Reading the + in-band key there also found nothing, because the block's own comments had + moved to the meta, so neither list was complete. + """ + + SOURCE = '# lead comment\nterraform {\n required_version = ">= 1.0"\n}\n' + + def _to_dict(self, options): + from hcl2.query import DocumentView + + return DocumentView.parse(self.SOURCE).blocks("terraform")[0].to_dict(options=options) + + def test_the_comment_lands_in_the_meta(self): + body = self._to_dict(SerializationOptions(metadata_sidecar=True, with_comments=True)) + self.assertEqual(meta_of(body).comments, [{"value": "lead comment"}]) + self.assertNotIn(COMMENTS_KEY, body) + + def test_the_block_still_writes_as_a_block(self): + body = self._to_dict(SerializationOptions(metadata_sidecar=True, with_comments=True)) + self.assertEqual(dumps({"terraform": [body]}), 'terraform {\n required_version = ">= 1.0"\n}\n') + + def test_the_in_band_form_is_unchanged(self): + body = self._to_dict(SerializationOptions(with_comments=True)) + self.assertEqual(body[COMMENTS_KEY], [{"value": "lead comment"}]) + + +class TestTheTypeIsPartOfThePublicSurface(TestCase): + """The CHANGELOG makes `HclDict` part of the contract, so it has to be reachable.""" + + def test_it_is_exported(self): + import hcl2 + + self.assertIs(hcl2.HclDict, HclDict) + self.assertIs(hcl2.HclMeta, HclMeta) + self.assertIs(hcl2.meta_of, meta_of) + + +class TestNoKeyNameIsReserved(TestCase): + """Including `meta`, which the constructor would otherwise have taken. + + `meta` is a real attribute name in real configs -- Nomad `meta` stanzas, + provider `meta` blocks -- and a class whose purpose is that no key name is + reserved cannot quietly reserve one. Taking keyword items would have: + `HclDict(**{"meta": "prod"})` swallowed the attribute and stored a string + where the metadata goes, and `repr` then raised `AttributeError`. + """ + + def test_a_key_called_meta_is_kept(self): + body = HclDict({"meta": '"prod"', "ami": '"a"'}, meta=HclMeta(is_block=True)) + self.assertEqual(body["meta"], '"prod"') + self.assertTrue(meta_of(body).is_block) + + def test_a_document_declaring_it_round_trips(self): + written = dumps(loads('block "a" {\n meta = "prod"\n}\n', serialization_options=SIDECAR)) + self.assertIn("meta", written) + + def test_passing_something_else_as_meta_is_refused(self): + with self.assertRaises(TypeError): + HclDict({"a": 1}, meta="prod") + + +class TestMergingKeepsTheSidecar(TestCase): + """`body | {...}` is the idiomatic non-mutating edit.""" + + def setUp(self): + self.body = loads('resource "aws_instance" "web" {\n ami = "a"\n}\n', serialization_options=SIDECAR)[ + "resource" + ][0]['"aws_instance"']['"web"'] + + def test_or_keeps_it(self): + merged = self.body | {"size": '"t2.micro"'} + self.assertTrue(meta_of(merged).is_block) + self.assertEqual(merged["size"], '"t2.micro"') + + def test_ror_keeps_it(self): + merged = {"first": 1} | self.body + self.assertTrue(meta_of(merged).is_block) + + def test_unpacking_cannot_keep_it(self): + # `{**body}` always builds a plain dict and there is no hook for it. + # Stated rather than left to be discovered. + self.assertIsNone(meta_of({**self.body})) + + +class TestDeepcopyHandlesACycle(TestCase): + """`dict` copies a self-referencing mapping; a subclass that did not would + make cyclic structures worse than the mapping it replaces. + + `copy.deepcopy` passes a memo so that a value reached twice is copied once. + Registering the duplicate in it has to happen before the children are + copied: a child holding a reference back to this dict otherwise arrives + with nothing memoised, and the descent does not terminate. + """ + + def test_a_self_reference_is_copied_rather_than_recursed(self): + body = HclDict({"x": 1}, meta=HclMeta(is_block=True)) + body["self"] = body + + duplicate = copy.deepcopy(body) + + self.assertIsNot(duplicate, body) + self.assertIs(duplicate["self"], duplicate) + self.assertEqual(duplicate["x"], 1) + self.assertTrue(meta_of(duplicate).is_block) + + def test_two_dicts_referring_to_each_other(self): + first = HclDict({"name": "first"}, meta=HclMeta(is_block=True)) + second = HclDict({"name": "second"}) + first["other"] = second + second["other"] = first + + duplicate = copy.deepcopy(first) + + self.assertIs(duplicate["other"]["other"], duplicate) + self.assertEqual(duplicate["other"]["name"], "second") + self.assertTrue(meta_of(duplicate).is_block) + + def test_a_dict_reached_twice_is_copied_once(self): + shared = HclDict({"n": 1}) + body = HclDict({"a": shared, "b": shared}, meta=HclMeta(is_block=True)) + + duplicate = copy.deepcopy(body) + + self.assertIs(duplicate["a"], duplicate["b"]) + self.assertIsNot(duplicate["a"], shared)