From c79d84fdff52abf818e58dd4736eb1c26d7de792 Mon Sep 17 00:00:00 2001 From: Jonathan Kaczynski Date: Fri, 4 Sep 2026 13:57:10 -0400 Subject: [PATCH] fix: accept HCL keywords as block labels and object keys HCL does not reserve its keywords, so `in`, `for`, `true`, etc. are legal identifiers. Two positions rejected them. Block labels. `block : identifier ...` where `identifier : NAME`, but `in` lexes as the dedicated `IN` terminal and never as `NAME`. In a body position the contextual lexer therefore only accepted `IN` as the start of an *attribute*, so the Snowflake provider's nested `in` block failed with "Expected one of: * EQ" right after `Token('IN', 'in')`: data "snowflake_schemas" "in" { in { database = "database" } } Object keys. `object_elem_key : expression` cannot reach `keyword` either, which regressed #148 (fixed once by #164, then lost when 8.x moved keywords out of `identifier` into their own rule and wired it only into `_attribute_name`). #164's regression test covered a block-body attribute, a different grammar path from the object element the issue actually reports. The object path then worked only by accident: Lark's contextual lexer falls back to `NAME` only in states that do not accept the keyword terminal, so a key's *position* in the object decided whether the file parsed. { in = "h", name = "n" } # parsed { name = "n", in = "h" } # failed <- the shape in #148 { for = 1 } # failed in every position Both now route through new `_block_label` / `keyword` alternatives, and the transformer normalizes the resulting KeywordRule and LiteralValueRule nodes to IdentifierRule, mirroring what `attribute()` already did. That normalization is load-bearing beyond tidiness: `_label_to_str` in hcl2/query/blocks.py falls through to `str(label.serialize())`, and LiteralValueRule serializes `true` to Python `True`, so a `true`-named block would otherwise query as "True". Fixes #148 Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 +- hcl2/hcl2.lark | 14 +- hcl2/transformer.py | 14 ++ .../hcl2_original/object_keyword_keys.tf | 32 +++++ .../hcl2_original/resource_keyword_block.tf | 43 ++++++ .../hcl2_reconstructed/object_keyword_keys.tf | 31 ++++ .../resource_keyword_block.tf | 62 ++++++++ .../object_keyword_keys.json | 30 ++++ .../resource_keyword_block.json | 96 +++++++++++++ .../json_serialized/object_keyword_keys.json | 30 ++++ .../resource_keyword_block.json | 96 +++++++++++++ test/unit/test_api.py | 132 ++++++++++++++++++ 12 files changed, 582 insertions(+), 3 deletions(-) create mode 100644 test/integration/hcl2_original/object_keyword_keys.tf create mode 100644 test/integration/hcl2_original/resource_keyword_block.tf create mode 100644 test/integration/hcl2_reconstructed/object_keyword_keys.tf create mode 100644 test/integration/hcl2_reconstructed/resource_keyword_block.tf create mode 100644 test/integration/json_reserialized/object_keyword_keys.json create mode 100644 test/integration/json_reserialized/resource_keyword_block.json create mode 100644 test/integration/json_serialized/object_keyword_keys.json create mode 100644 test/integration/json_serialized/resource_keyword_block.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 61dc143a..937b0260 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## \[Unreleased\] -- Nothing yet. +### Fixed + +- Parse blocks whose type or unquoted label is an HCL keyword, such as the `in` block of the Snowflake provider's `snowflake_schemas` data source. HCL does not reserve its keywords, so `if`, `in`, `for`, `for_each`, `else`, `endif`, `endfor`, `true`, `false`, and `null` are now accepted in every block label position and normalized to identifiers — matching the existing behaviour for keyword attribute names. +- Parse keyword-named *object* keys reliably, fixing a regression of [#148](https://github.com/amplify-education/python-hcl2/issues/148). `object_elem_key` did not accept the keyword terminals, so a key such as `in` parsed only in states where the contextual lexer happened to fall back to `NAME` — which made the key's position inside the object decide whether the file parsed. `{ in = "header", name = "n" }` worked while `{ name = "n", in = "header" }` failed, so the `jsonencode` OpenAPI body from the original report still raised. Keys such as `for` failed in every position. ## \[8.1.3\] - 2026-08-26 diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark index 12e55fb9..e9e36a5e 100644 --- a/hcl2/hcl2.lark +++ b/hcl2/hcl2.lark @@ -113,7 +113,11 @@ start : body body : (new_line_or_comment? (attribute | block))* new_line_or_comment? attribute : _attribute_name EQ expression _attribute_name : identifier | keyword | literal_value -block : identifier (identifier | string)* new_line_or_comment? LBRACE body RBRACE +// HCL does not reserve its keywords, so a block type or an unquoted label may +// be spelled `in`, `for`, `true`, etc. (e.g. the `in` block in the Snowflake +// provider). The transformer normalizes those back to identifiers. +block : _block_label (_block_label | string)* new_line_or_comment? LBRACE body RBRACE +_block_label : identifier | keyword | literal_value // Whitespace and comments new_line_or_comment: ( NL_OR_COMMENT )+ @@ -225,7 +229,13 @@ template_string : TEMPLATE_STRING tuple : LSQB new_line_or_comment? (expression new_line_or_comment? COMMA new_line_or_comment?)* (expression new_line_or_comment? COMMA? new_line_or_comment?)? RSQB object : LBRACE new_line_or_comment? ((object_elem | (object_elem new_line_or_comment? COMMA)) new_line_or_comment?)* RBRACE object_elem : object_elem_key ( EQ | COLON ) expression -object_elem_key : expression +// `keyword` is listed explicitly because it is not reachable through +// `expression`: `in`, `for`, etc. lex as their own terminals, never as NAME. +// Without this, `{ type = "apiKey", in = "header" }` fails — the contextual +// lexer only falls back to NAME in states that do not accept the keyword +// terminal, which made the key's position in the object decide whether it +// parsed. The transformer normalizes these to identifiers. +object_elem_key : expression | keyword // Heredocs heredoc_template : HEREDOC_TEMPLATE diff --git a/hcl2/transformer.py b/hcl2/transformer.py index 2d5e9a64..bc8b8097 100644 --- a/hcl2/transformer.py +++ b/hcl2/transformer.py @@ -130,6 +130,15 @@ def body(self, meta: Meta, args) -> BodyRule: @v_args(meta=True) def block(self, meta: Meta, args) -> BlockRule: + # _block_label is flattened, so a label may be a KeywordRule or a + # LiteralValueRule (HCL keywords are not reserved, so `in {}` is a + # legal block). Normalize them so labels are always identifiers. + args = [ + IdentifierRule([NAME(arg.token.value)], meta) + if isinstance(arg, (KeywordRule, LiteralValueRule)) + else arg + for arg in args + ] return BlockRule(args, meta) @v_args(meta=True) @@ -331,6 +340,11 @@ def object_elem(self, meta: Meta, args) -> ObjectElemRule: @v_args(meta=True) def object_elem_key(self, meta: Meta, args): expr = args[0] + # A bare keyword key (`in = "header"`) arrives unwrapped, since + # `keyword` is its own alternative in the grammar rather than being + # reachable through `expression`. Treat it as an identifier key. + if isinstance(expr, KeywordRule): + return ObjectElemKeyRule([IdentifierRule([NAME(expr.token.value)], meta)], meta) # Simple literals (identifier, string, int, float) wrapped in ExprTermRule if isinstance(expr, ExprTermRule) and len(expr.children) == 5: inner = expr.children[2] # position 2 in [None, None, inner, None, None] diff --git a/test/integration/hcl2_original/object_keyword_keys.tf b/test/integration/hcl2_original/object_keyword_keys.tf new file mode 100644 index 00000000..280b9362 --- /dev/null +++ b/test/integration/hcl2_original/object_keyword_keys.tf @@ -0,0 +1,32 @@ +resource "aws_api_gateway_rest_api" "example" { + body = jsonencode({ + security_definitions = { + sigv4 = { + type = "apiKey" + name = "Authorization" + in = "header" + x-amazon-apigateway-authtype = "awsSigv4" + } + } + }) +} + +keywords_in_every_position = { + leading = 0 + if = 1 + in = 2 + for = 3 + for_each = 4 + else = 5 + endif = 6 + endfor = 7 + true = 8 + false = 9 + null = 10 + trailing = 11 +} + +colon_separated = { + a : 0, + in : "header" +} diff --git a/test/integration/hcl2_original/resource_keyword_block.tf b/test/integration/hcl2_original/resource_keyword_block.tf new file mode 100644 index 00000000..54b01ea6 --- /dev/null +++ b/test/integration/hcl2_original/resource_keyword_block.tf @@ -0,0 +1,43 @@ +data "snowflake_schemas" "in" { + in { + database = "database" + } +} + +resource "custom_provider_resource" "resource_name" { + if { + name = "if_block" + } + for { + name = "for_block" + } + for_each { + name = "for_each_block" + } + else { + name = "else_block" + } + endif { + name = "endif_block" + } + endfor { + name = "endfor_block" + } + true { + name = "true_block" + } + false { + name = "false_block" + } + null { + name = "null_block" + } +} + +in "quoted_label" { + attribute = "value" +} + +block in { + attribute = "value" +} diff --git a/test/integration/hcl2_reconstructed/object_keyword_keys.tf b/test/integration/hcl2_reconstructed/object_keyword_keys.tf new file mode 100644 index 00000000..fddbd20b --- /dev/null +++ b/test/integration/hcl2_reconstructed/object_keyword_keys.tf @@ -0,0 +1,31 @@ +resource "aws_api_gateway_rest_api" "example" { + body = jsonencode({ + security_definitions = { + sigv4 = { + type = "apiKey", + name = "Authorization", + in = "header", + x-amazon-apigateway-authtype = "awsSigv4" + } + } + }) +} + +keywords_in_every_position = { + leading = 0, + if = 1, + in = 2, + for = 3, + for_each = 4, + else = 5, + endif = 6, + endfor = 7, + true = 8, + false = 9, + null = 10, + trailing = 11, +} +colon_separated = { + a = 0, + in = "header", +} diff --git a/test/integration/hcl2_reconstructed/resource_keyword_block.tf b/test/integration/hcl2_reconstructed/resource_keyword_block.tf new file mode 100644 index 00000000..59113d63 --- /dev/null +++ b/test/integration/hcl2_reconstructed/resource_keyword_block.tf @@ -0,0 +1,62 @@ +data "snowflake_schemas" "in" { + in { + database = "database" + } +} + + +resource "custom_provider_resource" "resource_name" { + if { + name = "if_block" + } + + + for { + name = "for_block" + } + + + for_each { + name = "for_each_block" + } + + + else { + name = "else_block" + } + + + endif { + name = "endif_block" + } + + + endfor { + name = "endfor_block" + } + + + true { + name = "true_block" + } + + + false { + name = "false_block" + } + + + null { + name = "null_block" + } +} + + +in "quoted_label" { + attribute = "value" +} + + +block in { + attribute = "value" +} diff --git a/test/integration/json_reserialized/object_keyword_keys.json b/test/integration/json_reserialized/object_keyword_keys.json new file mode 100644 index 00000000..0ad1dc64 --- /dev/null +++ b/test/integration/json_reserialized/object_keyword_keys.json @@ -0,0 +1,30 @@ +{ + "resource": [ + { + "\"aws_api_gateway_rest_api\"": { + "\"example\"": { + "body": "${jsonencode({security_definitions = {sigv4 = {type = \"apiKey\", name = \"Authorization\", in = \"header\", x-amazon-apigateway-authtype = \"awsSigv4\"}}})}", + "__is_block__": true + } + } + } + ], + "keywords_in_every_position": { + "leading": 0, + "if": 1, + "in": 2, + "for": 3, + "for_each": 4, + "else": 5, + "endif": 6, + "endfor": 7, + "true": 8, + "false": 9, + "null": 10, + "trailing": 11 + }, + "colon_separated": { + "a": 0, + "in": "\"header\"" + } +} diff --git a/test/integration/json_reserialized/resource_keyword_block.json b/test/integration/json_reserialized/resource_keyword_block.json new file mode 100644 index 00000000..0fe3c31b --- /dev/null +++ b/test/integration/json_reserialized/resource_keyword_block.json @@ -0,0 +1,96 @@ +{ + "data": [ + { + "\"snowflake_schemas\"": { + "\"in\"": { + "in": [ + { + "database": "\"database\"", + "__is_block__": true + } + ], + "__is_block__": true + } + } + } + ], + "resource": [ + { + "\"custom_provider_resource\"": { + "\"resource_name\"": { + "if": [ + { + "name": "\"if_block\"", + "__is_block__": true + } + ], + "for": [ + { + "name": "\"for_block\"", + "__is_block__": true + } + ], + "for_each": [ + { + "name": "\"for_each_block\"", + "__is_block__": true + } + ], + "else": [ + { + "name": "\"else_block\"", + "__is_block__": true + } + ], + "endif": [ + { + "name": "\"endif_block\"", + "__is_block__": true + } + ], + "endfor": [ + { + "name": "\"endfor_block\"", + "__is_block__": true + } + ], + "true": [ + { + "name": "\"true_block\"", + "__is_block__": true + } + ], + "false": [ + { + "name": "\"false_block\"", + "__is_block__": true + } + ], + "null": [ + { + "name": "\"null_block\"", + "__is_block__": true + } + ], + "__is_block__": true + } + } + } + ], + "in": [ + { + "\"quoted_label\"": { + "attribute": "\"value\"", + "__is_block__": true + } + } + ], + "block": [ + { + "in": { + "attribute": "\"value\"", + "__is_block__": true + } + } + ] +} diff --git a/test/integration/json_serialized/object_keyword_keys.json b/test/integration/json_serialized/object_keyword_keys.json new file mode 100644 index 00000000..0ad1dc64 --- /dev/null +++ b/test/integration/json_serialized/object_keyword_keys.json @@ -0,0 +1,30 @@ +{ + "resource": [ + { + "\"aws_api_gateway_rest_api\"": { + "\"example\"": { + "body": "${jsonencode({security_definitions = {sigv4 = {type = \"apiKey\", name = \"Authorization\", in = \"header\", x-amazon-apigateway-authtype = \"awsSigv4\"}}})}", + "__is_block__": true + } + } + } + ], + "keywords_in_every_position": { + "leading": 0, + "if": 1, + "in": 2, + "for": 3, + "for_each": 4, + "else": 5, + "endif": 6, + "endfor": 7, + "true": 8, + "false": 9, + "null": 10, + "trailing": 11 + }, + "colon_separated": { + "a": 0, + "in": "\"header\"" + } +} diff --git a/test/integration/json_serialized/resource_keyword_block.json b/test/integration/json_serialized/resource_keyword_block.json new file mode 100644 index 00000000..0fe3c31b --- /dev/null +++ b/test/integration/json_serialized/resource_keyword_block.json @@ -0,0 +1,96 @@ +{ + "data": [ + { + "\"snowflake_schemas\"": { + "\"in\"": { + "in": [ + { + "database": "\"database\"", + "__is_block__": true + } + ], + "__is_block__": true + } + } + } + ], + "resource": [ + { + "\"custom_provider_resource\"": { + "\"resource_name\"": { + "if": [ + { + "name": "\"if_block\"", + "__is_block__": true + } + ], + "for": [ + { + "name": "\"for_block\"", + "__is_block__": true + } + ], + "for_each": [ + { + "name": "\"for_each_block\"", + "__is_block__": true + } + ], + "else": [ + { + "name": "\"else_block\"", + "__is_block__": true + } + ], + "endif": [ + { + "name": "\"endif_block\"", + "__is_block__": true + } + ], + "endfor": [ + { + "name": "\"endfor_block\"", + "__is_block__": true + } + ], + "true": [ + { + "name": "\"true_block\"", + "__is_block__": true + } + ], + "false": [ + { + "name": "\"false_block\"", + "__is_block__": true + } + ], + "null": [ + { + "name": "\"null_block\"", + "__is_block__": true + } + ], + "__is_block__": true + } + } + } + ], + "in": [ + { + "\"quoted_label\"": { + "attribute": "\"value\"", + "__is_block__": true + } + } + ], + "block": [ + { + "in": { + "attribute": "\"value\"", + "__is_block__": true + } + } + ] +} diff --git a/test/unit/test_api.py b/test/unit/test_api.py index d6599fec..fbd996c0 100644 --- a/test/unit/test_api.py +++ b/test/unit/test_api.py @@ -20,6 +20,7 @@ serialize, transform, ) +from hcl2.const import IS_BLOCK from hcl2.deserializer import DeserializerOptions from hcl2.formatter import FormatterOptions from hcl2.rules.base import StartRule @@ -448,6 +449,137 @@ def test_bare_keywords_are_still_python_values(self): self.assertEqual(loads("x = true\ny = false\nz = null\n"), {"x": True, "y": False, "z": None}) +class TestKeywordBlocks(TestCase): + """HCL does not reserve its keywords, so `in {}` is a legal block. + + The Snowflake provider's `snowflake_schemas` data source nests an `in` + block, and keyword-named block types used to fail to lex because `in` + only ever produced the `IN` terminal. Keywords are accepted in every + label position and normalized to identifiers, matching the long-standing + behaviour for keyword *attribute* names. + """ + + def test_in_block(self): + source = 'data "snowflake_schemas" "in" {\n in {\n database = "database"\n }\n}\n' + inner = {"database": '"database"', IS_BLOCK: True} + self.assertEqual( + loads(source), + {"data": [{'"snowflake_schemas"': {'"in"': {"in": [inner], IS_BLOCK: True}}}]}, + ) + + def test_every_keyword_is_a_valid_block_type(self): + for keyword in ("if", "in", "for", "for_each", "else", "endif", "endfor", "true", "false", "null"): + with self.subTest(keyword=keyword): + self.assertEqual(loads(f"{keyword} {{\n a = 1\n}}\n"), {keyword: [{"a": 1, IS_BLOCK: True}]}) + + def test_keyword_as_quoted_and_unquoted_label(self): + self.assertEqual(loads('block "in" {\n a = 1\n}\n'), {"block": [{'"in"': {"a": 1, IS_BLOCK: True}}]}) + self.assertEqual(loads("block in {\n a = 1\n}\n"), {"block": [{"in": {"a": 1, IS_BLOCK: True}}]}) + + def test_keyword_block_survives_the_direct_pipeline(self): + source = "in {\n for = 1\n}\n" + self.assertEqual(reconstruct(parses(source).to_lark()), source) + + def test_keyword_block_survives_the_dict_pipeline(self): + self.assertEqual(loads(dumps(loads("in {\n a = 1\n}\n"))), {"in": [{"a": 1, IS_BLOCK: True}]}) + + def test_keyword_labels_are_normalized_to_identifiers(self): + """A `true` label must read back as the string "true", not Python `True`.""" + block = query("true in {\n a = 1\n}\n").blocks()[0] + self.assertEqual(block.block_type, "true") + self.assertEqual(block.labels, ["true", "in"]) + + def test_expression_keywords_still_parse(self): + """Accepting keyword labels must not break `for`/`if`/`in` in expressions.""" + self.assertEqual( + loads("x = [for i in [1, 2] : i if i > 1]\n"), {"x": "${[for i in [1, 2] : i if i > 1]}"} + ) + + +class TestKeywordAttributeNames(TestCase): + """`in` as an attribute name — issue #148, fixed by PR #164. + + The report is an OpenAPI body built with `jsonencode`, where `in` names + an *object element*. That is a different grammar path from a block-body + attribute (`object_elem_key` vs `_attribute_name`), and only the latter + got a regression test in #164. The object path then worked by accident: + the contextual lexer falls back to NAME only in states that do not accept + the `IN` terminal, so whether the key parsed depended on its position in + the object. + """ + + # https://github.com/amplify-education/python-hcl2/issues/148 + # The report's own snippet, with its tab indentation normalized to spaces + # (whitespace is ignored here and mixing tabs into the source trips ruff). + ISSUE_148_SOURCE = """resource "aws_api_gateway_rest_api" "example" { + + body = jsonencode({ + security_definitions = { + sigv4 = { + type = "apiKey" + name = "Authorization" + in = "header" + x-amazon-apigateway-authtype = "awsSigv4" + } + } + }) +} +""" + + def test_issue_148_report(self): + body = loads(self.ISSUE_148_SOURCE)["resource"][0]['"aws_api_gateway_rest_api"']['"example"'] + self.assertIn('in = "header"', body["body"]) + + def test_issue_148_report_with_tab_indentation(self): + """The report used tabs; whitespace must not matter.""" + source = self.ISSUE_148_SOURCE.replace(" ", "\t") + body = loads(source)["resource"][0]['"aws_api_gateway_rest_api"']['"example"'] + self.assertIn('in = "header"', body["body"]) + + def test_in_key_in_any_position(self): + """Position in the object must not decide whether the key parses.""" + self.assertEqual(loads('x = {\n in = "h"\n name = "n"\n}\n'), {"x": {"in": '"h"', "name": '"n"'}}) + self.assertEqual(loads('x = {\n name = "n"\n in = "h"\n}\n'), {"x": {"name": '"n"', "in": '"h"'}}) + + def test_every_keyword_is_a_valid_object_key(self): + for keyword in ("if", "in", "for", "for_each", "else", "endif", "endfor", "true", "false", "null"): + with self.subTest(keyword=keyword): + self.assertEqual( + loads(f"x = {{\n a = 0\n {keyword} = 1\n}}\n"), {"x": {"a": 0, keyword: 1}} + ) + + def test_keyword_object_key_with_colon_separator(self): + self.assertEqual(loads('x = {\n a : 0\n in : "h"\n}\n'), {"x": {"a": 0, "in": '"h"'}}) + + # https://github.com/amplify-education/python-hcl2/pull/164 + def test_pr_164_fixture(self): + source = ( + 'resource "custom_provider_resource" "resource_name" {\n' + ' name = "resource_name"\n' + ' attribute = "attribute_value"\n' + ' in = "attribute_value2"\n' + "}\n" + ) + body = loads(source)["resource"][0]['"custom_provider_resource"']['"resource_name"'] + self.assertEqual(body["in"], '"attribute_value2"') + + def test_keyword_object_key_survives_the_direct_pipeline(self): + source = 'x = {\n name = "n"\n in = "h"\n}\n' + self.assertEqual(reconstruct(parses(source).to_lark()), source) + + def test_keyword_object_key_survives_the_dict_pipeline(self): + original = loads('x = {\n name = "n"\n in = "h"\n}\n') + self.assertEqual(loads(dumps(original)), original) + + def test_for_expression_in_is_not_shadowed(self): + """`in` as an object key must not break `in` as the for-expression keyword.""" + self.assertEqual(loads("x = {in = [for i in y : i]}\n"), {"x": {"in": "${[for i in y : i]}"}}) + self.assertEqual( + loads("x = {for k, v in var.m : k => {in = v}}\n"), + {"x": "${{for k, v in var.m : k => {in = v}}}"}, + ) + + class TestStripStringQuotes(TestCase): """`strip_string_quotes=True` asks for values, not source text.