From 2280fb36153d30189c5de8c86af3d8b7ee835bbf Mon Sep 17 00:00:00 2001
From: Tim <code@tim.life>
Date: Tue, 1 Sep 2026 22:44:56 -0700
Subject: [PATCH 1/2] fix: a closing marker may carry trailing whitespace
 (#343)

The spec puts the delimiter "alone on its own line", and Terraform's
scanner ends the heredoc at a line holding the word and nothing else
that matters -- trailing spaces and tabs included. Both terminals
required the newline to follow the word immediately, so `EOF  ` was not
a marker: the heredoc ran on, swallowed the rest of the file, and the
parse failed with an error pointing somewhere else entirely.

Trailing whitespace is invisible, survives copy-paste, and is left by
editors that do not trim it, so a file someone has been running through
Terraform for months could fail here for a reason nothing in the message
suggests.

Only whitespace: `EOFX` and `EOF x` are still body text, which OpenTofu
agrees with -- it reads `<<EOF\nEOF x\nbody\nEOF  \n` as "EOF x\nbody\n"
and closes at the last line.
---
 CHANGELOG.md                                |  4 +-
 hcl2/hcl2.lark                              |  4 +-
 test/unit/test_heredoc_marker_whitespace.py | 67 +++++++++++++++++++++
 3 files changed, 72 insertions(+), 3 deletions(-)
 create mode 100644 test/unit/test_heredoc_marker_whitespace.py

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 61dc143a..798710f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,7 +7,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## \[Unreleased\]
 
-- Nothing yet.
+### Fixed
+
+- A heredoc whose closing marker carries trailing whitespace parses. The spec puts the delimiter "alone on its own line" and Terraform's scanner ends the heredoc at a line holding the word and nothing else that matters, trailing spaces and tabs included; `HEREDOC_TEMPLATE` required the newline to follow the word immediately, so `EOF  ` was not a marker at all -- the heredoc ran on, swallowed the rest of the file, and the parse failed with an error pointing somewhere else. Trailing whitespace is invisible and survives copy-paste, so a file that has been running through Terraform for months could fail here. OpenTofu evaluates `<<EOF\nbody\nEOF  \n` to `"body\n"`. ([#343](https://github.com/amplify-education/python-hcl2/issues/343))
 
 ## \[8.1.3\] - 2026-08-26
 
diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark
index 12e55fb9..3feacb9c 100644
--- a/hcl2/hcl2.lark
+++ b/hcl2/hcl2.lark
@@ -89,8 +89,8 @@ COLONS: "::"
 // to a later marker. The delimiter itself is `[a-zA-Z][a-zA-Z0-9._-]*` — the
 // trailing `*` rather than `+` because the spec's Identifier permits a single
 // character, so `<<E` is valid.
-HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/
-HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/
+HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)[ \t]*\r?\n/
+HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)[ \t]*\r?\n/
 
 // Ignore whitespace (but not newlines, as they're significant in HCL).
 // \r is ignored too so CRLF line endings (\r\n) parse the same as LF: the
diff --git a/test/unit/test_heredoc_marker_whitespace.py b/test/unit/test_heredoc_marker_whitespace.py
new file mode 100644
index 00000000..94029734
--- /dev/null
+++ b/test/unit/test_heredoc_marker_whitespace.py
@@ -0,0 +1,67 @@
+# pylint: disable=C0103,C0114,C0115,C0116
+r"""A closing marker may carry trailing whitespace (GH #343).
+
+The spec puts the delimiter "alone on its own line", and Terraform's scanner
+ends the heredoc at a line holding the word and nothing else that matters --
+trailing spaces and tabs included. `HEREDOC_TEMPLATE` required the newline to
+follow the word immediately, so `EOF  ` was not a marker: the heredoc ran on,
+swallowed the rest of the file, and the parse failed with an error pointing
+somewhere else entirely.
+
+Trailing whitespace is invisible, survives copy-paste, and is left behind by
+editors that do not trim it, so a file someone has been running through
+Terraform for months could fail here.
+
+Checked against OpenTofu v1.12.5: `<<EOF\nbody\nEOF  \n` evaluates to
+`"body\n"` and the attribute after it is read normally.
+"""
+
+from unittest import TestCase
+
+from hcl2.api import loads
+
+
+class TestATrailingSpaceClosesTheHeredoc(TestCase):
+    def test_spaces(self):
+        self.assertEqual(loads("a = <<EOF\nbody\nEOF  \nb = 1\n")["b"], 1)
+
+    def test_a_tab(self):
+        self.assertEqual(loads("a = <<EOF\nbody\nEOF\t\nb = 1\n")["b"], 1)
+
+    def test_mixed(self):
+        self.assertEqual(loads("a = <<EOF\nbody\nEOF \t \nb = 1\n")["b"], 1)
+
+    def test_the_trim_form_too(self):
+        self.assertEqual(loads("a = <<-EOF\n  body\n  EOF  \nb = 1\n")["b"], 1)
+
+    def test_indented_and_trailing_together(self):
+        self.assertEqual(loads("a = <<-EOF\n  body\n  EOF  \nb = 1\n")["a"].startswith('"'), True)
+
+
+class TestWhatIsStillBodyText(TestCase):
+    """Only whitespace is allowed after the word; anything else is content."""
+
+    def test_a_word_with_a_suffix_does_not_close_it(self):
+        # `EOFX` is not the delimiter, so the heredoc continues past it.
+        result = loads("a = <<EOF\nEOFX\nbody\nEOF\nb = 1\n")
+        self.assertIn("EOFX", result["a"])
+        self.assertEqual(result["b"], 1)
+
+    def test_a_marker_with_trailing_text_does_not_close_it(self):
+        result = loads("a = <<EOF\nEOF x\nbody\nEOF\nb = 1\n")
+        self.assertIn("EOF x", result["a"])
+        self.assertEqual(result["b"], 1)
+
+
+class TestNothingElseChanged(TestCase):
+    def test_a_plain_marker_still_works(self):
+        self.assertEqual(loads("a = <<EOF\nbody\nEOF\nb = 1\n")["b"], 1)
+
+    def test_an_indented_marker_still_works(self):
+        self.assertEqual(loads("a = <<EOF\nbody\n   EOF\nb = 1\n")["b"], 1)
+
+    def test_crlf_still_works(self):
+        self.assertEqual(loads("a = <<EOF\r\nbody\r\nEOF\r\nb = 1\r\n")["b"], 1)
+
+    def test_crlf_with_trailing_space(self):
+        self.assertEqual(loads("a = <<EOF\r\nbody\r\nEOF  \r\nb = 1\r\n")["b"], 1)

From 117fafbc409b265957d32d5d564d2602360fb83b Mon Sep 17 00:00:00 2001
From: Kamil Kozik <kkozik@amplify.com>
Date: Mon, 7 Sep 2026 14:12:33 +0200
Subject: [PATCH 2/2] test: pin the marker-whitespace behaviour against
 Terraform (#343)

The suite asserted that the file parsed -- every case checked `b == 1` --
so a regression leaking the marker's trailing whitespace into the heredoc
body would have passed all of it. Assert the value instead, as an equality
against the same heredoc without the trailing whitespace, so the tests stay
honest across the separate fixes to the body-value rules (#326).

Adds the case that changes meaning: a body line consisting of the delimiter
plus trailing whitespace now closes the heredoc rather than being content.
Terraform v1.11.4 reads `a = <<EOF\nEOF  \nb = 1\n` as an empty string
followed by `b = 1`, so closing there is what the reference implementation
does. Adds round-trip assertions too, since the marker line is part of the
token and has to be written back byte for byte.

Drops `assertEqual(value.startswith('"'), True)`, which held for every
serialized string in the library and so tested nothing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 CHANGELOG.md                                |   2 +-
 hcl2/hcl2.lark                              |   5 +
 test/unit/test_heredoc_marker_whitespace.py | 128 ++++++++++++++++++--
 3 files changed, 123 insertions(+), 12 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 798710f7..d2794db7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 
-- A heredoc whose closing marker carries trailing whitespace parses. The spec puts the delimiter "alone on its own line" and Terraform's scanner ends the heredoc at a line holding the word and nothing else that matters, trailing spaces and tabs included; `HEREDOC_TEMPLATE` required the newline to follow the word immediately, so `EOF  ` was not a marker at all -- the heredoc ran on, swallowed the rest of the file, and the parse failed with an error pointing somewhere else. Trailing whitespace is invisible and survives copy-paste, so a file that has been running through Terraform for months could fail here. OpenTofu evaluates `<<EOF\nbody\nEOF  \n` to `"body\n"`. ([#343](https://github.com/amplify-education/python-hcl2/issues/343))
+- Parse heredocs whose closing marker carries trailing spaces or tabs, such as `EOF  `. Both heredoc terminals required the newline to follow the delimiter immediately, so the marker went unrecognised, the heredoc ran on to a later one, and the parse failed pointing at an unrelated line. Terraform ends a heredoc at any line holding the delimiter and nothing else that matters, so such a file parses everywhere else. One input changes meaning: a body line consisting of the delimiter plus trailing whitespace now closes the heredoc rather than being content, as it does in Terraform. Thanks, @livingstaccato ([#349](https://github.com/amplify-education/python-hcl2/pull/349))
 
 ## \[8.1.3\] - 2026-08-26
 
diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark
index 3feacb9c..4edf8afd 100644
--- a/hcl2/hcl2.lark
+++ b/hcl2/hcl2.lark
@@ -89,6 +89,11 @@ COLONS: "::"
 // to a later marker. The delimiter itself is `[a-zA-Z][a-zA-Z0-9._-]*` — the
 // trailing `*` rather than `+` because the spec's Identifier permits a single
 // character, so `<<E` is valid.
+// `[ \t]*` after the closing delimiter accepts the trailing whitespace an
+// editor may leave on that line: Terraform ends the heredoc at a line holding
+// the delimiter and nothing else that matters. Only spaces and tabs, so
+// `EOF x` stays body text. Because the body group is lazy, the earliest such
+// line closes it, which is also where Terraform closes.
 HEREDOC_TEMPLATE : /<<(?P<heredoc>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)[ \t]*\r?\n/
 HEREDOC_TEMPLATE_TRIM : /<<-(?P<heredoc_trim>[a-zA-Z][a-zA-Z0-9._-]*)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)[ \t]*\r?\n/
 
diff --git a/test/unit/test_heredoc_marker_whitespace.py b/test/unit/test_heredoc_marker_whitespace.py
index 94029734..52a4b85c 100644
--- a/test/unit/test_heredoc_marker_whitespace.py
+++ b/test/unit/test_heredoc_marker_whitespace.py
@@ -12,16 +12,34 @@
 editors that do not trim it, so a file someone has been running through
 Terraform for months could fail here.
 
-Checked against OpenTofu v1.12.5: `<<EOF\nbody\nEOF  \n` evaluates to
-`"body\n"` and the attribute after it is read normally.
+Every expectation below was checked against Terraform v1.11.4:
+
+    <<EOF\nbody\nEOF  \n            -> "body\n", and the next attribute reads
+    <<EOF\nEOF  \nb = 1\n           -> "", and b reads as 1
+    <<EOF\nEOFX\nEOF x\nbody\nEOF   -> "EOFX\nEOF x\nbody\n"
+
+The value assertions compare a marker carrying trailing whitespace against the
+same heredoc without it, rather than pinning an absolute string. The two must
+agree whatever the body-value rules are, so stating it as an equality keeps
+these tests honest across the separate fixes to those rules (GH #326).
 """
 
 from unittest import TestCase
 
-from hcl2.api import loads
+from hcl2.api import loads, parses_to_tree, reconstruct, transform
+from hcl2.utils import SerializationOptions
+
+VALUE = SerializationOptions(preserve_heredocs=False, strip_string_quotes=True)
+
+
+def value_of(src: str) -> str:
+    """Return the body that the heredoc assigned to `a` evaluates to."""
+    return loads(src, serialization_options=VALUE)["a"]
 
 
 class TestATrailingSpaceClosesTheHeredoc(TestCase):
+    """The attribute after the heredoc is reached, so the marker closed it."""
+
     def test_spaces(self):
         self.assertEqual(loads("a = <<EOF\nbody\nEOF  \nb = 1\n")["b"], 1)
 
@@ -34,8 +52,61 @@ def test_mixed(self):
     def test_the_trim_form_too(self):
         self.assertEqual(loads("a = <<-EOF\n  body\n  EOF  \nb = 1\n")["b"], 1)
 
-    def test_indented_and_trailing_together(self):
-        self.assertEqual(loads("a = <<-EOF\n  body\n  EOF  \nb = 1\n")["a"].startswith('"'), True)
+
+class TestTheWhitespaceIsNotPartOfTheValue(TestCase):
+    """A marker's trailing whitespace belongs to the marker, not the body."""
+
+    def test_spaces_give_the_same_value_as_no_spaces(self):
+        self.assertEqual(
+            value_of("a = <<EOF\nbody\nEOF  \n"),
+            value_of("a = <<EOF\nbody\nEOF\n"),
+        )
+
+    def test_a_tab_gives_the_same_value(self):
+        self.assertEqual(
+            value_of("a = <<EOF\nbody\nEOF\t\n"),
+            value_of("a = <<EOF\nbody\nEOF\n"),
+        )
+
+    def test_the_trim_form_gives_the_same_value(self):
+        self.assertEqual(
+            value_of("a = <<-EOF\n  body\n  EOF  \n"),
+            value_of("a = <<-EOF\n  body\n  EOF\n"),
+        )
+
+    def test_crlf_gives_the_same_value(self):
+        self.assertEqual(
+            value_of("a = <<EOF\r\nbody\r\nEOF  \r\n"),
+            value_of("a = <<EOF\r\nbody\r\nEOF\r\n"),
+        )
+
+    def test_the_equality_is_not_vacuous(self):
+        # Two empty strings, or two identical failures, would satisfy the
+        # tests above without the marker having been recognised at all.
+        self.assertIn("body", value_of("a = <<EOF\nbody\nEOF  \n"))
+
+    def test_no_part_of_the_marker_reaches_the_value(self):
+        self.assertNotIn("EOF", value_of("a = <<EOF\nbody\nEOF  \n"))
+        self.assertNotIn("EOF", value_of("a = <<-EOF\n  body\n  EOF  \n"))
+
+
+class TestItClosesAtTheFirstSuchLine(TestCase):
+    r"""The line that closes it is body text no longer.
+
+    This is the one input whose meaning changes: `EOF  ` used to be content,
+    because it was not a marker, and is now the marker. Terraform reads
+    `a = <<EOF\nEOF  \nb = 1\n` as an empty string followed by `b = 1`, so
+    closing there is what the reference implementation does.
+    """
+
+    def test_an_immediate_marker_ends_an_empty_heredoc(self):
+        self.assertEqual(value_of("a = <<EOF\nEOF  \n"), "")
+        self.assertEqual(loads("a = <<EOF\nEOF  \nb = 1\n")["b"], 1)
+
+    def test_what_follows_it_is_no_longer_swallowed(self):
+        # Before the fix the heredoc ran on to the second marker and the value
+        # was "EOF  \nmore\n"; now the first line closes it, as Terraform does.
+        self.assertNotIn("more", value_of("a = <<EOF\nEOF  \nmore = 1\n"))
 
 
 class TestWhatIsStillBodyText(TestCase):
@@ -43,14 +114,49 @@ class TestWhatIsStillBodyText(TestCase):
 
     def test_a_word_with_a_suffix_does_not_close_it(self):
         # `EOFX` is not the delimiter, so the heredoc continues past it.
-        result = loads("a = <<EOF\nEOFX\nbody\nEOF\nb = 1\n")
-        self.assertIn("EOFX", result["a"])
-        self.assertEqual(result["b"], 1)
+        self.assertIn("EOFX", value_of("a = <<EOF\nEOFX\nbody\nEOF\n"))
+        self.assertEqual(loads("a = <<EOF\nEOFX\nbody\nEOF\nb = 1\n")["b"], 1)
 
     def test_a_marker_with_trailing_text_does_not_close_it(self):
-        result = loads("a = <<EOF\nEOF x\nbody\nEOF\nb = 1\n")
-        self.assertIn("EOF x", result["a"])
-        self.assertEqual(result["b"], 1)
+        self.assertIn("EOF x", value_of("a = <<EOF\nEOF x\nbody\nEOF\n"))
+        self.assertEqual(loads("a = <<EOF\nEOF x\nbody\nEOF\nb = 1\n")["b"], 1)
+
+    def test_both_together_close_at_the_last_line(self):
+        # Terraform reads this as "EOFX\nEOF x\nbody\n".
+        body = value_of("a = <<EOF\nEOFX\nEOF x\nbody\nEOF  \n")
+        self.assertIn("EOFX", body)
+        self.assertIn("EOF x", body)
+        self.assertIn("body", body)
+
+
+class TestTheSourceSurvivesARoundTrip(TestCase):
+    """Whitespace the marker carries is written back where it was.
+
+    The marker line is part of the heredoc token, so a reconstruct has to
+    reproduce it byte for byte rather than normalise it away.
+    """
+
+    def assert_round_trips(self, src: str):
+        rules = transform(parses_to_tree(src))
+        self.assertEqual(reconstruct(rules.to_lark()), src)
+
+    def test_spaces(self):
+        self.assert_round_trips("a = <<EOF\nbody\nEOF  \n")
+
+    def test_a_tab(self):
+        self.assert_round_trips("a = <<EOF\nbody\nEOF\t\n")
+
+    def test_the_trim_form(self):
+        self.assert_round_trips("a = <<-EOF\n  body\n  EOF  \n")
+
+    def test_an_indented_marker(self):
+        self.assert_round_trips("a = <<EOF\nbody\n   EOF  \n")
+
+    def test_crlf(self):
+        self.assert_round_trips("a = <<EOF\r\nbody\r\nEOF  \r\n")
+
+    def test_alongside_a_later_attribute(self):
+        self.assert_round_trips("a = <<EOF\nbody\nEOF  \nb = 1\n")
 
 
 class TestNothingElseChanged(TestCase):