From b1c492f1aadcfb49010d6f7a65d2aad4c2620a47 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 9 Sep 2026 12:00:39 +0000 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20attrs:=20unterminated?= =?UTF-8?q?=20attribute=20group=20no=20longer=20swallows=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `parse()` returned success when the scanner ran off the end of the string without reaching DONE, so callers advanced past the end of the inline range with an empty attribute dict and the group, plus anything after it, never entered the token stream. It now raises `ParseError`, which all three call sites already catch and decline. A `}` also ends a `%` comment now, as the plugin docstring already promises and as djot does, so comments closed only by `}` keep working. --- mdit_py_plugins/attrs/parse.py | 7 +- tests/fixtures/attrs.md | 162 +++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/attrs/parse.py b/mdit_py_plugins/attrs/parse.py index 061574f..0361fff 100644 --- a/mdit_py_plugins/attrs/parse.py +++ b/mdit_py_plugins/attrs/parse.py @@ -100,6 +100,8 @@ def parse(string: str) -> tuple[int, dict[str, str]]: """Parse attributes from start of string. :returns: (length of parsed string, dict of attributes) + :raises ParseError: if the attributes are malformed, + or the string ends before the closing `}` """ pos = 0 state: State = State.START @@ -110,7 +112,7 @@ def parse(string: str) -> tuple[int, dict[str, str]]: return pos, tokens.compile(string) pos = pos + 1 - return pos, tokens.compile(string) + raise ParseError("Attributes not terminated", pos) def handle_start(char: str, pos: int, tokens: TokenState) -> State: @@ -144,6 +146,9 @@ def handle_scanning_comment(char: str, pos: int, tokens: TokenState) -> State: if char == "%": return State.SCANNING + if char == "}": + return State.DONE + return State.SCANNING_COMMENT diff --git a/tests/fixtures/attrs.md b/tests/fixtures/attrs.md index b6d3890..7e3c0b1 100644 --- a/tests/fixtures/attrs.md +++ b/tests/fixtures/attrs.md @@ -70,6 +70,47 @@ a {#a .a c=1}

. +unterminated: block group is not an attribute block +. +{.a +para +. +

{.a +para

+. + +comment: block +. +{%x} +para +. +

para

+. + +comment: block after a class +. +{.a %c} +para +. +

para

+. + +comment: block closed by a percent +. +{% just a comment %} +para +. +

para

+. + +comment: block with a brace inside the comment +. +{% x } y %} +para +. +

para

+. + simple reference link . @@ -134,6 +175,106 @@ merging attributes

a

. +unterminated: inline code +. +`a`{ +. +

a{

+. + +unterminated: inline code with a class +. +`a`{.a +. +

a{.a

+. + +unterminated: link +. +[a](u){ +. +

a{

+. + +unterminated: image +. +![a](u){.x +. +

a{.x

+. + +unterminated: following text is not consumed +. +`a`{.a +more +. +

a{.a +more

+. + +unterminated: group after a terminated group +. +`a`{.a}{ +. +

a{

+. + +unterminated: partially scanned attributes are not applied +. +`a`{ .a b +. +

a{ .a b

+. + +unterminated: escaped quote never re-closed +. +`a`{k="a\} +. +

a{k="a}

+. + +unterminated: inside a link label +. +[x `a`{.a](u) +. +

x a{.a

+. + +comment: inline code +. +`a`{%c} +. +

a

+. + +comment: after a class +. +`a`{.a %c} +. +

a

+. + +comment: link +. +[a](u){%c} +. +

a

+. + +comment: closed by a brace, following text is kept +. +`a`{%a}b +. +

ab

+. + +comment: a brace ends the comment +. +`a`{% x } y %} +. +

a y %}

+. + spans: simple . [a]{#id .b}c @@ -176,6 +317,27 @@ spans: escaped span attribute

[a]{.b}

. +spans: unterminated attributes are not a span +. +[a]{ +. +

[a]{

+. + +spans: unterminated attributes with a class are not a span +. +[a]{.x +. +

[a]{.x

+. + +spans: comment +. +[a]{%c} +. +

a

+. + spans: nested text syntax . [*a*]{.b}c From bf4bbf32b71fe014bb0df7d438d73d981d4009bd Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 9 Sep 2026 12:02:00 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20attrs:=20accumulate?= =?UTF-8?q?=20classes=20across=20groups=20on=20spans=20and=20links?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_attr_inline_rule` looked for an existing class on `state.tokens[-1]`, which for a span or a link is the closing token and never carries attributes, so a second group replaced the first group's classes instead of joining them. It now reads the opening token, the one `_add_attrs` writes to. Concatenation is not de-duplicated, matching the existing image path ("merging attributes" gives `class="a b x x g"`) and djot, so the "spans: merge attributes" row moves from `class="a b"` to `class="a a b"`. --- mdit_py_plugins/attrs/index.py | 4 ++-- tests/fixtures/attrs.md | 44 +++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mdit_py_plugins/attrs/index.py b/mdit_py_plugins/attrs/index.py index 0f608c8..0a3e5db 100644 --- a/mdit_py_plugins/attrs/index.py +++ b/mdit_py_plugins/attrs/index.py @@ -175,8 +175,8 @@ def _attr_inline_rule( state.pos += new_pos + 1 if not silent: attr_token = state.tokens[token_index] - if "class" in attrs and "class" in token.attrs: - attrs["class"] = f"{token.attrs['class']} {attrs['class']}" + if "class" in attrs and "class" in attr_token.attrs: + attrs["class"] = f"{attr_token.attrs['class']} {attrs['class']}" _add_attrs(attr_token, attrs, allowed) return True diff --git a/tests/fixtures/attrs.md b/tests/fixtures/attrs.md index 7e3c0b1..76573cf 100644 --- a/tests/fixtures/attrs.md +++ b/tests/fixtures/attrs.md @@ -400,7 +400,49 @@ spans: merge attributes . [a]{#a .a}{#b .a .b other=c}{other=d} . -

a

+

a

+. + +spans: merge classes from two groups +. +[a]{.x}{.y} +. +

a

+. + +spans: merge classes from three groups +. +[a]{.x}{.y}{.z} +. +

a

+. + +spans: merge classes with an id in each group +. +[a]{.x #p}{.y #q} +. +

a

+. + +spans: merge classes on the outer of nested spans +. +[[a]{.i}]{.x}{.y} +. +

a

+. + +links: merge classes from two groups +. +[a](u){.x}{.y} +. +

a

+. + +links: merge classes from three groups +. +[a](u){.x}{.y}{.z} +. +

a

. Indented by 4 spaces From f76a5f056dd73bf7c21df8e9cd0affeb60af2a62 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 9 Sep 2026 12:03:18 +0000 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20attrs:=20do=20not=20m?= =?UTF-8?q?erge=20block=20attributes=20onto=20a=20closing=20token?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When an attributes block was the last thing inside a container, the token after it was that container's closing token, and merging onto it rendered attributes into a closing tag: `> {.a}` alone in a blockquote produced ``. The merge is now skipped for a token with negative nesting; the attributes block is still popped, so such a group is dropped exactly as one with nothing after it already is. --- mdit_py_plugins/attrs/index.py | 3 +- tests/fixtures/attrs.md | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/attrs/index.py b/mdit_py_plugins/attrs/index.py index 0a3e5db..2e5e621 100644 --- a/mdit_py_plugins/attrs/index.py +++ b/mdit_py_plugins/attrs/index.py @@ -238,7 +238,8 @@ def _attr_resolve_block_rule(state: StateCore, *, allowed: set[str] | None) -> N i += 1 continue - if i + 1 < len_tokens: + # skip closing tokens; attributes would be rendered into the closing tag + if i + 1 < len_tokens and state.tokens[i + 1].nesting >= 0: next_token = state.tokens[i + 1] # classes are appended diff --git a/tests/fixtures/attrs.md b/tests/fixtures/attrs.md index 76573cf..cb579ce 100644 --- a/tests/fixtures/attrs.md +++ b/tests/fixtures/attrs.md @@ -61,6 +61,61 @@ a = 1 . +block: attrs last in a blockquote +. +> {.a} + +para +. +
+

para

+. + +block: attrs last in a list item +. +- {.a} +. +
    +
  • +
+. + +block: attrs last in an ordered list item +. +1. {.a} +. +
    +
  1. +
+. + +block: two attrs blocks last in a blockquote +. +> {.a} +> {.b} +. +
+. + +block: attrs before a blockquote containing only attrs +. +{.a} +{.b} +> {.c} +. +
+. + +block: attrs followed by a paragraph in a blockquote +. +> {.a} +> para +. +
+

para

+
+. + block after paragraph . a From 7684d8ab083f915f82534eea1e1c5eda2f092dc6 Mon Sep 17 00:00:00 2001 From: Chris Sewell Date: Wed, 9 Sep 2026 12:03:56 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20attrs:=20state=20tha?= =?UTF-8?q?t=20backslash=20escapes=20are=20kept=20in=20quoted=20values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `{k="a\"b"}` gives `k` the value `a\"b`: the escape lets the quote through the scanner but is not stripped, and the docstring is the rendered docs page (docs/index.md is `autofunction` only). The old sentence read as though the backslash was removed. The example uses double backticks because RST eats the backslash inside single backticks. A fixture row pins the behaviour. --- mdit_py_plugins/attrs/index.py | 5 ++++- tests/fixtures/attrs.md | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/mdit_py_plugins/attrs/index.py b/mdit_py_plugins/attrs/index.py index 2e5e621..531875e 100644 --- a/mdit_py_plugins/attrs/index.py +++ b/mdit_py_plugins/attrs/index.py @@ -41,7 +41,10 @@ def attrs_plugin( - `key="value"` or `key=value` specifies a key-value attribute. Quotes are not needed when the value consists entirely of ASCII alphanumeric characters or `_` or `:` or `-`. - Backslash escapes may be used inside quoted values. + Backslash escapes may be used inside quoted values, + to allow a ``"`` character within the value. + Note that the backslash is retained in the value: + ``{k="a\\"b"}`` gives ``k`` the value ``a\\"b``. - `%` begins a comment, which ends with the next `%` or the end of the attribute (`}`). Multiple attribute blocks are merged. diff --git a/tests/fixtures/attrs.md b/tests/fixtures/attrs.md index cb579ce..69f2f5c 100644 --- a/tests/fixtures/attrs.md +++ b/tests/fixtures/attrs.md @@ -197,6 +197,13 @@ simple inline code

a

. +quoted value keeps a backslash escape +. +`a`{k="a\"b"} +. +

a

+. + ignore if space . ![a](b) {#id key="*"} From 860e3eeed2240eb3516f9284bb721042bae43e89 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:03:39 +0000 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20FIX:=20attrs:=20a=20brace=20?= =?UTF-8?q?ends=20a=20comment=20only=20when=20no=20`%`=20follows?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit made a `}` end a `%` comment unconditionally. That changed input accepted today: in `{% c } % .b}` the comment is closed by the second `%` and `class="b"` applies, but ending the group at the first `}` dropped the class and, at block level, let a line such as `{% see } below %{x}` be consumed as an empty attribute block instead of rendering as a paragraph. A comment now ends at the next `%`; only when no `%` follows anywhere in the rest of the string does the next `}` end both the comment and the group. Every group that already terminated is unchanged, a comment closed only by `}` still works without running off the end of the string, and text after such a comment is kept. This matches neither reference exactly: djot.js ends a comment at any `}`, the Lua original at none, and both would change accepted input. The docstring is reworded to say what the code does. --- mdit_py_plugins/attrs/index.py | 3 +- mdit_py_plugins/attrs/parse.py | 14 +++++--- tests/fixtures/attrs.md | 58 ++++++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 8 deletions(-) diff --git a/mdit_py_plugins/attrs/index.py b/mdit_py_plugins/attrs/index.py index 531875e..2109ff2 100644 --- a/mdit_py_plugins/attrs/index.py +++ b/mdit_py_plugins/attrs/index.py @@ -45,7 +45,8 @@ def attrs_plugin( to allow a ``"`` character within the value. Note that the backslash is retained in the value: ``{k="a\\"b"}`` gives ``k`` the value ``a\\"b``. - - `%` begins a comment, which ends with the next `%` or the end of the attribute (`}`). + - `%` begins a comment, which ends with the next `%`, + or with the end of the attribute (`}`) if no `%` follows. Multiple attribute blocks are merged. diff --git a/mdit_py_plugins/attrs/parse.py b/mdit_py_plugins/attrs/parse.py index 0361fff..605bbfb 100644 --- a/mdit_py_plugins/attrs/parse.py +++ b/mdit_py_plugins/attrs/parse.py @@ -107,7 +107,16 @@ def parse(string: str) -> tuple[int, dict[str, str]]: state: State = State.START tokens = TokenState() while pos < len(string): - state = HANDLERS[state](string[pos], pos, tokens) + if ( + state == State.SCANNING_COMMENT + and string[pos] == "}" + and "%" not in string[pos + 1 :] + ): + # a comment ends at the next `%`, but there is none left to close it, + # so this `}` ends both the comment and the attributes + state = State.DONE + else: + state = HANDLERS[state](string[pos], pos, tokens) if state == State.DONE: return pos, tokens.compile(string) pos = pos + 1 @@ -146,9 +155,6 @@ def handle_scanning_comment(char: str, pos: int, tokens: TokenState) -> State: if char == "%": return State.SCANNING - if char == "}": - return State.DONE - return State.SCANNING_COMMENT diff --git a/tests/fixtures/attrs.md b/tests/fixtures/attrs.md index 69f2f5c..6523b94 100644 --- a/tests/fixtures/attrs.md +++ b/tests/fixtures/attrs.md @@ -166,6 +166,23 @@ para

para

. +comment: block, attributes after a brace inside the comment +. +{% c } % .b} +para +. +

para

+. + +comment: block, a brace inside the comment does not terminate the group +. +{% see } below %{x} +para +. +

{% see } below %{x} +para

+. + simple reference link . @@ -288,7 +305,7 @@ unterminated: partially scanned attributes are not applied

a{ .a b

. -unterminated: escaped quote never re-closed +unterminated: escaped brace inside a quoted value . `a`{k="a\} . @@ -330,11 +347,39 @@ comment: closed by a brace, following text is kept

ab

. -comment: a brace ends the comment +comment: a brace inside a comment closed by a percent . `a`{% x } y %} . -

a y %}

+

a

+. + +comment: attributes after a brace inside the comment +. +`a`{.a % c } % #i} +. +

a

+. + +comment: an empty comment containing a brace +. +`a`{% } %} +. +

a

+. + +comment: text after a brace-closed comment is kept +. +`a`{.a %c} and {more} +. +

a and {more}

+. + +comment: a later percent keeps a brace-closed comment open +. +`a`{.a %c} and 100% sure +. +

a{.a %c} and 100% sure

. spans: simple @@ -400,6 +445,13 @@ spans: comment

a

. +spans: a brace inside an unterminated comment is not a span +. +[a]{% a } %{b} +. +

[a]{% a } %{b}

+. + spans: nested text syntax . [*a*]{.b}c From 3f5d498a240f420547a10c172cab27f9b2e63383 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:15:24 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=93=9A=20DOCS:=20attrs:=20say=20which?= =?UTF-8?q?=20text=20a=20brace-closed=20comment=20scans=20for=20a=20`%`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous wording, "if no `%` follows", read as "within the attribute". The scan actually covers the rest of the scanned string: the line for a block attribute and the rest of the paragraph for an inline one, so a `%` in later prose keeps a comment open. Say so. --- mdit_py_plugins/attrs/index.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mdit_py_plugins/attrs/index.py b/mdit_py_plugins/attrs/index.py index 2109ff2..124b6d0 100644 --- a/mdit_py_plugins/attrs/index.py +++ b/mdit_py_plugins/attrs/index.py @@ -45,8 +45,10 @@ def attrs_plugin( to allow a ``"`` character within the value. Note that the backslash is retained in the value: ``{k="a\\"b"}`` gives ``k`` the value ``a\\"b``. - - `%` begins a comment, which ends with the next `%`, - or with the end of the attribute (`}`) if no `%` follows. + - `%` begins a comment, which ends with the next `%`. + If no further `%` occurs in the rest of the line (for a block attribute) + or of the paragraph (for an inline attribute), + the comment instead ends at the next `}`, which also ends the attribute. Multiple attribute blocks are merged.