From 30ac050d35c40c738a6a61a1430ecf8cfef5c167 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 18:34:20 +0800 Subject: [PATCH 1/8] fix: compose release document normalization --- scripts/_checks.py | 75 ++++++++++++--------------------- scripts/check_version.py | 2 +- tests/test_changelog_gate.py | 5 ++- tests/test_release_documents.py | 4 ++ 4 files changed, 35 insertions(+), 51 deletions(-) diff --git a/scripts/_checks.py b/scripts/_checks.py index e40f520..898c1df 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -681,19 +681,22 @@ def _canonical_actions_run_violations( ) -> list[tuple[int, str]]: """Require every Actions URL in immutable evidence to name one canonical run.""" findings: list[tuple[int, str]] = [] - normalized = unicodedata.normalize("NFKC", body) - direct = normalized - for _ in range(3): - decoded = html.unescape(direct) - if decoded == direct: + direct = unicodedata.normalize("NFKC", body) + variants = [direct] + for _ in range(8): + previous = variants[-1] + decoded = html.unescape(previous) + decoded = unquote(decoded) + decoded = re.sub(r"\\([/\\.:?&=%#])", r"\1", decoded) + decoded = decoded.replace("\t", "").replace("\n", "").replace("\r", "") + decoded = re.sub( + r"(?i)(https?://[^\s<>)\]]*)\\([^\s<>)\]]*)", + lambda match: match.group(0).replace("\\", "/"), + decoded, + ) + if decoded == previous: break - direct = decoded - markdown_unescaped = re.sub(r"\\([/\\.:?&=%#])", r"\1", direct) - browser_normalized = re.sub( - r"(?i)(https?://[^\s<>)\]]*)\\([^\s<>)\]]*)", - lambda match: match.group(0).replace("\\", "/"), - markdown_unescaped, - ) + variants.append(decoded) path_matches = list(_ACTIONS_PATH.finditer(direct)) canonical_matches = list(_CANONICAL_ACTIONS_URL.finditer(direct)) @@ -704,39 +707,18 @@ def _canonical_actions_run_violations( for path in path_matches ) - percent_decoded = direct - for _ in range(3): - decoded = unquote(percent_decoded) - if decoded == percent_decoded: - break - percent_decoded = decoded direct_paths = Counter((match.group("run"), match.group("attempt")) for match in path_matches) - decoded_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(percent_decoded) - ) - malformed = malformed or any( - count > direct_paths[identity] for identity, count in decoded_paths.items() - ) - normalized_paths = Counter( - (match.group("run"), match.group("attempt")) for match in _ACTIONS_PATH.finditer(normalized) - ) - malformed = malformed or any( - count > normalized_paths[identity] for identity, count in direct_paths.items() - ) - unescaped_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(markdown_unescaped) - ) - malformed = malformed or any( - count > normalized_paths[identity] for identity, count in unescaped_paths.items() - ) - browser_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(browser_normalized) - ) + variant_paths = [ + Counter( + (match.group("run"), match.group("attempt")) + for match in _ACTIONS_PATH.finditer(variant) + ) + for variant in variants + ] malformed = malformed or any( - count > normalized_paths[identity] for identity, count in browser_paths.items() + count > direct_paths[identity] + for paths in variant_paths[1:] + for identity, count in paths.items() ) if malformed: @@ -748,12 +730,7 @@ def _canonical_actions_run_violations( "/attempts/ suffix", ) ) - run_values = ( - {run for run, _attempt in direct_paths} - | {run for run, _attempt in decoded_paths} - | {run for run, _attempt in unescaped_paths} - | {run for run, _attempt in browser_paths} - ) + run_values = {run for paths in variant_paths for run, _attempt in paths} if run_values - {expected_run}: findings.append( ( diff --git a/scripts/check_version.py b/scripts/check_version.py index 7814c28..5617131 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -269,7 +269,7 @@ def _visible_heading_is_unreleased(label: str) -> bool: for value in label ) words = re.findall(r"[A-Za-z]+", label) - return bool(words and words[0].casefold() == "unreleased") + return any(word.casefold() == "unreleased" for word in words) def _unreleased_heading_lines(text: str) -> list[int]: diff --git a/tests/test_changelog_gate.py b/tests/test_changelog_gate.py index 17bcb57..503e6fa 100644 --- a/tests/test_changelog_gate.py +++ b/tests/test_changelog_gate.py @@ -97,12 +97,15 @@ def test_changelog_parser_rejects_obfuscated_mutable_versions(claim: str) -> Non "## Unreleased", "## \uff35\uff4e\uff52\uff45\uff4c\uff45\uff41\uff53\uff45\uff44", "## Un\u034freleased", + "## Current Unreleased", + "Next release (Unreleased)\n-------------------------", "Unreleased\n----------", "> ## Unreleased", "

Unreleased

", "

Unreleased", "

Unreleased

Archive

", "

Unreleased

", + "

Current Unreleased

", "Intro

Unreleased

", ], ) @@ -155,7 +158,7 @@ def test_changelog_parser_rejects_rendered_unreleased_heading_indentation( "## Un*released", "## [Current](https://example.invalid/Unreleased)", '## Current', - "## `Current Unreleased`", + "## `Current release`", "## Unrelea\u0301sed", ], ) diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index c2a352f..35be217 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -714,6 +714,10 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "https://github.com/cometapi-dev/cometapi-python/actions\\/runs\\/30511373822", "https://github.com/cometapi-dev/cometapi-python/actions\\runs\\30511373822", "https://github.com/cometapi-dev/cometapi-python\actions\runs\30511373822", + "https://github.com/cometapi-dev/cometapi-python%5Cactions%5Cruns%5C30511373822", + "https://evil.example/?next=https://github.com/cometapi-dev/cometapi-python%5Cactions%5Cruns%5C30511373822", + 'Required CI', "https://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822.evil", "https://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822/attempts/0", ], From a940723b9ae9370581397ca6bbae26da320ceb71 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 18:42:12 +0800 Subject: [PATCH 2/8] fix: normalize special-scheme backslashes --- scripts/_checks.py | 2 +- tests/test_release_documents.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/_checks.py b/scripts/_checks.py index 898c1df..2a4fc60 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -690,7 +690,7 @@ def _canonical_actions_run_violations( decoded = re.sub(r"\\([/\\.:?&=%#])", r"\1", decoded) decoded = decoded.replace("\t", "").replace("\n", "").replace("\r", "") decoded = re.sub( - r"(?i)(https?://[^\s<>)\]]*)\\([^\s<>)\]]*)", + r"(?i)(https?:[^\s<>)\]]*)\\([^\s<>)\]]*)", lambda match: match.group(0).replace("\\", "/"), decoded, ) diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index 35be217..c257027 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -718,6 +718,10 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "https://evil.example/?next=https://github.com/cometapi-dev/cometapi-python%5Cactions%5Cruns%5C30511373822", 'Required CI', + "https:\\github.com\\cometapi-dev\\cometapi-python\\actions\\runs\\30511373822", + "https:\\\\github.com\\cometapi-dev\\cometapi-python\\actions\\runs\\30511373822", + "https:/\\github.com\\cometapi-dev\\cometapi-python\\actions\\runs\\30511373822", + "https:\\/github.com\\cometapi-dev\\cometapi-python\\actions\\runs\\30511373822", "https://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822.evil", "https://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822/attempts/0", ], From 8314fa611c891191f336882e91009d6db23aa62b Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 18:47:08 +0800 Subject: [PATCH 3/8] fix: reject wrapped and bidirectional release claims --- scripts/_checks.py | 58 +++++++++++++++++++++++++++++---- scripts/check_version.py | 6 ++++ tests/test_changelog_gate.py | 2 ++ tests/test_release_documents.py | 29 +++++++++++++++++ 4 files changed, 89 insertions(+), 6 deletions(-) diff --git a/scripts/_checks.py b/scripts/_checks.py index 2a4fc60..31dfd7b 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -132,6 +132,7 @@ r")(?=$|[/?#>])" ) _HTTP_URL = re.compile(r"https?://[^\s<>)\]]+", re.IGNORECASE) +_URLISH_TOKEN = re.compile(r"https?:[^\s<>\"']+", re.IGNORECASE) _RECOVERY_TAGS = {"0.1.0a1": "v0.1.0-alpha.1+recovery.1"} _FULL_COMMIT = re.compile(r"(?[1-9]\d*)" + rf"(?[1-9]\d*)" r"(?:/attempts/(?P[1-9]\d*))?" r"(?=$|[\s<>\"')\]}]|[.,;:!?](?=$|\s))" ) +_RAW_HTML_URL_ATTRIBUTE = re.compile( + r"(?is)\b(?:href|src)\s*=\s*(?P['\"])(?P.*?)(?P=quote)" +) + + +def _actions_path_has_canonical_url( + text: str, + path: re.Match[str], + canonical_matches: list[re.Match[str]], +) -> bool: + canonical = next( + ( + match + for match in canonical_matches + if match.start() <= path.start() and path.end() <= match.end() + ), + None, + ) + if canonical is None: + return False + + for attribute in _RAW_HTML_URL_ATTRIBUTE.finditer(text): + if attribute.start("url") <= path.start() and path.end() <= attribute.end("url"): + return _CANONICAL_ACTIONS_URL.fullmatch(attribute.group("url")) is not None + + containing_tokens = [ + match + for match in _URLISH_TOKEN.finditer(text) + if match.start() <= path.start() and path.end() <= match.end() + ] + if containing_tokens and min(match.start() for match in containing_tokens) < canonical.start(): + return False + + if canonical.start() == 0: + return True + boundary = text[canonical.start() - 1] + if boundary.isspace(): + return True + if boundary == "<": + return canonical.start() >= 2 and text[canonical.start() - 2].isspace() + if boundary == "(": + before_boundary = canonical.start() - 2 + return ( + before_boundary < 0 or text[before_boundary].isspace() or text[before_boundary] == "]" + ) + return False + + _WHEEL_DIGEST = re.compile( r"\bwheel\s+sha256\b[^0-9a-f]{0,96}(?P[0-9a-f]{64})(?![0-9a-f])", re.IGNORECASE | re.DOTALL, @@ -551,7 +600,7 @@ def _identity_violations( ), "exact release commit": re.compile(re.escape(identity.commit), re.IGNORECASE), "exact release workflow URL": re.compile( - rf"(?\"')\]}]|[.,;:!?](?=$|\s))" ), @@ -701,12 +750,9 @@ def _canonical_actions_run_violations( path_matches = list(_ACTIONS_PATH.finditer(direct)) canonical_matches = list(_CANONICAL_ACTIONS_URL.finditer(direct)) malformed = any( - not any( - url.start() <= path.start() and path.end() <= url.end() for url in canonical_matches - ) + not _actions_path_has_canonical_url(direct, path, canonical_matches) for path in path_matches ) - direct_paths = Counter((match.group("run"), match.group("attempt")) for match in path_matches) variant_paths = [ Counter( diff --git a/scripts/check_version.py b/scripts/check_version.py index 5617131..02467fc 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -261,6 +261,12 @@ def _close_heading(self) -> None: def _visible_heading_is_unreleased(label: str) -> bool: label = html.unescape(unicodedata.normalize("NFKC", label)) + if any( + unicodedata.bidirectional(value) + in {"LRE", "RLE", "LRO", "RLO", "PDF", "LRI", "RLI", "FSI", "PDI"} + for value in label + ): + return True label = "".join( "" if unicodedata.category(value) == "Cf" diff --git a/tests/test_changelog_gate.py b/tests/test_changelog_gate.py index 503e6fa..2abb6e9 100644 --- a/tests/test_changelog_gate.py +++ b/tests/test_changelog_gate.py @@ -97,6 +97,8 @@ def test_changelog_parser_rejects_obfuscated_mutable_versions(claim: str) -> Non "## Unreleased", "## \uff35\uff4e\uff52\uff45\uff4c\uff45\uff41\uff53\uff45\uff44", "## Un\u034freleased", + "## Un\u202edesaeler\u202c", + "## Un\u2067released\u2069", "## Current Unreleased", "Next release (Unreleased)\n-------------------------", "Unreleased\n----------", diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index c257027..b7a7463 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -700,6 +700,12 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "actions/runs/30511373822", "https://evil.example/#https://github.com/cometapi-dev/cometapi-python/" "actions/runs/30511373822", + "https://evil.example/?next=(https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30515861246)", + "https://evil.example/#(https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30515861246)", + "mailto:(https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246)", + "prefix", "http://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822", "https://evil.example/?next=https%3A%2F%2Fgithub.com%2Fcometapi-dev%2F" "cometapi-python%2Factions%2Fruns%2F30511373822", @@ -748,6 +754,29 @@ def test_release_evidence_rejects_noncanonical_workflow_reference_url( assert "/actions/runs/" in str(caught.value) +def test_release_evidence_accepts_canonical_raw_html_anchor( + releasable_documents: Path, +) -> None: + evidence = _release_evidence_block().replace( + "- Release workflow https://github.com/cometapi-dev/cometapi-python/actions/runs/" + "30515861246", + '- Release workflow canonical publication run', + 1, + ) + for name in ("ROADMAP.md", "RELEASING.md"): + with (releasable_documents / name).open("a", encoding="utf-8") as stream: + stream.write(evidence) + _replace( + releasable_documents, + "CHANGELOG.md", + "# Changelog\n", + "# Changelog\n\n## [0.1.2] - 2026-07-30\n\nHistory.\n", + ) + + require_public_preview_docs() + + def test_fenced_release_evidence_is_not_accepted_as_history( releasable_documents: Path, ) -> None: From 9599ba77407d61de499ad767ce9624f4b5ac9e8d Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 18:50:56 +0800 Subject: [PATCH 4/8] fix: bind canonical Markdown workflow targets --- scripts/_checks.py | 9 +++++---- tests/test_release_documents.py | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/_checks.py b/scripts/_checks.py index 31dfd7b..c69a5fe 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -166,6 +166,10 @@ def _actions_path_has_canonical_url( if canonical is None: return False + for link in _MARKDOWN_LINK.finditer(text): + if link.start("target") <= path.start() and path.end() <= link.end("target"): + return _CANONICAL_ACTIONS_URL.fullmatch(link.group("target")) is not None + for attribute in _RAW_HTML_URL_ATTRIBUTE.finditer(text): if attribute.start("url") <= path.start() and path.end() <= attribute.end("url"): return _CANONICAL_ACTIONS_URL.fullmatch(attribute.group("url")) is not None @@ -186,10 +190,7 @@ def _actions_path_has_canonical_url( if boundary == "<": return canonical.start() >= 2 and text[canonical.start() - 2].isspace() if boundary == "(": - before_boundary = canonical.start() - 2 - return ( - before_boundary < 0 or text[before_boundary].isspace() or text[before_boundary] == "]" - ) + return canonical.start() == 1 or text[canonical.start() - 2].isspace() return False diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index b7a7463..c3c8f4b 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -706,6 +706,8 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "actions/runs/30515861246)", "mailto:(https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246)", "prefix", + "[evil](mailto:foo](https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30515861246))", "http://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822", "https://evil.example/?next=https%3A%2F%2Fgithub.com%2Fcometapi-dev%2F" "cometapi-python%2Factions%2Fruns%2F30511373822", @@ -777,6 +779,29 @@ def test_release_evidence_accepts_canonical_raw_html_anchor( require_public_preview_docs() +def test_release_evidence_accepts_canonical_markdown_link( + releasable_documents: Path, +) -> None: + evidence = _release_evidence_block().replace( + "- Release workflow https://github.com/cometapi-dev/cometapi-python/actions/runs/" + "30515861246", + "- Release workflow [canonical publication run](https://github.com/" + "cometapi-dev/cometapi-python/actions/runs/30515861246)", + 1, + ) + for name in ("ROADMAP.md", "RELEASING.md"): + with (releasable_documents / name).open("a", encoding="utf-8") as stream: + stream.write(evidence) + _replace( + releasable_documents, + "CHANGELOG.md", + "# Changelog\n", + "# Changelog\n\n## [0.1.2] - 2026-07-30\n\nHistory.\n", + ) + + require_public_preview_docs() + + def test_fenced_release_evidence_is_not_accepted_as_history( releasable_documents: Path, ) -> None: From 862eca9a5f9e26ade447f24efd223ace99435e65 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 18:55:02 +0800 Subject: [PATCH 5/8] fix: validate rendered Markdown link targets --- scripts/_checks.py | 20 ++++++++++++++++++++ tests/test_release_documents.py | 2 ++ 2 files changed, 22 insertions(+) diff --git a/scripts/_checks.py b/scripts/_checks.py index c69a5fe..5caccbb 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -18,6 +18,8 @@ from typing import cast from urllib.parse import quote, unquote +from markdown_it import MarkdownIt + if sys.version_info >= (3, 11): import tomllib else: @@ -194,6 +196,18 @@ def _actions_path_has_canonical_url( return False +def _rendered_markdown_link_targets(text: str) -> list[str]: + targets: list[str] = [] + for token in MarkdownIt("commonmark", {"html": True}).parse(text): + for child in token.children or []: + if child.type != "link_open": + continue + target = child.attrGet("href") + if isinstance(target, str): + targets.append(target) + return targets + + _WHEEL_DIGEST = re.compile( r"\bwheel\s+sha256\b[^0-9a-f]{0,96}(?P[0-9a-f]{64})(?![0-9a-f])", re.IGNORECASE | re.DOTALL, @@ -762,6 +776,12 @@ def _canonical_actions_run_violations( ) for variant in variants ] + malformed = malformed or any( + _ACTIONS_PATH.search(target) is not None + and _CANONICAL_ACTIONS_URL.fullmatch(target) is None + for variant in variants + for target in _rendered_markdown_link_targets(variant) + ) malformed = malformed or any( count > direct_paths[identity] for paths in variant_paths[1:] diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index c3c8f4b..54f3332 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -708,6 +708,8 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "prefix", "[evil](mailto:foo](https://github.com/cometapi-dev/cometapi-python/" "actions/runs/30515861246))", + "[evil](mailto:foo([x](https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30515861246)))", "http://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822", "https://evil.example/?next=https%3A%2F%2Fgithub.com%2Fcometapi-dev%2F" "cometapi-python%2Factions%2Fruns%2F30511373822", From 58befcf82863651ab09fa3296b071071ac871e41 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 19:28:10 +0800 Subject: [PATCH 6/8] fix: close release evidence parser bypasses --- scripts/_checks.py | 154 +++++++++++++++----------------- tests/test_release_documents.py | 67 +++++++++++++- 2 files changed, 134 insertions(+), 87 deletions(-) diff --git a/scripts/_checks.py b/scripts/_checks.py index 5caccbb..3d08885 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -14,12 +14,11 @@ from datetime import date from email.message import Message from email.parser import Parser +from html.parser import HTMLParser from pathlib import Path from typing import cast from urllib.parse import quote, unquote -from markdown_it import MarkdownIt - if sys.version_info >= (3, 11): import tomllib else: @@ -134,7 +133,6 @@ r")(?=$|[/?#>])" ) _HTTP_URL = re.compile(r"https?://[^\s<>)\]]+", re.IGNORECASE) -_URLISH_TOKEN = re.compile(r"https?:[^\s<>\"']+", re.IGNORECASE) _RECOVERY_TAGS = {"0.1.0a1": "v0.1.0-alpha.1+recovery.1"} _FULL_COMMIT = re.compile(r"(?\d+)(?:/attempts/(?P\d+))?", re.IGNORECASE, ) -_CANONICAL_ACTIONS_URL = re.compile( - rf"(?[1-9]\d*)" +_CANONICAL_ACTIONS_DESTINATION = re.compile( + rf"{re.escape(_ACTIONS_PREFIX)}(?P[1-9]\d*)" r"(?:/attempts/(?P[1-9]\d*))?" - r"(?=$|[\s<>\"')\]}]|[.,;:!?](?=$|\s))" ) -_RAW_HTML_URL_ATTRIBUTE = re.compile( - r"(?is)\b(?:href|src)\s*=\s*(?P['\"])(?P.*?)(?P=quote)" +_RAW_ACTIONS_DESTINATION = re.compile( + rf"(?{re.escape(_ACTIONS_PREFIX)}[1-9]\d*" + r"(?:/attempts/[1-9]\d*)?)" + r"(?=$|[\s.,;:!?])" ) - - -def _actions_path_has_canonical_url( - text: str, - path: re.Match[str], - canonical_matches: list[re.Match[str]], -) -> bool: - canonical = next( - ( - match - for match in canonical_matches - if match.start() <= path.start() and path.end() <= match.end() - ), - None, - ) - if canonical is None: - return False - - for link in _MARKDOWN_LINK.finditer(text): - if link.start("target") <= path.start() and path.end() <= link.end("target"): - return _CANONICAL_ACTIONS_URL.fullmatch(link.group("target")) is not None - - for attribute in _RAW_HTML_URL_ATTRIBUTE.finditer(text): - if attribute.start("url") <= path.start() and path.end() <= attribute.end("url"): - return _CANONICAL_ACTIONS_URL.fullmatch(attribute.group("url")) is not None - - containing_tokens = [ - match - for match in _URLISH_TOKEN.finditer(text) - if match.start() <= path.start() and path.end() <= match.end() - ] - if containing_tokens and min(match.start() for match in containing_tokens) < canonical.start(): - return False - - if canonical.start() == 0: - return True - boundary = text[canonical.start() - 1] - if boundary.isspace(): - return True - if boundary == "<": - return canonical.start() >= 2 and text[canonical.start() - 2].isspace() - if boundary == "(": - return canonical.start() == 1 or text[canonical.start() - 2].isspace() - return False - - -def _rendered_markdown_link_targets(text: str) -> list[str]: - targets: list[str] = [] - for token in MarkdownIt("commonmark", {"html": True}).parse(text): - for child in token.children or []: - if child.type != "link_open": - continue - target = child.attrGet("href") - if isinstance(target, str): - targets.append(target) - return targets - - _WHEEL_DIGEST = re.compile( r"\bwheel\s+sha256\b[^0-9a-f]{0,96}(?P[0-9a-f]{64})(?![0-9a-f])", re.IGNORECASE | re.DOTALL, @@ -250,6 +191,24 @@ class CheckError(RuntimeError): """Raised when release-candidate evidence does not satisfy a local gate.""" +class _ActionsAnchorParser(HTMLParser): + """Collect exact Actions destinations from rendered HTML anchors.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.destinations: Counter[tuple[str, str | None]] = Counter() + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.casefold() != "a": + return + for name, value in attrs: + if name.casefold() != "href" or value is None: + continue + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(value) + if destination is not None: + self.destinations[(destination.group("run"), destination.group("attempt"))] += 1 + + @dataclass(frozen=True) class ReleaseEvidenceIdentity: """Machine-readable identity for one immutable historical release.""" @@ -747,53 +706,80 @@ def _canonical_actions_run_violations( findings: list[tuple[int, str]] = [] direct = unicodedata.normalize("NFKC", body) variants = [direct] - for _ in range(8): + converged = False + for _ in range(len(direct) + 1): previous = variants[-1] decoded = html.unescape(previous) decoded = unquote(decoded) decoded = re.sub(r"\\([/\\.:?&=%#])", r"\1", decoded) - decoded = decoded.replace("\t", "").replace("\n", "").replace("\r", "") + decoded = decoded.replace("\t", "") decoded = re.sub( r"(?i)(https?:[^\s<>)\]]*)\\([^\s<>)\]]*)", lambda match: match.group(0).replace("\\", "/"), decoded, ) if decoded == previous: + converged = True break variants.append(decoded) - path_matches = list(_ACTIONS_PATH.finditer(direct)) - canonical_matches = list(_CANONICAL_ACTIONS_URL.finditer(direct)) - malformed = any( - not _actions_path_has_canonical_url(direct, path, canonical_matches) - for path in path_matches - ) - direct_paths = Counter((match.group("run"), match.group("attempt")) for match in path_matches) variant_paths = [ Counter( - (match.group("run"), match.group("attempt")) + (match.group("run"), match.group("attempt") or None) for match in _ACTIONS_PATH.finditer(variant) ) for variant in variants ] + normalized = variants[-1] + rendered_destinations: Counter[tuple[str, str | None]] = Counter() + # Keep module import dependency-free for the copied-checkout pre-sync bootstrap. + from markdown_it import MarkdownIt + + parser = MarkdownIt("commonmark", {"html": True}) + for token in parser.parse(normalized): + for child in token.children or []: + if child.type == "html_inline": + html_parser = _ActionsAnchorParser() + html_parser.feed(child.content) + rendered_destinations.update(html_parser.destinations) + continue + if child.type != "link_open": + continue + target = child.attrGet("href") + if not isinstance(target, str): + continue + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(target) + if destination is not None: + rendered_destinations[(destination.group("run"), destination.group("attempt"))] += 1 + + if token.type == "html_block": + html_parser = _ActionsAnchorParser() + html_parser.feed(token.content) + rendered_destinations.update(html_parser.destinations) + + for raw in _RAW_ACTIONS_DESTINATION.finditer(normalized): + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(raw.group("url")) + if destination is not None: + rendered_destinations[(destination.group("run"), destination.group("attempt"))] += 1 + + malformed = not converged or variant_paths[-1] != rendered_destinations malformed = malformed or any( - _ACTIONS_PATH.search(target) is not None - and _CANONICAL_ACTIONS_URL.fullmatch(target) is None - for variant in variants - for target in _rendered_markdown_link_targets(variant) - ) - malformed = malformed or any( - count > direct_paths[identity] + count > variant_paths[0][identity] for paths in variant_paths[1:] for identity, count in paths.items() ) + malformed = malformed or any( + match.end() < len(normalized) and normalized[match.end()] not in " \t\r\n.,;:!?)]}>\"'" + for match in _ACTIONS_PATH.finditer(normalized) + ) if malformed: findings.append( ( line, f"release-evidence block for {version} contains a non-canonical Actions URL; " - "use the exact repository /actions/runs/ URL with an optional " + "use the exact repository /actions/runs/ URL as plain Markdown " + "text, an autolink, or a link destination, with an optional " "/attempts/ suffix", ) ) diff --git a/tests/test_release_documents.py b/tests/test_release_documents.py index 54f3332..41d5868 100644 --- a/tests/test_release_documents.py +++ b/tests/test_release_documents.py @@ -7,6 +7,7 @@ import zipfile from collections.abc import Callable from pathlib import Path +from urllib.parse import quote import pytest @@ -705,11 +706,18 @@ def test_release_evidence_rejects_obsolete_workflow_reference_marker( "https://evil.example/#(https://github.com/cometapi-dev/cometapi-python/" "actions/runs/30515861246)", "mailto:(https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246)", - "prefix", "[evil](mailto:foo](https://github.com/cometapi-dev/cometapi-python/" "actions/runs/30515861246))", "[evil](mailto:foo([x](https://github.com/cometapi-dev/cometapi-python/" "actions/runs/30515861246)))", + "mailto:x](https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246)", + "[release](mailto:evil (https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30515861246))", + "![release](https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246)", + 'release run', + '
release run
', "http://github.com/cometapi-dev/cometapi-python/actions/runs/30511373822", "https://evil.example/?next=https%3A%2F%2Fgithub.com%2Fcometapi-dev%2F" "cometapi-python%2Factions%2Fruns%2F30511373822", @@ -754,8 +762,40 @@ def test_release_evidence_rejects_noncanonical_workflow_reference_url( with pytest.raises(CheckError) as caught: require_public_preview_docs() - assert "non-canonical Actions URL" in str(caught.value) - assert "/actions/runs/" in str(caught.value) + message = str(caught.value) + assert ( + "non-canonical Actions URL" in message + or "contradicts its release-identity marker" in message + ) + if "non-canonical Actions URL" in message: + assert "/actions/runs/" in message + + +@pytest.mark.parametrize("depth", [9, 64]) +def test_release_evidence_rejects_deeply_encoded_workflow_reference_url( + releasable_documents: Path, + depth: int, +) -> None: + wrapped = ( + "https://evil.example/?next=https://github.com/cometapi-dev/cometapi-python/" + "actions/runs/30511373822" + ) + for _ in range(depth): + wrapped = quote(wrapped, safe="") + evidence = _release_evidence_block().replace( + "30515861246\n- https://pypi.org", + f"30515861246\n- Required CI {wrapped}\n- https://pypi.org", + 1, + ) + for name in ("ROADMAP.md", "RELEASING.md"): + with (releasable_documents / name).open("a", encoding="utf-8") as stream: + stream.write(evidence) + + with pytest.raises(CheckError) as caught: + require_public_preview_docs() + + message = str(caught.value) + assert "non-canonical Actions URL" in message def test_release_evidence_accepts_canonical_raw_html_anchor( @@ -1512,6 +1552,27 @@ def test_copied_repository_verification_runs_public_document_gate() -> None: ] +def test_copied_repository_checker_imports_before_dependency_sync( + tmp_path: Path, +) -> None: + root = tmp_path / "repository" + scripts = root / "scripts" + scripts.mkdir(parents=True) + for name in ("_checks.py", "check_repository_independence.py"): + shutil.copy2(PROJECT_ROOT / "scripts" / name, scripts / name) + + result = subprocess.run( + [sys.executable, "-S", "scripts/check_repository_independence.py", "--scan-only"], + cwd=root, + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "standalone copied-checkout verification passed" in result.stdout + + def test_artifact_metadata_must_match_source_readme_exactly() -> None: expected = ( "Stable 0.1.x maintenance releases are available from PyPI.\n" From 28fee2b2315d68f11dedd0e4ca6956ffb9057391 Mon Sep 17 00:00:00 2001 From: CometAPI Date: Fri, 31 Jul 2026 21:02:00 +0800 Subject: [PATCH 7/8] fix: harden immutable release verification --- .github/workflows/publish.yml | 2 + AGENTS.md | 16 +- ARCHITECTURE.md | 15 +- RELEASING.md | 16 +- scripts/_checks.py | 251 ++++++++++++++++++++---------- scripts/check_artifacts.py | 26 +++- scripts/check_version.py | 109 ++++--------- scripts/check_workflows.py | 19 ++- tests/test_changelog_gate.py | 21 +++ tests/test_release_documents.py | 266 +++++++++++++++++++++++++++++++- tests/test_release_workflow.py | 50 ++++++ 11 files changed, 601 insertions(+), 190 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 466ae16..2a96424 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -286,6 +286,8 @@ jobs: run: uv run python scripts/check_artifacts.py dist/* - name: Install and smoke-test each exact artifact run: uv run python scripts/check_clean_install.py dist/* + - name: Verify the immutable release as a copied standalone repository + run: python scripts/check_repository_independence.py - name: Record immutable artifact digests run: sha256sum dist/* > artifact-sha256.txt - name: Retain only the verified release bundle diff --git a/AGENTS.md b/AGENTS.md index f2130ab..ffbdb52 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,14 +157,16 @@ Post-stable invariants: 12. Keep `CHANGELOG.md` release-only: do not maintain an unmanaged `Unreleased` section. Record changes in Conventional Commits and let Release Please own the newest canonical dated release section after the changelog preamble. - The version gate must reject any `Unreleased` level-two heading before merge - or release. + Use only canonical Markdown for level-two changelog headings; raw HTML H2 + parsing is renderer-dependent and must fail the version gate before merge or + release. The same gate must reject any `Unreleased` level-two heading. 13. A validated release-evidence block binds one canonical publication workflow run to its machine-readable identity marker and may contain no other Actions run URL. Keep preparatory implementation, CI, Release Please, failed-attempt, and recovery history outside that block. The document gate must reject - non-canonical, wrapped, malformed, or contradictory workflow URLs regardless - of prose or Markdown labeling. + non-canonical, wrapped, encoded, control-obfuscated, malformed, or + contradictory workflow URLs regardless of prose or Markdown labeling, and + bind each source occurrence to exactly one rendered navigation destination. ## Repository independence @@ -289,8 +291,10 @@ workflow after maintainer authorization. Build wheel and source distribution into a clean output directory. Inspect their metadata and file lists, install each exact artifact independently outside the source tree, assert version and public imports, and run mocked-call -smokes. Generated artifacts, local environments, and credentials must never be -committed. +smokes. Require every reviewed sdist member to match the release checkout byte +for byte, and rerun the copied standalone repository verification in the +immutable-tag build before retaining digests. Generated artifacts, local +environments, and credentials must never be committed. ## Release and documentation rules diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3f560a8..eafcf24 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -133,7 +133,9 @@ sdist, and PyPI metadata are immutable, the README uses an unversioned install command and publication-neutral release language that remains accurate before and after a release. Source-document and artifact checks reject approval, unpublished, exact-version installation, and versioned release-link text; each -artifact long description must also exactly match the source README. +artifact long description must also exactly match the source README. Artifact +inspection additionally requires every reviewed source-distribution member to +match the release checkout byte for byte. Release Please v5.0.0 is pinned to the immutable commit whose action metadata uses `node24`. The workflow semantic contract fixes that SHA and runtime @@ -146,7 +148,8 @@ Conventional Commits and never maintain an `Unreleased` placeholder. The version gate rejects that structurally incompatible placeholder, accepts Release Please's native linked form and legacy dated history, and validates repository, previous tag, candidate tag, and calendar date without rewriting generated -history. +history. Raw HTML level-two headings are rejected rather than interpreted with +renderer-specific error recovery. The PyPI publisher remains directly in `publish.yml` and is pinned to its reviewed Node 24 maintenance release. Pinning its exact SHA prevents a syntactic @@ -167,9 +170,11 @@ validated release-evidence blocks in `ROADMAP.md` and `RELEASING.md`. Each block binds only the canonical publication run through its machine-readable identity; preparatory implementation, CI, Release Please, failed-attempt, and recovery history stays outside the block. The checker rejects every other run -identity plus wrapped or malformed Actions URLs, so Markdown wording cannot -disguise a contradictory workflow. Architecture documents mechanisms and -boundaries, not a second historical ledger. +identity and binds every source occurrence to exactly one rendered Markdown or +HTML navigation destination after bounded normalization. Wrapped, encoded, +control-obfuscated, malformed, or contradictory Actions URLs fail closed, so +prose and renderer syntax cannot disguise a workflow identity. Architecture +documents mechanisms and boundaries, not a second historical ledger. The scheduled/manual default-branch smoke is an operational canary only; it does not prove the release commit. `COMETAPI_KEY` is exposed only to the diff --git a/RELEASING.md b/RELEASING.md index 88da093..4c0c857 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -202,13 +202,16 @@ authorization failure. Conventional Commits carry pending changes, and Release Please owns the newest canonical dated section immediately after the preamble. The version gate rejects any unmanaged `Unreleased` level-two heading so the generated layout remains -valid on every patch release. +valid on every patch release. Use only canonical Markdown for level-two +headings; raw HTML H2 semantics vary by renderer and therefore fail closed. Each validated release-evidence block contains the immutable release identity and only its canonical publication workflow. Preparatory implementation CI, Release Please, failed-publication, and recovery history remains outside the -block. The document gate rejects every different run identity and every wrapped -or malformed Actions URL regardless of prose or Markdown labeling. +block. The document gate binds every source occurrence to exactly one rendered +navigation destination and rejects every different run identity plus wrapped, +encoded, control-obfuscated, or malformed Actions URLs regardless of prose or +Markdown labeling. Release mode (`check_version.py --require-releasable-docs`) also fails closed until project authorship, the canonical GitHub repository URL, the copyright @@ -225,7 +228,8 @@ description. The README therefore uses the unpinned PRs and post-release evidence changes must not introduce approval, unpublished, or exact-version availability statements. Artifact inspection applies the same policy to wheel `METADATA` and sdist `PKG-INFO`, so source and registry-facing -descriptions cannot drift. +descriptions cannot drift. Artifact inspection also requires every reviewed +source-distribution member to match the release checkout byte for byte. ## Workflow responsibilities @@ -290,7 +294,9 @@ descriptions cannot drift. against the exact pre-publication digests and Trusted Publisher provenance before a clean install explicitly from `https://pypi.org/simple/`. The exact release live model is the canonical active model enforced by the workflow - checker and cannot be overridden by repository variables. + checker and cannot be overridden by repository variables. Before digest + retention, the immutable-tag build also copies the checkout into an empty + parent and completes the standalone repository verification there. Because the unused Release Please or recovery path is intentionally skipped, every job after the selector must use `always() && !cancelled()`, reject reruns, and require each direct dependency's result to equal `success`. This diff --git a/scripts/_checks.py b/scripts/_checks.py index 3d08885..ddbdcf4 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -19,11 +19,6 @@ from typing import cast from urllib.parse import quote, unquote -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib - PROJECT_ROOT = Path(__file__).resolve().parents[1] DIST_NAME = "cometapi" CANONICAL_ACTIVE_MODEL = "gpt-5.6-sol" @@ -144,10 +139,8 @@ rf"{re.escape(_ACTIONS_PREFIX)}(?P[1-9]\d*)" r"(?:/attempts/(?P[1-9]\d*))?" ) -_RAW_ACTIONS_DESTINATION = re.compile( - rf"(?{re.escape(_ACTIONS_PREFIX)}[1-9]\d*" - r"(?:/attempts/[1-9]\d*)?)" - r"(?=$|[\s.,;:!?])" +_BARE_ACTIONS_DESTINATION = re.compile( + rf"(?P{re.escape(_ACTIONS_PREFIX)}[1-9]\d*(?:/attempts/[1-9]\d*)?)" ) _WHEEL_DIGEST = re.compile( r"\bwheel\s+sha256\b[^0-9a-f]{0,96}(?P[0-9a-f]{64})(?![0-9a-f])", @@ -192,21 +185,67 @@ class CheckError(RuntimeError): class _ActionsAnchorParser(HTMLParser): - """Collect exact Actions destinations from rendered HTML anchors.""" + """Collect canonical destinations from unambiguous raw-HTML anchors.""" def __init__(self) -> None: super().__init__(convert_charrefs=True) self.destinations: Counter[tuple[str, str | None]] = Counter() + self.anchor_depth = 0 + + def _handle_anchor( + self, + attrs: list[tuple[str, str | None]], + *, + self_closing: bool, + ) -> None: + hrefs = [value for name, value in attrs if name.casefold() == "href"] + if len(hrefs) == 1 and hrefs[0] is not None: + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(hrefs[0]) + if destination is not None: + self.destinations[(destination.group("run"), destination.group("attempt"))] += 1 + del self_closing def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: - if tag.casefold() != "a": - return - for name, value in attrs: - if name.casefold() != "href" or value is None: + if tag.casefold() == "a": + self._handle_anchor(attrs, self_closing=False) + self.anchor_depth += 1 + + def handle_endtag(self, tag: str) -> None: + if tag.casefold() == "a": + self.anchor_depth = max(0, self.anchor_depth - 1) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.casefold() == "a": + self._handle_anchor(attrs, self_closing=True) + + +def _bare_actions_destinations(text: str) -> Counter[tuple[str, str | None]]: + """Collect canonical visible URLs with unambiguous prose boundaries.""" + destinations: Counter[tuple[str, str | None]] = Counter() + for match in _BARE_ACTIONS_DESTINATION.finditer(text): + line_prefix = text[text.rfind("\n", 0, match.start()) + 1 : match.start()] + prefix_token = line_prefix.rsplit(maxsplit=1)[-1] if line_prefix.split() else "" + if re.search(r"(?i)(?:[a-z][a-z0-9+.-]*:|[?#=&])", prefix_token): + continue + before = text[match.start() - 1] if match.start() else "" + after = text[match.end() :] + if before and not (before.isspace() or before in "([{<"): + continue + if before in "([{<" and match.start() > 1: + before_opener = text[match.start() - 2] + if not (before_opener.isspace() or before_opener in "([{<"): continue - destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(value) - if destination is not None: - self.destinations[(destination.group("run"), destination.group("attempt"))] += 1 + if after: + if after[0].isspace() or after[0] in ")]}>\"'": + pass + elif after[0] in ".,;:!?" and (len(after) == 1 or after[1].isspace()): + pass + else: + continue + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(match.group("url")) + assert destination is not None + destinations[(destination.group("run"), destination.group("attempt"))] += 1 + return destinations @dataclass(frozen=True) @@ -225,6 +264,13 @@ class ReleaseEvidenceIdentity: def read_project_metadata(root: Path = PROJECT_ROOT) -> dict[str, object]: """Return the parsed PEP 621 project table.""" path = root / "pyproject.toml" + try: + if sys.version_info >= (3, 11): + import tomllib + else: + import tomli as tomllib + except ImportError as exc: + raise CheckError("TOML parsing requires tomli on Python 3.10") from exc try: document = cast(dict[str, object], tomllib.loads(path.read_text(encoding="utf-8"))) except (OSError, UnicodeError, tomllib.TOMLDecodeError) as exc: @@ -573,11 +619,6 @@ def _identity_violations( re.IGNORECASE, ), "exact release commit": re.compile(re.escape(identity.commit), re.IGNORECASE), - "exact release workflow URL": re.compile( - rf"(?\"')\]}]|[.,;:!?](?=$|\s))" - ), "exact wheel SHA256": re.compile( rf"\bwheel\s+sha256\b[^0-9a-f]{{0,96}}{identity.wheel_sha256}(?![0-9a-f])", re.IGNORECASE | re.DOTALL, @@ -595,6 +636,22 @@ def _identity_violations( (line, f"release-evidence block for {version} is missing {label} from its identity") ) + actions_findings, canonical_workflow_count = _canonical_actions_run_violations( + body, + version, + identity.workflow_run, + line, + ) + findings.extend(actions_findings) + if canonical_workflow_count == 0: + findings.append( + ( + line, + f"release-evidence block for {version} is missing exact release workflow URL " + "from its identity", + ) + ) + release_commit_values: set[str] = set() for commit in _FULL_COMMIT.finditer(prose): line_start = prose.rfind("\n", 0, commit.start()) + 1 @@ -701,16 +758,41 @@ def _canonical_actions_run_violations( version: str, expected_run: str, line: int, -) -> list[tuple[int, str]]: +) -> tuple[list[tuple[int, str]], int]: """Require every Actions URL in immutable evidence to name one canonical run.""" findings: list[tuple[int, str]] = [] + structural_control_counts = Counter(character for character in body if character in "\n\r\t") + encoded_control = re.compile(r"(?i)(?:%0[0-9a-f]|&#(?:x0*[0-9a-f]|0*(?:9|10|11|12|13));?)") + + def has_obfuscated_control(value: str) -> bool: + controls = Counter(character for character in value if character in "\n\r\t") + if encoded_control.search(value) is not None or any( + controls[character] > structural_control_counts[character] for character in "\n\r\t" + ): + return True + return any( + character == "\ufffd" + or unicodedata.category(character) in {"Zl", "Zp"} + or (unicodedata.category(character) == "Mn" and unicodedata.combining(character) == 0) + or ( + unicodedata.category(character) in {"Cc", "Cf"} + and character not in {"\n", "\r", "\t"} + ) + or unicodedata.bidirectional(character) + in {"LRE", "RLE", "LRO", "RLO", "PDF", "LRI", "RLI", "FSI", "PDI"} + for character in value + ) + direct = unicodedata.normalize("NFKC", body) variants = [direct] + control_obfuscation = has_obfuscated_control(direct) converged = False for _ in range(len(direct) + 1): previous = variants[-1] - decoded = html.unescape(previous) - decoded = unquote(decoded) + decoded = unicodedata.normalize("NFKC", html.unescape(previous)) + control_obfuscation = control_obfuscation or has_obfuscated_control(decoded) + decoded = unicodedata.normalize("NFKC", unquote(decoded)) + control_obfuscation = control_obfuscation or has_obfuscated_control(decoded) decoded = re.sub(r"\\([/\\.:?&=%#])", r"\1", decoded) decoded = decoded.replace("\t", "") decoded = re.sub( @@ -723,54 +805,67 @@ def _canonical_actions_run_violations( break variants.append(decoded) - variant_paths = [ - Counter( - (match.group("run"), match.group("attempt") or None) - for match in _ACTIONS_PATH.finditer(variant) - ) - for variant in variants - ] normalized = variants[-1] - rendered_destinations: Counter[tuple[str, str | None]] = Counter() - # Keep module import dependency-free for the copied-checkout pre-sync bootstrap. - from markdown_it import MarkdownIt - - parser = MarkdownIt("commonmark", {"html": True}) - for token in parser.parse(normalized): - for child in token.children or []: - if child.type == "html_inline": + normalized_paths = list(_ACTIONS_PATH.finditer(normalized)) + try: + from markdown_it import MarkdownIt + except ImportError as exc: + raise CheckError("release-evidence Markdown validation requires markdown-it-py") from exc + + bindings: list[int] = [] + used_sentinels = {path.group("run") for path in normalized_paths} + for index, path in enumerate(normalized_paths): + run_start, run_end = path.span("run") + width = run_end - run_start + sentinel_value = 10 ** (width - 1) + index + sentinel = str(sentinel_value) + while len(sentinel) == width and sentinel in used_sentinels: + sentinel_value += 1 + sentinel = str(sentinel_value) + if len(sentinel) != width: + raise CheckError("too many Actions URL occurrences to bind unambiguously") + used_sentinels.add(sentinel) + mutated = normalized[:run_start] + sentinel + normalized[run_end:] + identity = (sentinel, path.group("attempt") or None) + destinations: Counter[tuple[str, str | None]] = Counter() + parser = MarkdownIt("commonmark", {"html": True}) + for token in parser.parse(mutated): + if token.type == "inline": + link_depth = 0 html_parser = _ActionsAnchorParser() - html_parser.feed(child.content) - rendered_destinations.update(html_parser.destinations) - continue - if child.type != "link_open": - continue - target = child.attrGet("href") - if not isinstance(target, str): - continue - destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(target) - if destination is not None: - rendered_destinations[(destination.group("run"), destination.group("attempt"))] += 1 - - if token.type == "html_block": - html_parser = _ActionsAnchorParser() - html_parser.feed(token.content) - rendered_destinations.update(html_parser.destinations) - - for raw in _RAW_ACTIONS_DESTINATION.finditer(normalized): - destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(raw.group("url")) - if destination is not None: - rendered_destinations[(destination.group("run"), destination.group("attempt"))] += 1 - - malformed = not converged or variant_paths[-1] != rendered_destinations - malformed = malformed or any( - count > variant_paths[0][identity] - for paths in variant_paths[1:] - for identity, count in paths.items() - ) + for child in token.children or []: + if child.type == "link_open": + link_depth += 1 + target = child.attrGet("href") + if isinstance(target, str): + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(target) + if destination is not None: + destinations[ + (destination.group("run"), destination.group("attempt")) + ] += 1 + elif child.type == "link_close": + link_depth = max(0, link_depth - 1) + elif child.type == "html_inline": + html_parser.feed(child.content) + elif child.type == "text" and link_depth == 0 and html_parser.anchor_depth == 0: + destinations.update(_bare_actions_destinations(child.content)) + html_parser.close() + destinations.update(html_parser.destinations) + elif token.type == "html_block": + html_parser = _ActionsAnchorParser() + html_parser.feed(token.content) + html_parser.close() + destinations.update(html_parser.destinations) + bindings.append(destinations[identity]) + + malformed = control_obfuscation or not converged or any(count != 1 for count in bindings) + normalized_identities = [ + (match.group("run"), match.group("attempt")) for match in normalized_paths + ] malformed = malformed or any( - match.end() < len(normalized) and normalized[match.end()] not in " \t\r\n.,;:!?)]}>\"'" - for match in _ACTIONS_PATH.finditer(normalized) + [(match.group("run"), match.group("attempt")) for match in _ACTIONS_PATH.finditer(variant)] + != normalized_identities + for variant in variants[:-1] ) if malformed: @@ -783,7 +878,7 @@ def _canonical_actions_run_violations( "/attempts/ suffix", ) ) - run_values = {run for paths in variant_paths for run, _attempt in paths} + run_values = {match.group("run") for match in normalized_paths} if run_values - {expected_run}: findings.append( ( @@ -793,7 +888,11 @@ def _canonical_actions_run_violations( "outside the immutable evidence block", ) ) - return findings + canonical_workflow_count = sum( + count == 1 and path.group("run") == expected_run + for path, count in zip(normalized_paths, bindings, strict=True) + ) + return findings, canonical_workflow_count def _evidence_block_violations( @@ -891,14 +990,6 @@ def _evidence_block_violations( ) findings.extend(identity_findings) if identity is not None: - findings.extend( - _canonical_actions_run_violations( - body, - version, - identity.workflow_run, - start_line, - ) - ) if version in identities: findings.append((start_line, f"duplicate release-evidence block for {version}")) else: diff --git a/scripts/check_artifacts.py b/scripts/check_artifacts.py index 37a929a..7eb9a0b 100644 --- a/scripts/check_artifacts.py +++ b/scripts/check_artifacts.py @@ -177,6 +177,8 @@ def check_wheel(path: Path, expected_version: str, expected_description: str) -> with zipfile.ZipFile(path) as archive: paths = [_safe_path(info.filename, path) for info in archive.infolist()] names = {item.as_posix() for item in paths} + if len(names) != len(paths): + raise CheckError(f"{path.name}: duplicate normalized wheel member path") missing = REQUIRED_PACKAGE_FILES - names if missing: raise CheckError(f"{path.name}: missing package files: {sorted(missing)}") @@ -257,6 +259,9 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> with tarfile.open(path, mode="r:gz") as archive: members = archive.getmembers() paths = [_safe_path(member.name, path) for member in members] + normalized_paths = [item.as_posix() for item in paths] + if len(set(normalized_paths)) != len(normalized_paths): + raise CheckError(f"{path.name}: duplicate normalized sdist member path") if any(member.issym() or member.islnk() or member.isdev() for member in members): raise CheckError(f"{path.name}: links and device members are forbidden") if any(item.parts[0] != expected_root for item in paths): @@ -274,6 +279,19 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> unexpected = sorted(relative_files - expected_files) if unexpected: raise CheckError(f"{path.name}: unexpected sdist files: {unexpected}") + parity_violations: list[str] = [] + for name in sorted(REQUIRED_SDIST_FILES): + stream = archive.extractfile(relative_members[name]) + if stream is None: + raise CheckError(f"{path.name}:{name}: cannot read required source member") + try: + source = (PROJECT_ROOT / name).read_bytes() + except OSError as exc: + raise CheckError(f"cannot read source member {name}: {exc}") from exc + if stream.read() != source: + parity_violations.append( + f"{path.name}:{name}: source-distribution member differs from source tree" + ) document_violations: list[str] = [] documents: dict[str, str] = {} for name in SDIST_PUBLIC_DOCUMENTS: @@ -291,11 +309,13 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> for line, label in violations ) evidence_violations = _release_evidence_binding_violations(documents) - if document_violations or evidence_violations: + if parity_violations or document_violations or evidence_violations: raise CheckError( - f"{path.name}: source-distribution document and release-evidence violations:\n" + f"{path.name}: source-distribution source parity, document, and " + "release-evidence violations:\n" + "\n".join( - f"- {violation}" for violation in document_violations + evidence_violations + f"- {violation}" + for violation in parity_violations + document_violations + evidence_violations ) ) metadata_members = [ diff --git a/scripts/check_version.py b/scripts/check_version.py index 02467fc..fb1e8a0 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -9,7 +9,6 @@ import unicodedata from dataclasses import dataclass from datetime import date -from html.parser import HTMLParser from itertools import pairwise from pathlib import Path from typing import cast @@ -199,66 +198,6 @@ def _visible_inline_text(tokens: list[Token]) -> str: return "".join(visible) -class _RawH2Collector(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.headings: list[tuple[int, str]] = [] - self._line: int | None = None - self._parts: list[str] = [] - - def handle_starttag( - self, - tag: str, - attrs: list[tuple[str, str | None]], - ) -> None: - del attrs - if tag.casefold() == "h2" and self._line is not None: - self._close_heading() - if tag.casefold() == "h2": - self._line = self.getpos()[0] - self._parts = [] - elif tag.casefold() == "br" and self._line is not None: - self._parts.append(" ") - - def handle_startendtag( - self, - tag: str, - attrs: list[tuple[str, str | None]], - ) -> None: - self.handle_starttag(tag, attrs) - - def handle_endtag(self, tag: str) -> None: - if tag.casefold() == "h2" and self._line is not None: - self._close_heading() - - def handle_data(self, data: str) -> None: - if self._line is not None: - self._parts.append(data) - - @property - def collecting(self) -> bool: - return self._line is not None - - def handle_entityref(self, name: str) -> None: - if self._line is not None: - self._parts.append(html.unescape(f"&{name};")) - - def handle_charref(self, name: str) -> None: - if self._line is not None: - self._parts.append(html.unescape(f"&#{name};")) - - def close(self) -> None: - super().close() - if self._line is not None: - self._close_heading() - - def _close_heading(self) -> None: - assert self._line is not None - self.headings.append((self._line, "".join(self._parts))) - self._line = None - self._parts = [] - - def _visible_heading_is_unreleased(label: str) -> bool: label = html.unescape(unicodedata.normalize("NFKC", label)) if any( @@ -290,26 +229,26 @@ def _unreleased_heading_lines(text: str) -> list[int]: and _visible_heading_is_unreleased(_visible_inline_text(inline.children or [])) ): lines.add(token.map[0] + 1) - elif token.type == "inline" and token.map is not None: - collector = _RawH2Collector() - for child in token.children or []: - if child.type == "html_inline": - collector.feed(child.content) - elif collector.collecting and child.type in {"text", "code_inline"}: - collector.handle_data(child.content) - elif collector.collecting and child.type in {"softbreak", "hardbreak"}: - collector.handle_data(" ") - collector.close() - for relative_line, label in collector.headings: - if _visible_heading_is_unreleased(label): - lines.add(token.map[0] + relative_line) - elif token.type == "html_block" and token.map is not None: - collector = _RawH2Collector() - collector.feed(token.content) - collector.close() - for relative_line, label in collector.headings: - if _visible_heading_is_unreleased(label): - lines.add(token.map[0] + relative_line) + return sorted(lines) + + +def _raw_h2_heading_lines(text: str) -> list[int]: + """Reject raw HTML H2s instead of emulating browser error recovery.""" + tokens = MarkdownIt("commonmark", {"html": True}).parse(text) + lines: set[int] = set() + for token in tokens: + if token.map is None: + continue + candidates = ( + [token] + if token.type == "html_block" + else [child for child in token.children or [] if child.type == "html_inline"] + ) + for candidate in candidates: + visible_html = re.sub(r"", "", candidate.content, flags=re.DOTALL) + for match in re.finditer(r"(?i)<[ \t\r\n]*h2(?=[ \t\r\n>/])", visible_html): + relative_line = visible_html.count("\n", 0, match.start()) + lines.add(token.map[0] + relative_line + 1) return sorted(lines) @@ -336,6 +275,7 @@ def _changelog_mutable_region_violations( def _parse_changelog_releases(text: str) -> list[_ChangelogRelease]: unreleased_lines = set(_unreleased_heading_lines(text)) + raw_h2_lines = set(_raw_h2_heading_lines(text)) text = _mask_nonprose_changelog_regions(text) releases: list[_ChangelogRelease] = [] violations: list[str] = [] @@ -379,6 +319,13 @@ def _parse_changelog_releases(text: str) -> list[_ChangelogRelease]: "remove it, record changes in Conventional Commits, and let Release Please " "insert the newest canonical dated release section after the changelog preamble" ) + if line_number in raw_h2_lines: + violations.append( + f"CHANGELOG.md:{line_number}: unmanaged Unreleased heading is forbidden; " + "raw HTML level-two headings are also forbidden because their rendered text " + "is parser-dependent; remove it, use canonical Markdown dated release " + "headings, and let Release Please own the newest section" + ) offset += len(raw_line) if not releases: diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index 022711f..e596015 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -1360,6 +1360,7 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: "Check package metadata rendering", "Inspect artifact identity and shape", "Install and smoke-test each exact artifact", + "Verify the immutable release as a copied standalone repository", "Record immutable artifact digests", "Retain only the verified release bundle", ], @@ -1437,10 +1438,24 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: "Install and smoke-test each exact artifact": ( "uv run python scripts/check_clean_install.py dist/*" ), + "Verify the immutable release as a copied standalone repository": ( + "python scripts/check_repository_independence.py" + ), "Record immutable artifact digests": "sha256sum dist/* > artifact-sha256.txt", } - for name, command in build_commands.items(): - _named_run_step(build, name, command, "release build job") + build_command_indices = { + name: _named_run_step(build, name, command, "release build job")[0] + for name, command in build_commands.items() + } + if not ( + build_command_indices["Install and smoke-test each exact artifact"] + < build_command_indices["Verify the immutable release as a copied standalone repository"] + < build_command_indices["Record immutable artifact digests"] + ): + raise CheckError( + "release build must verify the copied standalone repository after exact-artifact " + "clean install and before recording immutable digests" + ) _, version_step = _named_run_step( build, "Verify project, manifest, changelog, release docs, and tag agreement", diff --git a/tests/test_changelog_gate.py b/tests/test_changelog_gate.py index 2abb6e9..37480d0 100644 --- a/tests/test_changelog_gate.py +++ b/tests/test_changelog_gate.py @@ -109,6 +109,12 @@ def test_changelog_parser_rejects_obfuscated_mutable_versions(claim: str) -> Non "

Unreleased

", "

Current Unreleased

", "Intro

Unreleased

", + "

Unreleased

", + "

Unreleased

", + "

Unreleased

", + "

Un