diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 466ae16..2a96424 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -286,6 +286,8 @@ jobs: run: uv run python scripts/check_artifacts.py dist/* - name: Install and smoke-test each exact artifact run: uv run python scripts/check_clean_install.py dist/* + - name: Verify the immutable release as a copied standalone repository + run: python scripts/check_repository_independence.py - name: Record immutable artifact digests run: sha256sum dist/* > artifact-sha256.txt - name: Retain only the verified release bundle diff --git a/AGENTS.md b/AGENTS.md index f2130ab..42ae877 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -157,14 +157,17 @@ Post-stable invariants: 12. Keep `CHANGELOG.md` release-only: do not maintain an unmanaged `Unreleased` section. Record changes in Conventional Commits and let Release Please own the newest canonical dated release section after the changelog preamble. - The version gate must reject any `Unreleased` level-two heading before merge - or release. + Use only canonical Markdown for level-two changelog headings; raw HTML H2 + parsing is renderer-dependent and must fail the version gate before merge or + release. The same gate must reject any `Unreleased` level-two heading. 13. A validated release-evidence block binds one canonical publication workflow - run to its machine-readable identity marker and may contain no other Actions - run URL. Keep preparatory implementation, CI, Release Please, failed-attempt, + run URL without an attempt suffix to its machine-readable identity marker and + may contain no other Actions run URL. Record attempt provenance as plain + text. Keep preparatory implementation, CI, Release Please, failed-attempt, and recovery history outside that block. The document gate must reject - non-canonical, wrapped, malformed, or contradictory workflow URLs regardless - of prose or Markdown labeling. + non-canonical, wrapped, encoded, control-obfuscated, malformed, or + contradictory workflow URLs regardless of prose or Markdown labeling, and + bind each source occurrence to exactly one rendered navigation destination. ## Repository independence @@ -289,8 +292,10 @@ workflow after maintainer authorization. Build wheel and source distribution into a clean output directory. Inspect their metadata and file lists, install each exact artifact independently outside the source tree, assert version and public imports, and run mocked-call -smokes. Generated artifacts, local environments, and credentials must never be -committed. +smokes. Require every reviewed sdist member to match the release checkout byte +for byte, and rerun the copied standalone repository verification in the +immutable-tag build before retaining digests. Generated artifacts, local +environments, and credentials must never be committed. ## Release and documentation rules diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3f560a8..b76c81e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -133,7 +133,9 @@ sdist, and PyPI metadata are immutable, the README uses an unversioned install command and publication-neutral release language that remains accurate before and after a release. Source-document and artifact checks reject approval, unpublished, exact-version installation, and versioned release-link text; each -artifact long description must also exactly match the source README. +artifact long description must also exactly match the source README. Artifact +inspection additionally requires every reviewed source-distribution member to +match the release checkout byte for byte. Release Please v5.0.0 is pinned to the immutable commit whose action metadata uses `node24`. The workflow semantic contract fixes that SHA and runtime @@ -146,7 +148,8 @@ Conventional Commits and never maintain an `Unreleased` placeholder. The version gate rejects that structurally incompatible placeholder, accepts Release Please's native linked form and legacy dated history, and validates repository, previous tag, candidate tag, and calendar date without rewriting generated -history. +history. Raw HTML level-two headings are rejected rather than interpreted with +renderer-specific error recovery. The PyPI publisher remains directly in `publish.yml` and is pinned to its reviewed Node 24 maintenance release. Pinning its exact SHA prevents a syntactic @@ -165,11 +168,14 @@ ambiguous external state and is forbidden. Immutable run, tag, commit, registry, and digest records live only in the validated release-evidence blocks in `ROADMAP.md` and `RELEASING.md`. Each block binds only the canonical publication run through its machine-readable -identity; preparatory implementation, CI, Release Please, failed-attempt, and -recovery history stays outside the block. The checker rejects every other run -identity plus wrapped or malformed Actions URLs, so Markdown wording cannot -disguise a contradictory workflow. Architecture documents mechanisms and -boundaries, not a second historical ledger. +identity, without an attempt suffix; attempt provenance remains plain text. +Preparatory implementation, CI, Release Please, failed-attempt, and recovery +history stays outside the block. The checker rejects every other run +identity and binds every source occurrence to exactly one rendered Markdown or +HTML navigation destination after bounded normalization. Wrapped, encoded, +control-obfuscated, malformed, or contradictory Actions URLs fail closed, so +prose and renderer syntax cannot disguise a workflow identity. Architecture +documents mechanisms and boundaries, not a second historical ledger. The scheduled/manual default-branch smoke is an operational canary only; it does not prove the release commit. `COMETAPI_KEY` is exposed only to the diff --git a/RELEASING.md b/RELEASING.md index 88da093..4f58b8a 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -202,13 +202,17 @@ authorization failure. Conventional Commits carry pending changes, and Release Please owns the newest canonical dated section immediately after the preamble. The version gate rejects any unmanaged `Unreleased` level-two heading so the generated layout remains -valid on every patch release. +valid on every patch release. Use only canonical Markdown for level-two +headings; raw HTML H2 semantics vary by renderer and therefore fail closed. Each validated release-evidence block contains the immutable release identity -and only its canonical publication workflow. Preparatory implementation CI, -Release Please, failed-publication, and recovery history remains outside the -block. The document gate rejects every different run identity and every wrapped -or malformed Actions URL regardless of prose or Markdown labeling. +and only its canonical publication workflow URL without an attempt suffix. +Record attempt provenance as plain text. Preparatory implementation CI, Release +Please, failed-publication, and recovery history remains outside the block. The +document gate binds every source occurrence to exactly one rendered navigation +destination and rejects every different run identity plus wrapped, encoded, +control-obfuscated, or malformed Actions URLs regardless of prose or Markdown +labeling. Release mode (`check_version.py --require-releasable-docs`) also fails closed until project authorship, the canonical GitHub repository URL, the copyright @@ -225,7 +229,8 @@ description. The README therefore uses the unpinned PRs and post-release evidence changes must not introduce approval, unpublished, or exact-version availability statements. Artifact inspection applies the same policy to wheel `METADATA` and sdist `PKG-INFO`, so source and registry-facing -descriptions cannot drift. +descriptions cannot drift. Artifact inspection also requires every reviewed +source-distribution member to match the release checkout byte for byte. ## Workflow responsibilities @@ -290,7 +295,9 @@ descriptions cannot drift. against the exact pre-publication digests and Trusted Publisher provenance before a clean install explicitly from `https://pypi.org/simple/`. The exact release live model is the canonical active model enforced by the workflow - checker and cannot be overridden by repository variables. + checker and cannot be overridden by repository variables. Before digest + retention, the immutable-tag build also copies the checkout into an empty + parent and completes the standalone repository verification there. Because the unused Release Please or recovery path is intentionally skipped, every job after the selector must use `always() && !cancelled()`, reject reruns, and require each direct dependency's result to equal `success`. This @@ -556,8 +563,8 @@ authorized. `64c7cb87745032703b3374cc562ea00b979416c54908862dbcebd116b2dc44c8`. Both files match the retained pre-publication digest record. PyPI Integrity API provenance identifies repository `cometapi-dev/cometapi-python`, workflow - `publish.yml`, environment `pypi`, release commit `576e7503`, and release - [run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30429821548/attempts/1). + `publish.yml`, environment `pypi`, release commit `576e7503`, and release run + attempt 1. - An independent post-workflow verification downloaded both public files, verified both provenance records with `pypi-attestations==0.0.29`, installed `cometapi==0.1.1` from `https://pypi.org/simple/`, verified the public version @@ -612,8 +619,8 @@ authorized. `21c8edc0586610de1a9a8cd39b54ed23d2b1e20552100f69f53938cb7678da3d`. Both files match the retained pre-publication digest record. PyPI Integrity API provenance identifies repository `cometapi-dev/cometapi-python`, workflow - `publish.yml`, environment `pypi`, release commit `710c5649`, and release - [run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246/attempts/1). + `publish.yml`, environment `pypi`, release commit `710c5649`, and release run + attempt 1. - An independent post-workflow verification downloaded both public files, verified both provenance records with `pypi-attestations==0.0.29`, installed `cometapi==0.1.2` from `https://pypi.org/simple/`, verified the public version @@ -670,8 +677,8 @@ authorized. `07ded54606d50f44b689dad38cf93a74e1175370efaa33be84a3c01240d48e66`. Both files match the retained pre-publication digest record. PyPI Integrity API provenance identifies repository `cometapi-dev/cometapi-python`, workflow - `publish.yml`, environment `pypi`, release commit `45429f37`, and release - [run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30550536000/attempts/1). + `publish.yml`, environment `pypi`, release commit `45429f37`, and release run + attempt 1. - Independent post-workflow verification downloaded both public files, verified both provenance records with `pypi-attestations==0.0.29`, installed `cometapi==0.1.3` from `https://pypi.org/simple/`, verified the public version diff --git a/ROADMAP.md b/ROADMAP.md index 6c681cc..28e0717 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -579,8 +579,8 @@ and source-distribution SHA256 `64c7cb87745032703b3374cc562ea00b979416c54908862dbcebd116b2dc44c8`. Both files matched the retained pre-publication digest record. Their PyPI Integrity API provenance names repository `cometapi-dev/cometapi-python`, -workflow `publish.yml`, environment `pypi`, release commit `576e7503`, and -[run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30429821548/attempts/1). +workflow `publish.yml`, environment `pypi`, release commit `576e7503`, and run +attempt 1. An independent post-workflow verification downloaded both public files, verified their provenance with `pypi-attestations==0.0.29`, installed `cometapi==0.1.1` from the public simple index, checked version and public @@ -651,8 +651,8 @@ and source-distribution SHA256 `21c8edc0586610de1a9a8cd39b54ed23d2b1e20552100f69f53938cb7678da3d`. Both files match the retained pre-publication digest record. Their PyPI Integrity API provenance names repository `cometapi-dev/cometapi-python`, -workflow `publish.yml`, environment `pypi`, release commit `710c5649`, and -[run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30515861246/attempts/1). +workflow `publish.yml`, environment `pypi`, release commit `710c5649`, and run +attempt 1. An independent post-workflow verification downloaded both public files, verified both provenance records with `pypi-attestations==0.0.29`, installed `cometapi==0.1.2` from the public simple index, verified the public version and @@ -716,8 +716,7 @@ and source-distribution SHA256 Both public files match the retained pre-publication digest record. PyPI Integrity API provenance identifies repository `cometapi-dev/cometapi-python`, workflow `publish.yml`, environment `pypi`, -release commit `45429f37`, and release -[run attempt 1](https://github.com/cometapi-dev/cometapi-python/actions/runs/30550536000/attempts/1). +release commit `45429f37`, and release run attempt 1. An independent post-workflow verification downloaded both public files, verified both provenance records with `pypi-attestations==0.0.29`, installed `cometapi==0.1.3` from `https://pypi.org/simple/`, verified the public version diff --git a/scripts/_checks.py b/scripts/_checks.py index e40f520..1bf98f1 100644 --- a/scripts/_checks.py +++ b/scripts/_checks.py @@ -14,15 +14,11 @@ from datetime import date from email.message import Message from email.parser import Parser +from html.parser import HTMLParser from pathlib import Path from typing import cast from urllib.parse import quote, unquote -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib - PROJECT_ROOT = Path(__file__).resolve().parents[1] DIST_NAME = "cometapi" CANONICAL_ACTIVE_MODEL = "gpt-5.6-sol" @@ -139,10 +135,12 @@ r"actions/runs/(?P\d+)(?:/attempts/(?P\d+))?", re.IGNORECASE, ) -_CANONICAL_ACTIONS_URL = re.compile( - rf"(?[1-9]\d*)" +_CANONICAL_ACTIONS_DESTINATION = re.compile( + rf"{re.escape(_ACTIONS_PREFIX)}(?P[1-9]\d*)" r"(?:/attempts/(?P[1-9]\d*))?" - r"(?=$|[\s<>\"')\]}]|[.,;:!?](?=$|\s))" +) +_BARE_ACTIONS_DESTINATION = re.compile( + rf"(?P{re.escape(_ACTIONS_PREFIX)}[1-9]\d*(?:/attempts/[1-9]\d*)?)" ) _WHEEL_DIGEST = re.compile( r"\bwheel\s+sha256\b[^0-9a-f]{0,96}(?P[0-9a-f]{64})(?![0-9a-f])", @@ -186,6 +184,70 @@ class CheckError(RuntimeError): """Raised when release-candidate evidence does not satisfy a local gate.""" +class _ActionsAnchorParser(HTMLParser): + """Collect canonical destinations from unambiguous raw-HTML anchors.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.destinations: Counter[tuple[str, str | None]] = Counter() + self.anchor_depth = 0 + + def _handle_anchor( + self, + attrs: list[tuple[str, str | None]], + *, + self_closing: bool, + ) -> None: + hrefs = [value for name, value in attrs if name.casefold() == "href"] + if len(hrefs) == 1 and hrefs[0] is not None: + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(hrefs[0]) + if destination is not None: + self.destinations[(destination.group("run"), destination.group("attempt"))] += 1 + del self_closing + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.casefold() == "a": + self._handle_anchor(attrs, self_closing=False) + self.anchor_depth += 1 + + def handle_endtag(self, tag: str) -> None: + if tag.casefold() == "a": + self.anchor_depth = max(0, self.anchor_depth - 1) + + def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag.casefold() == "a": + self._handle_anchor(attrs, self_closing=True) + + +def _bare_actions_destinations(text: str) -> Counter[tuple[str, str | None]]: + """Collect canonical visible URLs with unambiguous prose boundaries.""" + destinations: Counter[tuple[str, str | None]] = Counter() + for match in _BARE_ACTIONS_DESTINATION.finditer(text): + line_prefix = text[text.rfind("\n", 0, match.start()) + 1 : match.start()] + prefix_token = line_prefix.rsplit(maxsplit=1)[-1] if line_prefix.split() else "" + if re.search(r"(?i)(?:[a-z][a-z0-9+.-]*:|[?#=&])", prefix_token): + continue + before = text[match.start() - 1] if match.start() else "" + after = text[match.end() :] + if before and not (before.isspace() or before in "([{<"): + continue + if before in "([{<" and match.start() > 1: + before_opener = text[match.start() - 2] + if not (before_opener.isspace() or before_opener in "([{<"): + continue + if after: + if after[0].isspace() or after[0] in ")]}>\"'": + pass + elif after[0] in ".,;:!?" and (len(after) == 1 or after[1].isspace()): + pass + else: + continue + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(match.group("url")) + assert destination is not None + destinations[(destination.group("run"), destination.group("attempt"))] += 1 + return destinations + + @dataclass(frozen=True) class ReleaseEvidenceIdentity: """Machine-readable identity for one immutable historical release.""" @@ -202,6 +264,13 @@ class ReleaseEvidenceIdentity: def read_project_metadata(root: Path = PROJECT_ROOT) -> dict[str, object]: """Return the parsed PEP 621 project table.""" path = root / "pyproject.toml" + try: + if sys.version_info >= (3, 11): + import tomllib + else: + import tomli as tomllib + except ImportError as exc: + raise CheckError("TOML parsing requires tomli on Python 3.10") from exc try: document = cast(dict[str, object], tomllib.loads(path.read_text(encoding="utf-8"))) except (OSError, UnicodeError, tomllib.TOMLDecodeError) as exc: @@ -550,11 +619,6 @@ def _identity_violations( re.IGNORECASE, ), "exact release commit": re.compile(re.escape(identity.commit), re.IGNORECASE), - "exact release workflow URL": re.compile( - rf"(?\"')\]}]|[.,;:!?](?=$|\s))" - ), "exact wheel SHA256": re.compile( rf"\bwheel\s+sha256\b[^0-9a-f]{{0,96}}{identity.wheel_sha256}(?![0-9a-f])", re.IGNORECASE | re.DOTALL, @@ -572,6 +636,22 @@ def _identity_violations( (line, f"release-evidence block for {version} is missing {label} from its identity") ) + actions_findings, canonical_workflow_count = _canonical_actions_run_violations( + body, + version, + identity.workflow_run, + line, + ) + findings.extend(actions_findings) + if canonical_workflow_count == 0: + findings.append( + ( + line, + f"release-evidence block for {version} is missing exact release workflow URL " + "from its identity", + ) + ) + release_commit_values: set[str] = set() for commit in _FULL_COMMIT.finditer(prose): line_start = prose.rfind("\n", 0, commit.start()) + 1 @@ -678,65 +758,123 @@ def _canonical_actions_run_violations( version: str, expected_run: str, line: int, -) -> list[tuple[int, str]]: +) -> tuple[list[tuple[int, str]], int]: """Require every Actions URL in immutable evidence to name one canonical run.""" findings: list[tuple[int, str]] = [] - normalized = unicodedata.normalize("NFKC", body) - direct = normalized - for _ in range(3): - decoded = html.unescape(direct) - if decoded == direct: - break - direct = decoded - markdown_unescaped = re.sub(r"\\([/\\.:?&=%#])", r"\1", direct) - browser_normalized = re.sub( - r"(?i)(https?://[^\s<>)\]]*)\\([^\s<>)\]]*)", - lambda match: match.group(0).replace("\\", "/"), - markdown_unescaped, - ) + structural_control_counts = Counter(character for character in body if character in "\n\r\t") + encoded_control = re.compile(r"(?i)(?:%0[0-9a-f]|&#(?:x0*[0-9a-f]|0*(?:9|10|11|12|13));?)") - path_matches = list(_ACTIONS_PATH.finditer(direct)) - canonical_matches = list(_CANONICAL_ACTIONS_URL.finditer(direct)) - malformed = any( - not any( - url.start() <= path.start() and path.end() <= url.end() for url in canonical_matches + def has_obfuscated_control(value: str) -> bool: + controls = Counter(character for character in value if character in "\n\r\t") + if encoded_control.search(value) is not None or any( + controls[character] > structural_control_counts[character] for character in "\n\r\t" + ): + return True + return any( + character == "\ufffd" + or unicodedata.category(character) in {"Zl", "Zp"} + or (unicodedata.category(character) == "Mn" and unicodedata.combining(character) == 0) + or ( + unicodedata.category(character) in {"Cc", "Cf"} + and character not in {"\n", "\r", "\t"} + ) + or unicodedata.bidirectional(character) + in {"LRE", "RLE", "LRO", "RLO", "PDF", "LRI", "RLI", "FSI", "PDI"} + for character in value ) - for path in path_matches - ) - percent_decoded = direct - for _ in range(3): - decoded = unquote(percent_decoded) - if decoded == percent_decoded: + direct = unicodedata.normalize("NFKC", body) + variants = [body] + if direct != body: + variants.append(direct) + control_obfuscation = has_obfuscated_control(direct) + converged = False + for _ in range(len(direct) + 1): + previous = variants[-1] + decoded = unicodedata.normalize("NFKC", html.unescape(previous)) + control_obfuscation = control_obfuscation or has_obfuscated_control(decoded) + decoded = unicodedata.normalize("NFKC", unquote(decoded)) + control_obfuscation = control_obfuscation or has_obfuscated_control(decoded) + decoded = re.sub(r"\\([/\\.:?&=%#])", r"\1", decoded) + decoded = decoded.replace("\t", "") + decoded = re.sub( + r"(?i)(https?:[^\s<>)\]]*)\\([^\s<>)\]]*)", + lambda match: match.group(0).replace("\\", "/"), + decoded, + ) + if decoded == previous: + converged = True break - percent_decoded = decoded - direct_paths = Counter((match.group("run"), match.group("attempt")) for match in path_matches) - decoded_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(percent_decoded) - ) - malformed = malformed or any( - count > direct_paths[identity] for identity, count in decoded_paths.items() - ) - normalized_paths = Counter( - (match.group("run"), match.group("attempt")) for match in _ACTIONS_PATH.finditer(normalized) - ) - malformed = malformed or any( - count > normalized_paths[identity] for identity, count in direct_paths.items() - ) - unescaped_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(markdown_unescaped) - ) - malformed = malformed or any( - count > normalized_paths[identity] for identity, count in unescaped_paths.items() - ) - browser_paths = Counter( - (match.group("run"), match.group("attempt")) - for match in _ACTIONS_PATH.finditer(browser_normalized) - ) + variants.append(decoded) + + try: + from markdown_it import MarkdownIt + except ImportError as exc: + raise CheckError("release-evidence Markdown validation requires markdown-it-py") from exc + + def rendered_bindings( + source: str, + ) -> tuple[list[re.Match[str]], list[tuple[str, str | None]], list[int]]: + paths = list(_ACTIONS_PATH.finditer(source)) + identities = [(path.group("run"), path.group("attempt")) for path in paths] + bindings: list[int] = [] + used_sentinels = {path.group("run") for path in paths} + for index, path in enumerate(paths): + run_start, run_end = path.span("run") + width = run_end - run_start + sentinel_value = 10 ** (width - 1) + index + sentinel = str(sentinel_value) + while len(sentinel) == width and sentinel in used_sentinels: + sentinel_value += 1 + sentinel = str(sentinel_value) + if len(sentinel) != width: + raise CheckError("too many Actions URL occurrences to bind unambiguously") + used_sentinels.add(sentinel) + mutated = source[:run_start] + sentinel + source[run_end:] + identity = (sentinel, path.group("attempt") or None) + destinations: Counter[tuple[str, str | None]] = Counter() + parser = MarkdownIt("commonmark", {"html": True}) + for token in parser.parse(mutated): + if token.type == "inline": + link_depth = 0 + html_parser = _ActionsAnchorParser() + for child in token.children or []: + if child.type == "link_open": + link_depth += 1 + target = child.attrGet("href") + if isinstance(target, str): + destination = _CANONICAL_ACTIONS_DESTINATION.fullmatch(target) + if destination is not None: + destinations[ + (destination.group("run"), destination.group("attempt")) + ] += 1 + elif child.type == "link_close": + link_depth = max(0, link_depth - 1) + elif child.type == "html_inline": + html_parser.feed(child.content) + elif ( + child.type == "text" + and link_depth == 0 + and html_parser.anchor_depth == 0 + ): + destinations.update(_bare_actions_destinations(child.content)) + html_parser.close() + destinations.update(html_parser.destinations) + elif token.type == "html_block": + html_parser = _ActionsAnchorParser() + html_parser.feed(token.content) + html_parser.close() + destinations.update(html_parser.destinations) + bindings.append(destinations[identity]) + return paths, identities, bindings + + rendered_variants = [rendered_bindings(variant) for variant in variants] + normalized_paths, normalized_identities, bindings = rendered_variants[-1] + + malformed = control_obfuscation or not converged or any(count != 1 for count in bindings) malformed = malformed or any( - count > normalized_paths[identity] for identity, count in browser_paths.items() + identities != normalized_identities or variant_bindings != bindings + for _, identities, variant_bindings in rendered_variants[:-1] ) if malformed: @@ -744,16 +882,11 @@ def _canonical_actions_run_violations( ( line, f"release-evidence block for {version} contains a non-canonical Actions URL; " - "use the exact repository /actions/runs/ URL with an optional " - "/attempts/ suffix", + "use the exact repository /actions/runs/ URL as plain Markdown " + "text, an autolink, or a link destination without an attempt suffix", ) ) - run_values = ( - {run for run, _attempt in direct_paths} - | {run for run, _attempt in decoded_paths} - | {run for run, _attempt in unescaped_paths} - | {run for run, _attempt in browser_paths} - ) + run_values = {match.group("run") for match in normalized_paths} if run_values - {expected_run}: findings.append( ( @@ -763,7 +896,25 @@ def _canonical_actions_run_violations( "outside the immutable evidence block", ) ) - return findings + matching_base_count = sum( + count == 1 and path.group("run") == expected_run and path.group("attempt") is None + for path, count in zip(normalized_paths, bindings, strict=True) + ) + canonical_workflow_count = ( + 1 + if normalized_identities == [(expected_run, None)] and bindings == [1] and not malformed + else 0 + ) + if len(normalized_paths) != 1 or matching_base_count != 1: + findings.append( + ( + line, + f"release-evidence block for {version} must contain exactly one canonical " + "release workflow URL without an attempt suffix and no other Actions URL; " + "record the attempt number as plain provenance text", + ) + ) + return findings, canonical_workflow_count def _evidence_block_violations( @@ -861,14 +1012,6 @@ def _evidence_block_violations( ) findings.extend(identity_findings) if identity is not None: - findings.extend( - _canonical_actions_run_violations( - body, - version, - identity.workflow_run, - start_line, - ) - ) if version in identities: findings.append((start_line, f"duplicate release-evidence block for {version}")) else: @@ -928,10 +1071,9 @@ def exact_release_version_violations( if document == "CHANGELOG.md": return [] if document in RELEASE_EVIDENCE_DOCUMENTS: - structural = unicodedata.normalize("NFKC", text) searchable, marker_findings, _ = _evidence_block_violations( document, - structural, + text, ) normalized_searchable = _normalized_document_text(searchable) searchable = normalized_searchable @@ -953,8 +1095,7 @@ def release_evidence_identities( """Return internally validated immutable identities from an evidence document.""" if document not in RELEASE_EVIDENCE_DOCUMENTS: return {} - structural = unicodedata.normalize("NFKC", text) - _, findings, identities = _evidence_block_violations(document, structural) + _, findings, identities = _evidence_block_violations(document, text) if findings: rendered = "; ".join(f"{document}:{line}: {label}" for line, label in findings) raise CheckError(rendered) diff --git a/scripts/check_artifacts.py b/scripts/check_artifacts.py index 37a929a..7eb9a0b 100644 --- a/scripts/check_artifacts.py +++ b/scripts/check_artifacts.py @@ -177,6 +177,8 @@ def check_wheel(path: Path, expected_version: str, expected_description: str) -> with zipfile.ZipFile(path) as archive: paths = [_safe_path(info.filename, path) for info in archive.infolist()] names = {item.as_posix() for item in paths} + if len(names) != len(paths): + raise CheckError(f"{path.name}: duplicate normalized wheel member path") missing = REQUIRED_PACKAGE_FILES - names if missing: raise CheckError(f"{path.name}: missing package files: {sorted(missing)}") @@ -257,6 +259,9 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> with tarfile.open(path, mode="r:gz") as archive: members = archive.getmembers() paths = [_safe_path(member.name, path) for member in members] + normalized_paths = [item.as_posix() for item in paths] + if len(set(normalized_paths)) != len(normalized_paths): + raise CheckError(f"{path.name}: duplicate normalized sdist member path") if any(member.issym() or member.islnk() or member.isdev() for member in members): raise CheckError(f"{path.name}: links and device members are forbidden") if any(item.parts[0] != expected_root for item in paths): @@ -274,6 +279,19 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> unexpected = sorted(relative_files - expected_files) if unexpected: raise CheckError(f"{path.name}: unexpected sdist files: {unexpected}") + parity_violations: list[str] = [] + for name in sorted(REQUIRED_SDIST_FILES): + stream = archive.extractfile(relative_members[name]) + if stream is None: + raise CheckError(f"{path.name}:{name}: cannot read required source member") + try: + source = (PROJECT_ROOT / name).read_bytes() + except OSError as exc: + raise CheckError(f"cannot read source member {name}: {exc}") from exc + if stream.read() != source: + parity_violations.append( + f"{path.name}:{name}: source-distribution member differs from source tree" + ) document_violations: list[str] = [] documents: dict[str, str] = {} for name in SDIST_PUBLIC_DOCUMENTS: @@ -291,11 +309,13 @@ def check_sdist(path: Path, expected_version: str, expected_description: str) -> for line, label in violations ) evidence_violations = _release_evidence_binding_violations(documents) - if document_violations or evidence_violations: + if parity_violations or document_violations or evidence_violations: raise CheckError( - f"{path.name}: source-distribution document and release-evidence violations:\n" + f"{path.name}: source-distribution source parity, document, and " + "release-evidence violations:\n" + "\n".join( - f"- {violation}" for violation in document_violations + evidence_violations + f"- {violation}" + for violation in parity_violations + document_violations + evidence_violations ) ) metadata_members = [ diff --git a/scripts/check_version.py b/scripts/check_version.py index 7814c28..679baa8 100644 --- a/scripts/check_version.py +++ b/scripts/check_version.py @@ -9,7 +9,6 @@ import unicodedata from dataclasses import dataclass from datetime import date -from html.parser import HTMLParser from itertools import pairwise from pathlib import Path from typing import cast @@ -199,68 +198,14 @@ def _visible_inline_text(tokens: list[Token]) -> str: return "".join(visible) -class _RawH2Collector(HTMLParser): - def __init__(self) -> None: - super().__init__(convert_charrefs=True) - self.headings: list[tuple[int, str]] = [] - self._line: int | None = None - self._parts: list[str] = [] - - def handle_starttag( - self, - tag: str, - attrs: list[tuple[str, str | None]], - ) -> None: - del attrs - if tag.casefold() == "h2" and self._line is not None: - self._close_heading() - if tag.casefold() == "h2": - self._line = self.getpos()[0] - self._parts = [] - elif tag.casefold() == "br" and self._line is not None: - self._parts.append(" ") - - def handle_startendtag( - self, - tag: str, - attrs: list[tuple[str, str | None]], - ) -> None: - self.handle_starttag(tag, attrs) - - def handle_endtag(self, tag: str) -> None: - if tag.casefold() == "h2" and self._line is not None: - self._close_heading() - - def handle_data(self, data: str) -> None: - if self._line is not None: - self._parts.append(data) - - @property - def collecting(self) -> bool: - return self._line is not None - - def handle_entityref(self, name: str) -> None: - if self._line is not None: - self._parts.append(html.unescape(f"&{name};")) - - def handle_charref(self, name: str) -> None: - if self._line is not None: - self._parts.append(html.unescape(f"&#{name};")) - - def close(self) -> None: - super().close() - if self._line is not None: - self._close_heading() - - def _close_heading(self) -> None: - assert self._line is not None - self.headings.append((self._line, "".join(self._parts))) - self._line = None - self._parts = [] - - def _visible_heading_is_unreleased(label: str) -> bool: label = html.unescape(unicodedata.normalize("NFKC", label)) + if any( + unicodedata.bidirectional(value) + in {"LRE", "RLE", "LRO", "RLO", "PDF", "LRI", "RLI", "FSI", "PDI"} + for value in label + ): + return True label = "".join( "" if unicodedata.category(value) == "Cf" @@ -269,7 +214,7 @@ def _visible_heading_is_unreleased(label: str) -> bool: for value in label ) words = re.findall(r"[A-Za-z]+", label) - return bool(words and words[0].casefold() == "unreleased") + return any(word.casefold() == "unreleased" for word in words) def _unreleased_heading_lines(text: str) -> list[int]: @@ -284,26 +229,26 @@ def _unreleased_heading_lines(text: str) -> list[int]: and _visible_heading_is_unreleased(_visible_inline_text(inline.children or [])) ): lines.add(token.map[0] + 1) - elif token.type == "inline" and token.map is not None: - collector = _RawH2Collector() - for child in token.children or []: - if child.type == "html_inline": - collector.feed(child.content) - elif collector.collecting and child.type in {"text", "code_inline"}: - collector.handle_data(child.content) - elif collector.collecting and child.type in {"softbreak", "hardbreak"}: - collector.handle_data(" ") - collector.close() - for relative_line, label in collector.headings: - if _visible_heading_is_unreleased(label): - lines.add(token.map[0] + relative_line) - elif token.type == "html_block" and token.map is not None: - collector = _RawH2Collector() - collector.feed(token.content) - collector.close() - for relative_line, label in collector.headings: - if _visible_heading_is_unreleased(label): - lines.add(token.map[0] + relative_line) + return sorted(lines) + + +def _raw_h2_heading_lines(text: str) -> list[int]: + """Reject raw HTML H2s instead of emulating browser error recovery.""" + tokens = MarkdownIt("commonmark", {"html": True}).parse(text) + lines: set[int] = set() + for token in tokens: + if token.map is None: + continue + candidates = ( + [token] + if token.type == "html_block" + else [child for child in token.children or [] if child.type == "html_inline"] + ) + for candidate in candidates: + visible_html = re.sub(r"", "", candidate.content, flags=re.DOTALL) + for match in re.finditer(r"(?i)<[ \t\r\n\f]*h2(?=[ \t\r\n\f>/])", visible_html): + relative_line = visible_html.count("\n", 0, match.start()) + lines.add(token.map[0] + relative_line + 1) return sorted(lines) @@ -330,6 +275,7 @@ def _changelog_mutable_region_violations( def _parse_changelog_releases(text: str) -> list[_ChangelogRelease]: unreleased_lines = set(_unreleased_heading_lines(text)) + raw_h2_lines = set(_raw_h2_heading_lines(text)) text = _mask_nonprose_changelog_regions(text) releases: list[_ChangelogRelease] = [] violations: list[str] = [] @@ -373,6 +319,13 @@ def _parse_changelog_releases(text: str) -> list[_ChangelogRelease]: "remove it, record changes in Conventional Commits, and let Release Please " "insert the newest canonical dated release section after the changelog preamble" ) + if line_number in raw_h2_lines: + violations.append( + f"CHANGELOG.md:{line_number}: unmanaged Unreleased heading is forbidden; " + "raw HTML level-two headings are also forbidden because their rendered text " + "is parser-dependent; remove it, use canonical Markdown dated release " + "headings, and let Release Please own the newest section" + ) offset += len(raw_line) if not releases: diff --git a/scripts/check_workflows.py b/scripts/check_workflows.py index 022711f..e596015 100644 --- a/scripts/check_workflows.py +++ b/scripts/check_workflows.py @@ -1360,6 +1360,7 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: "Check package metadata rendering", "Inspect artifact identity and shape", "Install and smoke-test each exact artifact", + "Verify the immutable release as a copied standalone repository", "Record immutable artifact digests", "Retain only the verified release bundle", ], @@ -1437,10 +1438,24 @@ def check_publish_workflow(text: str, live_smoke_text: str) -> None: "Install and smoke-test each exact artifact": ( "uv run python scripts/check_clean_install.py dist/*" ), + "Verify the immutable release as a copied standalone repository": ( + "python scripts/check_repository_independence.py" + ), "Record immutable artifact digests": "sha256sum dist/* > artifact-sha256.txt", } - for name, command in build_commands.items(): - _named_run_step(build, name, command, "release build job") + build_command_indices = { + name: _named_run_step(build, name, command, "release build job")[0] + for name, command in build_commands.items() + } + if not ( + build_command_indices["Install and smoke-test each exact artifact"] + < build_command_indices["Verify the immutable release as a copied standalone repository"] + < build_command_indices["Record immutable artifact digests"] + ): + raise CheckError( + "release build must verify the copied standalone repository after exact-artifact " + "clean install and before recording immutable digests" + ) _, version_step = _named_run_step( build, "Verify project, manifest, changelog, release docs, and tag agreement", diff --git a/tests/test_changelog_gate.py b/tests/test_changelog_gate.py index 17bcb57..3603ab8 100644 --- a/tests/test_changelog_gate.py +++ b/tests/test_changelog_gate.py @@ -97,13 +97,24 @@ def test_changelog_parser_rejects_obfuscated_mutable_versions(claim: str) -> Non "## Unreleased", "## \uff35\uff4e\uff52\uff45\uff4c\uff45\uff41\uff53\uff45\uff44", "## Un\u034freleased", + "## Un\u202edesaeler\u202c", + "## Un\u2067released\u2069", + "## Current Unreleased", + "Next release (Unreleased)\n-------------------------", "Unreleased\n----------", "> ## Unreleased", "

Unreleased

", "

Unreleased", "

Unreleased

Archive

", "

Unreleased

", + "

Current Unreleased

", "Intro

Unreleased

", + "

Unreleased

", + "

Unreleased

", + "

Unreleased

", + "

Un