From 4e1209ca5db129881cba6b514d056679b2e9c94e Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 2 Sep 2026 10:56:36 +0200 Subject: [PATCH 1/2] POC: add documentation delta tool --- .github/workflows/on-pr.yml | 92 ++++ tools/BUILD | 8 + tools/docs_delta.py | 914 +++++++++++++++++++++++++++++++++ tools/tests/BUILD | 9 + tools/tests/docs_delta_test.py | 443 ++++++++++++++++ 5 files changed, 1466 insertions(+) create mode 100644 tools/docs_delta.py create mode 100644 tools/tests/docs_delta_test.py diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index fa8a7e7f6..837edf575 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -41,3 +41,95 @@ jobs: with: bazel-target: "//:docs" tests-report-artifact: tests-report + + docs-delta: + needs: [docs-build] + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read + steps: + - name: Check out pull request + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check out published documentation history + continue-on-error: true + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ github.repository }} + ref: gh-pages + path: .docs-baseline + fetch-depth: 0 + persist-credentials: false + + - name: Download documentation artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: github-pages + path: docs-artifact + + - name: Set up uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + - name: Generate documentation delta report + run: | + set -euo pipefail + uv run --no-project python tools/docs_delta.py + + - name: Upload documentation delta artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0 # v7.0.1 + with: + name: docs-delta + path: docs-artifact/docs-delta.md + if-no-files-found: error + + docs-comment: + needs: [docs-delta] + if: >- + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-24.04 + permissions: + actions: read + pull-requests: write + steps: + - name: Download documentation delta artifact + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: docs-delta + path: docs-delta + + - name: Prepare documentation preview comment + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPOSITORY_OWNER: ${{ github.repository_owner }} + REPOSITORY_NAME: ${{ github.event.repository.name }} + run: | + set -euo pipefail + { + echo "Documentation preview for this pull request is available at:" + echo "**pr-${PR_NUMBER}**: https://${REPOSITORY_OWNER}.github.io/${REPOSITORY_NAME}/pr-${PR_NUMBER}/" + echo + cat docs-delta/docs-delta.md + } > "$RUNNER_TEMP/docs-comment.md" + + - name: Find existing documentation preview comment + id: find-comment + uses: peter-evans/find-comment@b30e6a3c0ed37e7c023ccd3f1db5c6c0b0c23aad # v4.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: github-actions[bot] + body-includes: Documentation preview for this pull request + + - name: Create or update documentation preview comment + uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.find-comment.outputs.comment-id }} + body-path: ${{ runner.temp }}/docs-comment.md + edit-mode: replace diff --git a/tools/BUILD b/tools/BUILD index 8736e0af8..a9f41da5e 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -30,3 +30,11 @@ py_binary( main = "module_verification_reports.py", deps = [], ) + +py_binary( + name = "docs_delta", + srcs = ["docs_delta.py"], + main = "docs_delta.py", + visibility = ["//visibility:public"], + deps = [], +) diff --git a/tools/docs_delta.py b/tools/docs_delta.py new file mode 100644 index 000000000..c14552d92 --- /dev/null +++ b/tools/docs_delta.py @@ -0,0 +1,914 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Compare rendered documentation trees and write a Markdown delta report. + +The tool works on artifacts already produced by a docs build. In GitHub Actions +it can additionally resolve the matching published baseline from a local, +full-history checkout of ``gh-pages``. It never fetches from GitHub or invokes +Sphinx, which keeps the comparison useful both in CI and as a local command. +""" + +from __future__ import annotations + +import argparse +import io +import json +import os +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import quote + +JsonObject = dict[str, object] +NeedMap = Mapping[str, JsonObject] + +DETAIL_LIMIT = 15 +MAX_RENDERED_VALUE_LENGTH = 600 + +# These values are generated from source/test links or Sphinx's internal +# bookkeeping. They can change when a build is moved to another checkout or +# when the same source is rebuilt, without representing a documentation delta. +# Keep this list deliberately explicit: fields not listed here are part of the +# comparison contract and must be reviewed when Sphinx-Needs adds new output. +VOLATILE_NEED_FIELDS = frozenset( + { + "lineno", + "lineno_content", + "source", + "target_id", + "is_modified", + "source_code_link", + "testlink", + } +) + +_MISSING = object() +_META_TAG = re.compile(r"]*>", re.IGNORECASE) +_META_ATTRIBUTE = re.compile( + r"(?:name|property|itemprop)\s*=\s*([\"'])(.*?)\1", re.IGNORECASE +) +_HTML_COMMENT = re.compile(r"", re.IGNORECASE | re.DOTALL) +_VOLATILE_ATTRIBUTE = re.compile( + r"\s+data-(?:build|generated|timestamp|last-modified)(?:-[\w-]+)?\s*=\s*(?:\"[^\"]*\"|'[^']*'|[^\s>]+)", + re.IGNORECASE, +) +_VOLATILE_META_NAMES = frozenset( + { + "build-date", + "build_date", + "created", + "date", + "generated", + "generated-at", + "generated_at", + "generator", + "last-modified", + "last_modified", + "timestamp", + } +) +_VOLATILE_COMMENT_WORDS = re.compile( + r"\b(?:build|built|generated|generator|last\s+updated|timestamp)\b", + re.IGNORECASE, +) + + +class DocsDeltaError(ValueError): + """A user-actionable input or report-generation error.""" + + +@dataclass(frozen=True) +class GithubPullRequest: + """Pull request provenance obtained from the Actions environment.""" + + base_ref: str + base_sha: str + number: str | None + + +@dataclass(frozen=True) +class NeedChange: + """One Need and, for modified entries, its two versions.""" + + need_id: str + baseline: JsonObject | None + current: JsonObject | None + + @property + def changed_fields(self) -> tuple[str, ...]: + """Return changed non-volatile fields in deterministic order.""" + + if self.baseline is None or self.current is None: + return () + names = set(self.baseline) | set(self.current) + return tuple( + sorted( + name + for name in names + if name not in VOLATILE_NEED_FIELDS + and not _json_values_equal( + self.baseline.get(name, _MISSING), + self.current.get(name, _MISSING), + ) + ) + ) + + +@dataclass(frozen=True) +class NeedComparison: + """Categorized comparison result for a pair of Need inventories.""" + + added: tuple[NeedChange, ...] + removed: tuple[NeedChange, ...] + modified: tuple[NeedChange, ...] + unchanged_count: int + + +@dataclass(frozen=True) +class PageChange: + """One rendered HTML path that was added, removed, or modified.""" + + path: str + + +@dataclass(frozen=True) +class PageComparison: + """Categorized comparison result for rendered HTML pages.""" + + added: tuple[PageChange, ...] + removed: tuple[PageChange, ...] + modified: tuple[PageChange, ...] + unchanged_count: int + + +def _json_values_equal(left: object, right: object) -> bool: + """Compare JSON values without conflating booleans and numbers.""" + + if left is _MISSING or right is _MISSING: + return left is right + return json.dumps(left, sort_keys=True, ensure_ascii=False) == json.dumps( + right, sort_keys=True, ensure_ascii=False + ) + + +def _flatten_needs(path: Path) -> dict[str, JsonObject]: + """Load all Need entries from every version in a Sphinx-Needs export.""" + + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DocsDeltaError(f"cannot read needs JSON {path}: {exc}") from exc + + if not isinstance(payload, dict) or not isinstance(payload.get("versions"), dict): + raise DocsDeltaError(f"needs JSON has no versions map: {path}") + + result: dict[str, JsonObject] = {} + for version in payload["versions"].values(): + if not isinstance(version, dict) or not isinstance(version.get("needs"), dict): + continue + for need_id, need in version["needs"].items(): + if not isinstance(need_id, str) or not isinstance(need, dict): + raise DocsDeltaError(f"invalid Need entry in {path}") + result[need_id] = need + return result + + +def load_needs(directory: Path) -> dict[str, JsonObject]: + """Load the root ``needs.json`` file from a documentation directory.""" + + needs_path = directory / "needs.json" + if not needs_path.is_file(): + raise DocsDeltaError(f"needs JSON is missing: {needs_path}") + return _flatten_needs(needs_path) + + +def compare_needs(baseline: NeedMap, current: NeedMap) -> NeedComparison: + """Compare Needs by ID while ignoring only the volatile field allowlist.""" + + added: list[NeedChange] = [] + removed: list[NeedChange] = [] + modified: list[NeedChange] = [] + unchanged_count = 0 + + for need_id in sorted(set(baseline) | set(current)): + old = baseline.get(need_id) + new = current.get(need_id) + change = NeedChange(need_id, old, new) + if old is None: + added.append(change) + elif new is None: + removed.append(change) + elif change.changed_fields: + modified.append(change) + else: + unchanged_count += 1 + + return NeedComparison( + added=tuple(added), + removed=tuple(removed), + modified=tuple(modified), + unchanged_count=unchanged_count, + ) + + +def _html_metadata_name(tag: str) -> str | None: + for match in _META_ATTRIBUTE.finditer(tag): + name = match.group(2).lower() + if name in _VOLATILE_META_NAMES: + return name + return None + + +def normalize_html(content: str) -> str: + """Normalize generated HTML without hiding ordinary page content changes.""" + + normalized = content.replace("\r\n", "\n").replace("\r", "\n") + + def remove_meta(match: re.Match[str]) -> str: + return "" if _html_metadata_name(match.group(0)) else match.group(0) + + normalized = _META_TAG.sub(remove_meta, normalized) + + def remove_comment(match: re.Match[str]) -> str: + return "" if _VOLATILE_COMMENT_WORDS.search(match.group(0)) else match.group(0) + + normalized = _HTML_COMMENT.sub(remove_comment, normalized) + normalized = _VOLATILE_ATTRIBUTE.sub("", normalized) + normalized = "\n".join(line.rstrip() for line in normalized.splitlines()) + return normalized.strip() + + +def _html_files(directory: Path) -> dict[str, str]: + """Return normalized HTML keyed by POSIX-relative path.""" + + if not directory.is_dir(): + raise DocsDeltaError(f"documentation directory is missing: {directory}") + result: dict[str, str] = {} + for path in sorted(directory.rglob("*.html")): + if path.is_file(): + relative = path.relative_to(directory).as_posix() + try: + result[relative] = normalize_html(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError) as exc: + raise DocsDeltaError( + f"cannot read rendered page {path}: {exc}" + ) from exc + return result + + +def compare_html(baseline_dir: Path, current_dir: Path) -> PageComparison: + """Compare normalized recursive HTML output from two build directories.""" + + baseline = _html_files(baseline_dir) + current = _html_files(current_dir) + added: list[PageChange] = [] + removed: list[PageChange] = [] + modified: list[PageChange] = [] + unchanged_count = 0 + + for path in sorted(set(baseline) | set(current)): + old = baseline.get(path) + new = current.get(path) + change = PageChange(path) + if old is None: + added.append(change) + elif new is None: + removed.append(change) + elif old != new: + modified.append(change) + else: + unchanged_count += 1 + + return PageComparison( + added=tuple(added), + removed=tuple(removed), + modified=tuple(modified), + unchanged_count=unchanged_count, + ) + + +def _url_with_path(base_url: str, relative_path: str, anchor: str | None = None) -> str: + url = ( + base_url.rstrip("/") + + "/" + + "/".join( + quote(part, safe="-._~") for part in relative_path.strip("/").split("/") + ) + ) + if anchor: + url += "#" + quote(anchor, safe="-._~") + return url + + +def _need_url(base_url: str, need: Mapping[str, object]) -> str | None: + docname = need.get("docname") + if not isinstance(docname, str) or not docname.strip(): + return None + rendered_name = docname.strip("/") + if not rendered_name.endswith(".html"): + rendered_name += ".html" + return _url_with_path(base_url, rendered_name) + + +def _need_link(base_url: str, need_id: str, need: Mapping[str, object]) -> str | None: + page_url = _need_url(base_url, need) + if page_url is None: + return None + return page_url + "#" + quote(need_id, safe="-._~") + + +def _markdown_link(label: str, url: str | None) -> str: + if url is None: + return f"`{label}`" + safe_label = label.replace("[", "\\[").replace("]", "\\]") + return f"[{safe_label}]({url})" + + +def _display_name(need: Mapping[str, object]) -> str: + for field in ("title", "name"): + value = need.get(field) + if isinstance(value, str) and value: + return value + return "" + + +def _format_value(value: object) -> str: + if value is _MISSING: + return "(missing)" + rendered = json.dumps( + value, sort_keys=True, ensure_ascii=False, separators=(",", ":") + ) + if len(rendered) > MAX_RENDERED_VALUE_LENGTH: + rendered = rendered[:MAX_RENDERED_VALUE_LENGTH] + "…" + return rendered.replace("`", "\\`").replace("\n", " ") + + +def _format_need_entry( + change: NeedChange, *, base_url: str, pr_url: str, include_diff: bool = False +) -> list[str]: + need = change.current or change.baseline + assert need is not None + links = [] + old_link = ( + _need_link(base_url, change.need_id, change.baseline) + if change.baseline + else None + ) + new_link = ( + _need_link(pr_url, change.need_id, change.current) if change.current else None + ) + if old_link: + links.append(_markdown_link("old", old_link)) + if new_link: + links.append(_markdown_link("new", new_link)) + link_text = " / ".join(links) + title = _display_name(need) + suffix = f" — {title}" if title else "" + line = f"- `{change.need_id}`{suffix}" + if link_text: + line += f" ({link_text})" + result = [line] + if include_diff: + assert change.baseline is not None and change.current is not None + for field in change.changed_fields: + result.append( + f" - `{field}`: {_format_value(change.baseline.get(field, _MISSING))} " + f"→ {_format_value(change.current.get(field, _MISSING))}" + ) + return result + + +def _format_page_entry( + change: PageChange, *, base_url: str, pr_url: str, kind: str +) -> str: + old = _url_with_path(base_url, change.path) if kind != "added" else None + new = _url_with_path(pr_url, change.path) if kind != "removed" else None + links = [] + if old: + links.append(_markdown_link("old", old)) + if new: + links.append(_markdown_link("new", new)) + return f"- `{change.path}` ({' / '.join(links)})" + + +def _section( + lines: list[str], + title: str, + entries: Sequence[object], + formatter, + *, + base_url: str, + pr_url: str, + kind: str = "", +) -> None: + if not entries: + return + lines.extend([f"### {title} ({len(entries)})", ""]) + if len(entries) > DETAIL_LIMIT: + lines.extend([f"{len(entries)} entries changed; details omitted.", ""]) + return + for entry in entries: + if isinstance(entry, NeedChange): + lines.extend( + formatter( + entry, + base_url=base_url, + pr_url=pr_url, + include_diff=kind == "modified", + ) + ) + else: + lines.append(formatter(entry, base_url=base_url, pr_url=pr_url, kind=kind)) + lines.append("") + + +def render_report( + needs: NeedComparison, + pages: PageComparison, + *, + base_url: str, + pr_url: str, +) -> str: + """Render a deterministic Markdown report for comparison results.""" + + lines = [ + "# Documentation delta", + "", + f"Baseline: {_markdown_link(base_url, base_url)} ", + f"PR preview: {_markdown_link(pr_url, pr_url)}", + "", + "## Summary", + "", + f"- Needs: {len(needs.added)} added, {len(needs.removed)} removed, " + f"{len(needs.modified)} modified, {needs.unchanged_count} unchanged", + f"- Rendered pages: {len(pages.added)} added, {len(pages.removed)} removed, " + f"{len(pages.modified)} modified, {pages.unchanged_count} unchanged", + "", + "## Needs", + "", + ] + _section( + lines, + "Added", + needs.added, + _format_need_entry, + base_url=base_url, + pr_url=pr_url, + kind="added", + ) + _section( + lines, + "Removed", + needs.removed, + _format_need_entry, + base_url=base_url, + pr_url=pr_url, + kind="removed", + ) + _section( + lines, + "Modified", + needs.modified, + _format_need_entry, + base_url=base_url, + pr_url=pr_url, + kind="modified", + ) + lines.extend(["## Rendered HTML pages", ""]) + _section( + lines, + "Added", + pages.added, + _format_page_entry, + base_url=base_url, + pr_url=pr_url, + kind="added", + ) + _section( + lines, + "Removed", + pages.removed, + _format_page_entry, + base_url=base_url, + pr_url=pr_url, + kind="removed", + ) + _section( + lines, + "Modified", + pages.modified, + _format_page_entry, + base_url=base_url, + pr_url=pr_url, + kind="modified", + ) + return "\n".join(lines).rstrip() + "\n" + + +def unavailable_report(reason: str) -> str: + """Render the successful, explicit report used when baseline is unavailable.""" + + return ( + "# Documentation delta\n\n" + "## Delta unavailable\n\n" + f"The documentation baseline is unavailable: {reason}\n" + ) + + +def _write_report(path: Path, content: str) -> None: + """Atomically write the report so a failed run never leaves partial Markdown.""" + + path.parent.mkdir(parents=True, exist_ok=True) + temporary_path: Path | None = None + file_descriptor: int | None = None + try: + file_descriptor, temporary_name = tempfile.mkstemp( + dir=path.parent, prefix=f".{path.name}.", suffix=".tmp" + ) + temporary_path = Path(temporary_name) + with os.fdopen(file_descriptor, "w", encoding="utf-8") as output: + file_descriptor = None + output.write(content) + output.flush() + os.replace(temporary_path, path) + temporary_path = None + finally: + if file_descriptor is not None: + os.close(file_descriptor) + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def _workspace_path(environ: Mapping[str, str], relative_path: str) -> Path: + workspace = environ.get("GITHUB_WORKSPACE") + if workspace: + return Path(workspace) / relative_path + return Path(relative_path) + + +def _github_pull_request(environ: Mapping[str, str]) -> GithubPullRequest | None: + """Read pull request provenance from standard GitHub Actions variables.""" + + event_name = environ.get("GITHUB_EVENT_NAME") + event_path = environ.get("GITHUB_EVENT_PATH") + if event_name != "pull_request": + return None + if not event_path: + raise DocsDeltaError( + "GITHUB_EVENT_PATH is required to resolve a pull request baseline" + ) + + try: + payload = json.loads(Path(event_path).read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise DocsDeltaError( + f"cannot read GitHub event payload {event_path}: {exc}" + ) from exc + if not isinstance(payload, dict): + raise DocsDeltaError(f"GitHub event payload is not an object: {event_path}") + + pull_request = payload.get("pull_request") + if not isinstance(pull_request, dict): + return None + base = pull_request.get("base") + if not isinstance(base, dict): + return None + + base_ref = environ.get("GITHUB_BASE_REF") or base.get("ref") + base_sha = environ.get("GITHUB_BASE_SHA") or base.get("sha") + if not isinstance(base_ref, str) or not base_ref: + return None + if not isinstance(base_sha, str) or not base_sha: + return None + + number = pull_request.get("number") + if not isinstance(number, int | str) or isinstance(number, bool): + number = None + else: + number = str(number) + return GithubPullRequest(base_ref=base_ref, base_sha=base_sha, number=number) + + +def _find_published_baseline_commit( + gh_pages_dir: Path, pull_request: GithubPullRequest +) -> tuple[str | None, str]: + """Find the newest gh-pages commit whose generated Needs contain the base SHA.""" + + if not gh_pages_dir.is_dir(): + return None, f"gh-pages checkout is missing: {gh_pages_dir}" + + try: + git_check = subprocess.run( + ["git", "-C", str(gh_pages_dir), "rev-parse", "--git-dir"], + check=False, + capture_output=True, + text=True, + ) + except OSError as exc: + return None, f"cannot inspect gh-pages checkout {gh_pages_dir}: {exc}" + if git_check.returncode != 0: + return None, f"gh-pages checkout is not a Git repository: {gh_pages_dir}" + + needs_path = f"{pull_request.base_ref}/needs.json" + try: + result = subprocess.run( + [ + "git", + "-C", + str(gh_pages_dir), + "log", + "--all", + "--full-history", + "--format=%H", + "--", + needs_path, + ], + check=False, + capture_output=True, + text=True, + ) + except OSError as exc: + return None, f"cannot search gh-pages history: {exc}" + if result.returncode != 0: + detail = result.stderr.strip() or "git log failed" + return None, f"cannot search gh-pages history: {detail}" + + candidate_commits = [ + line.strip() for line in result.stdout.splitlines() if line.strip() + ] + for commit in candidate_commits: + try: + needs = subprocess.run( + [ + "git", + "-C", + str(gh_pages_dir), + "show", + f"{commit}:{needs_path}", + ], + check=False, + capture_output=True, + ) + except OSError as exc: + return None, f"cannot inspect published needs JSON: {exc}" + if needs.returncode == 0 and pull_request.base_sha.encode() in needs.stdout: + return commit, "" + + return ( + None, + f"no published {pull_request.base_ref} baseline contains " + f"source commit {pull_request.base_sha}", + ) + + +def _extract_git_tree( + gh_pages_dir: Path, commit: str, tree_path: str, destination: Path +) -> None: + """Extract one published documentation tree without changing the checkout.""" + + try: + result = subprocess.run( + [ + "git", + "-C", + str(gh_pages_dir), + "archive", + "--format=tar", + f"{commit}:{tree_path}", + ], + check=False, + capture_output=True, + ) + except OSError as exc: + raise DocsDeltaError(f"cannot extract gh-pages baseline: {exc}") from exc + if result.returncode != 0: + detail = result.stderr.decode(errors="replace").strip() or "git archive failed" + raise DocsDeltaError(f"cannot extract gh-pages baseline: {detail}") + + destination.mkdir(parents=True, exist_ok=True) + try: + with tarfile.open(fileobj=io.BytesIO(result.stdout), mode="r:") as archive: + for member in archive.getmembers(): + relative = Path(member.name) + if relative.is_absolute() or ".." in relative.parts: + raise DocsDeltaError( + f"gh-pages archive contains unsafe path: {member.name}" + ) + target = destination / relative + if member.isdir(): + target.mkdir(parents=True, exist_ok=True) + elif member.isfile(): + target.parent.mkdir(parents=True, exist_ok=True) + source = archive.extractfile(member) + if source is None: + raise DocsDeltaError( + f"cannot read gh-pages archive member: {member.name}" + ) + with source, target.open("wb") as output: + shutil.copyfileobj(source, output) + else: + raise DocsDeltaError( + f"unsupported gh-pages archive member: {member.name}" + ) + except (OSError, tarfile.TarError) as exc: + raise DocsDeltaError(f"cannot extract gh-pages baseline: {exc}") from exc + + +@contextmanager +def _resolved_baseline( + args: argparse.Namespace, + github_pull_request: GithubPullRequest | None, + environ: Mapping[str, str], +) -> Iterator[tuple[Path | None, str]]: + """Resolve either a directory baseline or a published gh-pages baseline.""" + + if args.baseline_mode is None: + mode = ( + "directory" + if args.baseline_dir is not None or github_pull_request is None + else "gh-pages" + ) + else: + mode = args.baseline_mode + + if mode == "directory": + if args.baseline_dir is None: + raise DocsDeltaError( + "--baseline-dir is required outside GitHub pull request mode" + ) + yield args.baseline_dir, "" + return + + if args.baseline_dir is not None: + raise DocsDeltaError( + "--baseline-dir cannot be used with --baseline-mode gh-pages" + ) + if github_pull_request is None: + raise DocsDeltaError( + "gh-pages baseline mode requires a GitHub pull request event" + ) + + gh_pages_dir = args.gh_pages_dir or _workspace_path(environ, ".docs-baseline") + commit, reason = _find_published_baseline_commit(gh_pages_dir, github_pull_request) + if commit is None: + yield None, reason + return + + with tempfile.TemporaryDirectory(prefix="docs-delta-baseline-") as temporary_dir: + baseline_dir = Path(temporary_dir) + try: + _extract_git_tree( + gh_pages_dir, + commit, + github_pull_request.base_ref, + baseline_dir, + ) + except DocsDeltaError as exc: + yield None, str(exc) + return + yield baseline_dir, "" + + +def _github_url_defaults( + environ: Mapping[str, str], + github_pull_request: GithubPullRequest | None = None, +) -> tuple[str | None, str | None]: + """Derive conventional GitHub Pages URLs from standard Actions variables.""" + + repository = environ.get("GITHUB_REPOSITORY", "") + if "/" not in repository: + return None, None + owner, name = repository.split("/", 1) + if not owner or not name: + return None, None + pages_root = environ.get("GITHUB_PAGES_URL") or f"https://{owner}.github.io/{name}" + base_ref = ( + github_pull_request.base_ref + if github_pull_request + else (environ.get("GITHUB_BASE_REF") or "main") + ) + if github_pull_request and github_pull_request.number: + pr_ref = f"pr-{github_pull_request.number}" + else: + pr_ref = ( + environ.get("GITHUB_HEAD_REF") or environ.get("GITHUB_REF_NAME") or base_ref + ) + + def pages_ref_url(ref: str) -> str: + # A branch name is one URL path segment. In particular, a slash in a + # feature branch must not become a second documentation path segment. + return pages_root.rstrip("/") + "/" + quote(ref, safe="-._~") + + return ( + pages_ref_url(base_ref), + pages_ref_url(pr_ref), + ) + + +def _resolve_urls( + args: argparse.Namespace, github_pull_request: GithubPullRequest | None +) -> tuple[str, str]: + defaults = _github_url_defaults(os.environ, github_pull_request) + base_url = args.base_url or defaults[0] + pr_url = args.pr_url or defaults[1] + if not base_url or not pr_url: + raise DocsDeltaError( + "documentation URLs are required; provide --base-url and --pr-url " + "or run in GitHub Actions with GITHUB_REPOSITORY set" + ) + return base_url, pr_url + + +def argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--baseline-dir", + type=Path, + help="directory baseline; defaults to automatic gh-pages mode in PR Actions", + ) + parser.add_argument( + "--baseline-mode", + choices=("directory", "gh-pages"), + help="baseline source; inferred when omitted", + ) + parser.add_argument( + "--gh-pages-dir", + type=Path, + help="local full-history gh-pages checkout (defaults to .docs-baseline)", + ) + parser.add_argument( + "--current-dir", + type=Path, + help="current documentation directory (defaults to docs-artifact)", + ) + parser.add_argument("--base-url") + parser.add_argument("--pr-url") + parser.add_argument( + "--output", + type=Path, + help="report path (defaults to docs-artifact/docs-delta.md)", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the CLI and return a non-zero status only for current-input errors.""" + + args = argument_parser().parse_args(argv) + try: + github_pull_request = _github_pull_request(os.environ) + current_dir = args.current_dir or _workspace_path(os.environ, "docs-artifact") + output = args.output or current_dir / "docs-delta.md" + + with _resolved_baseline(args, github_pull_request, os.environ) as ( + baseline_dir, + baseline_reason, + ): + if baseline_dir is None or not baseline_dir.is_dir(): + reason = baseline_reason or f"directory is missing: {baseline_dir}" + _write_report(output, unavailable_report(reason)) + return 0 + try: + baseline_needs = load_needs(baseline_dir) + except DocsDeltaError as exc: + _write_report(output, unavailable_report(str(exc))) + return 0 + + if not current_dir.is_dir(): + raise DocsDeltaError( + f"documentation directory is missing: {current_dir}" + ) + current_needs = load_needs(current_dir) + base_url, pr_url = _resolve_urls(args, github_pull_request) + report = render_report( + compare_needs(baseline_needs, current_needs), + compare_html(baseline_dir, current_dir), + base_url=base_url, + pr_url=pr_url, + ) + _write_report(output, report) + except (DocsDeltaError, OSError) as exc: + print(f"docs_delta: error: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/tests/BUILD b/tools/tests/BUILD index e84b502a7..73772cdb9 100644 --- a/tools/tests/BUILD +++ b/tools/tests/BUILD @@ -23,3 +23,12 @@ score_pytest( ] + all_requirements, pytest_config = "//:pyproject.toml", ) + +score_pytest( + name = "docs_delta_test", + srcs = ["docs_delta_test.py"], + deps = [ + "//tools:docs_delta", + ] + all_requirements, + pytest_config = "//:pyproject.toml", +) diff --git a/tools/tests/docs_delta_test.py b/tools/tests/docs_delta_test.py new file mode 100644 index 000000000..2fb658b65 --- /dev/null +++ b/tools/tests/docs_delta_test.py @@ -0,0 +1,443 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License, Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Unit and fixture-based integration tests for the documentation delta tool.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest + +from tools import docs_delta + + +def _need(docname: str, title: str, **extra: object) -> dict[str, object]: + return {"id": title.lower(), "docname": docname, "title": title, **extra} + + +def _write_needs(directory: Path, needs: dict[str, dict[str, object]]) -> None: + (directory / "needs.json").write_text( + json.dumps({"versions": {"1": {"needs": needs}}}), encoding="utf-8" + ) + + +def _git(directory: Path, *arguments: str) -> None: + subprocess.run( + ["git", "-C", str(directory), *arguments], + check=True, + capture_output=True, + text=True, + ) + + +def test_compare_needs_categorizes_changes_and_ignores_volatile_fields() -> None: + baseline = { + "same": _need( + "same", + "Same", + id="same", + lineno=10, + source="old.rst", + source_code_link="old-commit", + testlink="old-result", + ), + "changed": _need( + "changed", + "Old", + id="changed", + lineno=10, + source="old.rst", + source_code_link="old-commit", + testlink="old-result", + ), + "removed": _need("removed", "Removed", id="removed"), + } + current = { + "same": _need( + "same", + "Same", + id="same", + lineno=999, + source="new.rst", + source_code_link="new-commit", + testlink="new-result", + ), + "changed": _need( + "changed", + "New", + id="changed", + lineno=20, + source="new.rst", + source_code_link="new-commit", + testlink="new-result", + ), + "added": _need("added", "Added", id="added"), + } + + result = docs_delta.compare_needs(baseline, current) + + assert [entry.need_id for entry in result.added] == ["added"] + assert [entry.need_id for entry in result.removed] == ["removed"] + assert [entry.need_id for entry in result.modified] == ["changed"] + assert result.modified[0].changed_fields == ("title",) + assert result.unchanged_count == 1 + + +def test_modified_need_reports_all_changed_fields_and_one_sided_links() -> None: + result = docs_delta.render_report( + docs_delta.NeedComparison( + added=(docs_delta.NeedChange("added", None, _need("new/page", "Added")),), + removed=( + docs_delta.NeedChange("removed", _need("old/page", "Removed"), None), + ), + modified=( + docs_delta.NeedChange( + "changed", + _need("old/page", "Old", tags=["old"], extra="before"), + _need("new/page", "New", tags=["new"], extra="after"), + ), + ), + unchanged_count=0, + ), + docs_delta.PageComparison((), (), (), 0), + base_url="https://docs.example/main", + pr_url="https://docs.example/pr/1", + ) + + assert "[old](https://docs.example/main/old/page.html#changed)" in result + assert "[new](https://docs.example/pr/1/new/page.html#changed)" in result + assert '`extra`: "before" → "after"' in result + assert '`tags`: ["old"] → ["new"]' in result + assert "https://docs.example/pr/1/new/page.html#added" in result + assert "https://docs.example/main/old/page.html#removed" in result + + +def test_large_changed_values_are_truncated() -> None: + large_old = "a" * (docs_delta.MAX_RENDERED_VALUE_LENGTH + 20) + large_new = "b" * (docs_delta.MAX_RENDERED_VALUE_LENGTH + 20) + change = docs_delta.NeedChange( + "changed", + _need("page", "Old", details=large_old), + _need("page", "New", details=large_new), + ) + + report = docs_delta.render_report( + docs_delta.NeedComparison((), (), (change,), 0), + docs_delta.PageComparison((), (), (), 0), + base_url="https://docs.example/main", + pr_url="https://docs.example/pr/1", + ) + + assert "…" in report + assert large_old not in report + assert large_new not in report + + +def test_normalize_html_ignores_generated_metadata_but_keeps_content_changes() -> None: + old = ( + '\r\n' + '\r\n' + "\r\n" + "

Documentation

\r\n" + ) + new = ( + '\n' + '\n' + "\n" + "

Documentation

\n" + ) + + assert docs_delta.normalize_html(old) == docs_delta.normalize_html(new) + assert docs_delta.normalize_html(new).replace( + "Documentation", "Changed" + ) != docs_delta.normalize_html(old) + + +def test_compare_html_categorizes_added_removed_modified_and_unchanged( + tmp_path: Path, +) -> None: + baseline = tmp_path / "baseline" + current = tmp_path / "current" + (baseline / "nested").mkdir(parents=True) + (current / "nested").mkdir(parents=True) + (baseline / "same.html").write_text("

same

\n", encoding="utf-8") + (current / "same.html").write_text("

same

\n", encoding="utf-8") + (baseline / "changed.html").write_text("

old

\n", encoding="utf-8") + (current / "changed.html").write_text("

new

\n", encoding="utf-8") + (baseline / "removed.html").write_text("

removed

\n", encoding="utf-8") + (current / "added.html").write_text("

added

\n", encoding="utf-8") + (baseline / "nested/page.html").write_text("

old

\n", encoding="utf-8") + (current / "nested/page.html").write_text("

old

\n", encoding="utf-8") + + result = docs_delta.compare_html(baseline, current) + + assert [entry.path for entry in result.added] == ["added.html"] + assert [entry.path for entry in result.removed] == ["removed.html"] + assert [entry.path for entry in result.modified] == ["changed.html"] + assert result.unchanged_count == 2 + + +def test_detail_threshold_is_independent_for_needs_and_pages() -> None: + needs = docs_delta.NeedComparison( + added=tuple( + docs_delta.NeedChange(str(index), None, _need("page", str(index))) + for index in range(16) + ), + removed=(), + modified=(), + unchanged_count=0, + ) + pages = docs_delta.PageComparison( + added=tuple(docs_delta.PageChange(f"page-{index}.html") for index in range(2)), + removed=(), + modified=(), + unchanged_count=0, + ) + + report = docs_delta.render_report( + needs, + pages, + base_url="https://docs.example/main", + pr_url="https://docs.example/pr/1", + ) + + assert "16 entries changed; details omitted." in report + assert "`page-0.html`" in report + assert "`0`" not in report + + +def test_missing_baseline_writes_unavailable_report(tmp_path: Path) -> None: + output = tmp_path / "nested" / "delta.md" + + assert ( + docs_delta.main( + [ + "--baseline-dir", + str(tmp_path / "missing"), + "--current-dir", + str(tmp_path / "current"), + "--base-url", + "https://docs.example/main", + "--pr-url", + "https://docs.example/pr/1", + "--output", + str(output), + ] + ) + == 0 + ) + assert "## Delta unavailable" in output.read_text(encoding="utf-8") + + +def test_cli_writes_complete_fixture_report(tmp_path: Path) -> None: + baseline = tmp_path / "baseline" + current = tmp_path / "current" + baseline.mkdir() + current.mkdir() + _write_needs( + baseline, + { + "same": _need("same", "Same", id="same"), + "changed": _need("guide", "Old title", id="changed"), + "removed": _need("removed", "Removed", id="removed"), + }, + ) + _write_needs( + current, + { + "same": _need("same", "Same", id="same"), + "changed": _need("guide", "New title", id="changed"), + "added": _need("new", "Added", id="added"), + }, + ) + (baseline / "guide.html").write_text("

old

\n", encoding="utf-8") + (current / "guide.html").write_text("

new

\n", encoding="utf-8") + (current / "new.html").write_text("

new page

\n", encoding="utf-8") + output = tmp_path / "delta.md" + + assert ( + docs_delta.main( + [ + "--baseline-dir", + str(baseline), + "--current-dir", + str(current), + "--base-url", + "https://docs.example/main", + "--pr-url", + "https://docs.example/pr/1", + "--output", + str(output), + ] + ) + == 0 + ) + report = output.read_text(encoding="utf-8") + assert "Needs: 1 added, 1 removed, 1 modified, 1 unchanged" in report + assert "Rendered pages: 1 added, 0 removed, 1 modified, 0 unchanged" in report + assert '`title`: "Old title" → "New title"' in report + assert ( + "`guide.html` ([old](https://docs.example/main/guide.html) / [new](https://docs.example/pr/1/guide.html))" + in report + ) + + +def test_cli_uses_github_environment_urls_when_options_are_omitted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + baseline = tmp_path / "baseline" + current = tmp_path / "current" + baseline.mkdir() + current.mkdir() + _write_needs(baseline, {}) + _write_needs(current, {}) + monkeypatch.setenv("GITHUB_REPOSITORY", "eclipse-score/docs-as-code") + monkeypatch.setenv("GITHUB_BASE_REF", "main") + monkeypatch.setenv("GITHUB_HEAD_REF", "feature/docs-delta") + output = tmp_path / "delta.md" + + assert ( + docs_delta.main( + [ + "--baseline-dir", + str(baseline), + "--current-dir", + str(current), + "--output", + str(output), + ] + ) + == 0 + ) + report = output.read_text(encoding="utf-8") + assert "https://eclipse-score.github.io/docs-as-code/main" in report + assert "https://eclipse-score.github.io/docs-as-code/feature%2Fdocs-delta" in report + + +def test_cli_automatically_selects_matching_gh_pages_baseline( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """PR mode must select the historical publication matching the base SHA.""" + + gh_pages = tmp_path / ".docs-baseline" + gh_pages.mkdir() + subprocess.run( + ["git", "init", "--initial-branch=gh-pages", str(gh_pages)], + check=True, + capture_output=True, + text=True, + ) + _git(gh_pages, "config", "user.name", "Documentation Delta Test") + _git(gh_pages, "config", "user.email", "docs-delta@example.invalid") + + base_sha = "0123456789abcdef0123456789abcdef01234567" + latest_sha = "fedcba9876543210fedcba9876543210fedcba98" + main_docs = gh_pages / "main" + main_docs.mkdir() + _write_needs( + main_docs, + { + "need": _need( + "guide", "Base publication", id="need", source_code_link=base_sha + ) + }, + ) + (main_docs / "guide.html").write_text("

Base publication

\n", encoding="utf-8") + _git(gh_pages, "add", ".") + _git(gh_pages, "commit", "-m", "Publish base documentation") + + _write_needs( + main_docs, + { + "need": _need( + "guide", + "Newer base publication", + id="need", + source_code_link=base_sha, + ) + }, + ) + (main_docs / "guide.html").write_text( + "

Newer base publication

\n", encoding="utf-8" + ) + _git(gh_pages, "add", ".") + _git(gh_pages, "commit", "-m", "Republish base documentation") + newer_base_publication_commit = subprocess.run( + ["git", "-C", str(gh_pages), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + _write_needs( + main_docs, + { + "need": _need( + "guide", "Latest publication", id="need", source_code_link=latest_sha + ) + }, + ) + (main_docs / "guide.html").write_text( + "

Latest publication

\n", encoding="utf-8" + ) + _git(gh_pages, "add", ".") + _git(gh_pages, "commit", "-m", "Publish latest documentation") + + selected_commit, selection_reason = docs_delta._find_published_baseline_commit( + gh_pages, + docs_delta.GithubPullRequest("main", base_sha, "123"), + ) + assert selected_commit == newer_base_publication_commit, selection_reason + + current = tmp_path / "docs-artifact" + current.mkdir() + _write_needs( + current, + { + "need": _need( + "guide", + "Newer base publication", + id="need", + source_code_link="pr-sha", + ) + }, + ) + (current / "guide.html").write_text( + "

Newer base publication

\n", encoding="utf-8" + ) + + event = tmp_path / "event.json" + event.write_text( + json.dumps( + { + "pull_request": { + "number": 123, + "base": {"ref": "main", "sha": base_sha}, + } + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("GITHUB_EVENT_NAME", "pull_request") + monkeypatch.setenv("GITHUB_EVENT_PATH", str(event)) + monkeypatch.setenv("GITHUB_WORKSPACE", str(tmp_path)) + monkeypatch.setenv("GITHUB_REPOSITORY", "eclipse-score/docs-as-code") + monkeypatch.setenv("GITHUB_BASE_REF", "main") + + assert docs_delta.main([]) == 0 + report = (current / "docs-delta.md").read_text(encoding="utf-8") + assert "Needs: 0 added, 0 removed, 0 modified, 1 unchanged" in report + assert "Rendered pages: 0 added, 0 removed, 0 modified, 1 unchanged" in report + assert "https://eclipse-score.github.io/docs-as-code/pr-123" in report From 33a080b75e630acbeab1545c857c4111fb89061f Mon Sep 17 00:00:00 2001 From: Alexander Lanin Date: Wed, 2 Sep 2026 11:12:18 +0200 Subject: [PATCH 2/2] Fix upload artifact action pin --- .github/workflows/on-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on-pr.yml b/.github/workflows/on-pr.yml index 837edf575..5d9d01218 100644 --- a/.github/workflows/on-pr.yml +++ b/.github/workflows/on-pr.yml @@ -82,7 +82,7 @@ jobs: uv run --no-project python tools/docs_delta.py - name: Upload documentation delta artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0 # v7.0.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: docs-delta path: docs-artifact/docs-delta.md