diff --git a/CHANGELOG.md b/CHANGELOG.md index 08dcbe8..4138872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,13 +48,14 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index e130f9f..8c197bd 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -452,6 +453,75 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (e.g. the Turkish dotless i); + NFC for names that arrive decomposed (e.g. ones zipped on macOS). An + approximation of NTFS's and APFS's tables, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + +def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: + """The file ``candidate`` names, matched exactly or by folded name. + + Where the authoring filesystem folds case, as on Windows and macOS, an + inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside + ``Dict.lift-ranges`` is a pair there but not everywhere. + + An exact hit is always returned unchanged: folding runs only after the + exact name misses, and then only on the final component — the hrefs this + serves are basenames or same-folder relatives. + + Among names that fold together the first in code point order wins: + arbitrary, but deterministic, which directory order is not. + + ``listings`` caches one directory read per folder. + """ + try: + if candidate.is_file(): + return candidate + if candidate.is_dir(): + # An href of "" or "sub/" lands here; folding a folder's own name + # would search its parent and match anything spelled like it. + return None + except OSError: + pass # unstattable exact spelling: a case variant of it may still stat + folder = candidate.parent + if folder not in listings: + files: dict[str, Path] = {} + try: + # By name, not by Path: PurePath ordering is case-folded on Windows, + # which would leave the tie-break to directory order there. + for path in sorted(folder.iterdir(), key=lambda entry: entry.name): + if path.is_file(): + files.setdefault(_fold(path.name), path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = files + return listings[folder].get(_fold(candidate.name)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths that fold together denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. + + Both sides resolve first, so ``..`` segments and symlinks compare alike; + the fold pre-check then keeps the inode comparison from conflating + distinct files where ``st_ino`` is 0. + """ + try: + if _fold(str(left.resolve())) != _fold(str(right.resolve())): + return False + return left.samefile(right) + except OSError: + return False + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -505,13 +575,17 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L With ``resolve_ranges`` (the default), companion ``.lift-ranges`` files are loaded and tracked in :attr:`ranges_files`. Several - candidates are tried and every one that exists is loaded: the - conventional ``.lift-ranges`` sibling, and for each header + candidates are tried and every distinct file among them is loaded: + the conventional ``.lift-ranges`` sibling, and for each header ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). + A candidate matching no file exactly resolves across differences in + case or Unicode normalization, so a folder authored on Windows loads + the same way everywhere; the ``.lift`` is never its own companion. + A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's lifetime) and the single contained ``.lift`` is loaded. @@ -533,8 +607,10 @@ def _resolve_ranges(self) -> None: return base = self.path.parent candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) + # with_name and with_suffix agree on every name that has an extension, + # but with_suffix would raise on a name that has none — which + # parse_document accepts, since it never inspects the extension. + candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: continue @@ -544,14 +620,24 @@ def _resolve_ranges(self) -> None: basename = range_.href.replace("\\", "/").rpartition("/")[2] if basename: candidates.append(base / basename) + listings: dict[Path, dict[str, Path]] = {} for candidate in candidates: + found = _existing_file(candidate, listings) + if found is None: + continue try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + # A header href naming the .lift in another case folds onto it, and + # RangesFile.load rejects that root, failing the whole load; two + # spellings of one companion, which resolve() leaves distinct on + # macOS, would load and write it twice. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (self.path, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 39e0b25..ee9d468 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href, _same_file from ._text import Multitext, Trait if TYPE_CHECKING: @@ -431,13 +431,13 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: file=file, ) - # Header references (relative) that resolve to no companion. - # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve - # (they are resolved by basename when the companion is in the same folder) - # and are not checked here; this catches an exporter that writes a relative - # href but not the file. + # Header references that resolve to no companion — an exporter + # that wrote the href but not the file. Absolute and file:// hrefs are + # skipped: FLEx writes those knowing they will not resolve, and load reaches + # their companions by basename in the same folder instead. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, Path]] = {} for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -447,7 +447,10 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: resolved = all_ranges.get(range_.id) if resolved is not None and resolved.elements: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + found = _existing_file(base / relative, listings) + # _resolve_ranges refuses the lexicon as its own companion, so an + # href folding onto it supplies nothing and dangles too. + if found is None or _same_file(found, lexicon.path): yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..dde9835 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -225,6 +226,134 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename spellings. + + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + +def _case_sensitive(folder: Path) -> bool: + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the + # tie-break picks one: lexicographically first, the same one every run. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not look outside the folder for a name not in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> None: + # The ".." keeps the href from matching the lexicon's path as spelled, so + # both sides have to resolve before deciding what the reference supplies. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "../pkg/Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: + if not _case_sensitive(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # An empty href names the folder itself; folding it would search the parent. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") + (tmp_path / "PKG").write_bytes(b"") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [