From 80193cccb07adf384570d49bed3cdc4016392cea Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:37:00 -0400 Subject: [PATCH 01/11] Resolve companion ranges across filename case differences MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A LIFT folder written on Windows can spell its pair inconsistently — Dict.LIFT beside Dict.lift-ranges, or the reverse — and load fine there, because the filesystem folds case. On Linux the sibling candidate is built from the .lift's own suffix, so it missed, the companion was skipped without a word, and every range it defined went absent. Candidates that match no file exactly now fall back to one whose name differs only in case. The fallback is reached only after an exact miss, so a case-folding filesystem never enters it and behaves as before; a case-sensitive one gets one directory read per folder, cached across the candidate list. Where several names fold together the lexicographically first wins. The choice is arbitrary but fixed, which matters more than which file it picks: directory order varies between filesystems and runs, and a companion that loads differently on consecutive reads would be worse than one that never loads. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 10 ++++++++ src/sil_lift/_model.py | 48 ++++++++++++++++++++++++++++++++---- tests/test_ranges_folder.py | 49 +++++++++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08dcbe8..9e1b10e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,16 @@ releases may contain breaking changes. ## [Unreleased] +### Fixed + +- Companion `.lift-ranges` files now resolve when the folder's filenames + disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). + Such a folder loads on Windows and macOS, whose filesystems fold case, but + on Linux the companion was silently skipped and its ranges went missing. + A candidate that matches no file exactly now falls back to one whose name + differs only in case; where several fold together the lexicographically + first wins, so resolution is stable across runs. + ## [0.1.0] - 2026-07-TBD ### Added diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index e130f9f..3895046 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -452,6 +452,38 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: + """``candidate`` if it is a file, else one whose name differs only in case. + + LIFT folders are written on Windows, where the filesystem folds case, and + read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in + case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before + this fallback, silently did not on a case-sensitive filesystem. + + The fallback fires only where the exact name missed, so a case-folding + filesystem never reaches it and nothing changes there. Where several names + fold together, the lexicographically first wins — arbitrary, but stable + across runs, which "whatever the directory yields first" would not be. + ``listings`` caches one directory read per folder. + """ + try: + if candidate.is_file(): + return candidate + except OSError: + return None + folder = candidate.parent + if folder not in listings: + entries: dict[str, Path] = {} + try: + for path in sorted(folder.iterdir()): + if path.is_file(): + entries.setdefault(path.name.lower(), path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = entries + return listings[folder].get(candidate.name.lower()) + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -510,7 +542,10 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). + exporting machine, so the basename is what resolves locally). A + candidate no file matches exactly still resolves to one whose name + differs only in case, so a folder authored on Windows loads the same + way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -544,14 +579,17 @@ def _resolve_ranges(self) -> None: basename = range_.href.replace("\\", "/").rpartition("/")[2] if basename: candidates.append(base / basename) + listings: dict[Path, dict[str, Path]] = {} for candidate in candidates: + found = _existing_file(candidate, listings) + if found is None: + continue try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + if resolved not in self.ranges_files: + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..336a6bc 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -225,6 +225,55 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename casing. + + Named off the fixture stem so the header's ``range/@href`` basename + candidate finds nothing — only the sibling candidate can resolve these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _case_sensitive_fs(folder: Path) -> bool: + (folder / "CaseProbe").mkdir() + return not (folder / "caseprobe").exists() + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: + if not _case_sensitive_fs(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the + # tie-break picks one: lexicographically first, the same one every run. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not reach past a folder for a name that isn't in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + @pytest.mark.parametrize( ("href", "expected"), [ From 7d14ad7aa00a5dfea72d1b3a05eac2c8790ae7f1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 4 Aug 2026 16:43:47 -0400 Subject: [PATCH 02/11] Describe case-tolerant companion discovery under 0.1.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 0.1.0 has not shipped, so there is no released behavior for an Unreleased entry to be fixing — the tolerance is simply part of what companion discovery does in the first release. Fold it into that bullet. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9e1b10e..d4ed83a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,16 +18,6 @@ releases may contain breaking changes. ## [Unreleased] -### Fixed - -- Companion `.lift-ranges` files now resolve when the folder's filenames - disagree in case (`Dict.LIFT` beside `Dict.lift-ranges`, or the reverse). - Such a folder loads on Windows and macOS, whose filesystems fold case, but - on Linux the companion was silently skipped and its ranges went missing. - A candidate that matches no file exactly now falls back to one whose name - differs only in case; where several fold together the lexicographically - first wins, so resolution is stable across runs. - ## [0.1.0] - 2026-07-TBD ### Added @@ -58,13 +48,14 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, - `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, - build-from-scratch helpers `Lexicon.add_ranges_file()` / - `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and - header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + (`Lexicon.ranges_files`, resolving a companion whose filename differs from + the `.lift` only in case, as Windows-authored folders often do), `save()` + writes companions together, `all_ranges()` merged view, `media_refs()` / + `missing_media()` helpers, build-from-scratch helpers + `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / + `Range.add_element()` (`save()` writes and header-references a new companion + beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first + schema for standalone ranges documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other From a565d67a49514d11f46ba980bac87a9ea73842e1 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 12 Aug 2026 15:16:08 -0400 Subject: [PATCH 03/11] Keep "entry" for LIFT entries in the companion-case fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The directory listing the fallback builds called its files "entries", the word this module uses for a LIFT everywhere else — the same collision that keeps byte regions from being called spans. Name them files. Spell the surrounding prose the way the rest of the package does: a fallback that runs rather than fires, a name that matched no file rather than missed, a helper named for the filesystem it probes rather than abbreviating it, and fixture names deliberately not taken from the corpus file. Unpack the two densest clauses so each reads in one pass. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 22 +++++++++++----------- tests/test_ranges_folder.py | 10 +++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 3895046..2bde198 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -460,11 +460,11 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before this fallback, silently did not on a case-sensitive filesystem. - The fallback fires only where the exact name missed, so a case-folding - filesystem never reaches it and nothing changes there. Where several names - fold together, the lexicographically first wins — arbitrary, but stable - across runs, which "whatever the directory yields first" would not be. - ``listings`` caches one directory read per folder. + The fallback runs only where the exact name matched no file, so a + case-folding filesystem never reaches it and nothing changes there. Where + several names fold together, the lexicographically first wins — arbitrary, + but stable across runs, which "whatever the directory yields first" would + not be. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -473,14 +473,14 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa return None folder = candidate.parent if folder not in listings: - entries: dict[str, Path] = {} + files: dict[str, Path] = {} try: for path in sorted(folder.iterdir()): if path.is_file(): - entries.setdefault(path.name.lower(), path) + files.setdefault(path.name.lower(), path) except OSError: pass # unreadable folder: no candidate resolves out of it - listings[folder] = entries + listings[folder] = files return listings[folder].get(candidate.name.lower()) @@ -543,9 +543,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A - candidate no file matches exactly still resolves to one whose name - differs only in case, so a folder authored on Windows loads the same - way on a case-sensitive filesystem. + candidate that no file matches exactly still resolves to a file whose + name differs only in case, so a folder authored on Windows loads the + same way on a case-sensitive filesystem. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 336a6bc..4869bc2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -228,8 +228,8 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: """A loadable .lift plus companion under arbitrary filename casing. - Named off the fixture stem so the header's ``range/@href`` basename - candidate finds nothing — only the sibling candidate can resolve these. + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. """ folder.mkdir(parents=True, exist_ok=True) (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) @@ -237,7 +237,7 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name -def _case_sensitive_fs(folder: Path) -> bool: +def _case_sensitive_filesystem(folder: Path) -> bool: (folder / "CaseProbe").mkdir() return not (folder / "caseprobe").exists() @@ -255,7 +255,7 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: - if not _case_sensitive_fs(tmp_path): + if not _case_sensitive_filesystem(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the # tie-break picks one: lexicographically first, the same one every run. @@ -267,7 +267,7 @@ def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> Non def test_absent_companion_stays_absent(tmp_path: Path) -> None: - # The fallback must not reach past a folder for a name that isn't in it. + # The fallback must not look outside the folder for a name not in it. folder = tmp_path / "pkg" folder.mkdir() (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) From 8bf134f81227c0c360624150db4f98e016ca9b58 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:33 -0400 Subject: [PATCH 04/11] Resolve companions by folded name, and never load the .lift as one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Companion lookup folds names with casefold() over NFC rather than lower(), so a Turkish-cased or NFD-spelled name resolves the way it does on the filesystem that wrote it — FLEx mixes normalization forms within one export. Ties break in code point order on every platform; sorting Path objects left the choice to directory order on Windows, where PurePath ordering is itself case-folded. An unstattable exact spelling now falls through to the folded lookup instead of giving up. A candidate that folds onto the .lift itself is skipped: RangesFile.load rejects a root, so a header href naming the lexicon in another case took the whole load down. One that folds onto a companion already tracked is skipped too — Path.resolve() leaves case alone on macOS, so a single file reached under two spellings was loaded and tracked twice, and written twice by save(). The sibling candidate is built with with_name, which agrees with with_suffix on every name that has an extension and does not raise on a name without one. Nothing upstream requires the .lift extension: parse_document never inspects it. dangling-ranges-href decides existence with that same lookup, so a companion spelled in another case is no longer reported missing on a case-sensitive filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 85 ++++++++++++++++++++++++++++++------- src/sil_lift/_validate.py | 9 ++-- tests/test_ranges_folder.py | 66 ++++++++++++++++++++++++++-- 3 files changed, 138 insertions(+), 22 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 2bde198..b1e6155 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -452,36 +453,73 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a forgiving filesystem treats as one name. + + ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted + and dotless I), and NFC because normalization forms get mixed within a + single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — + and macOS folds them together where Linux does not. Neither NTFS's nor + APFS's own folding table is reproduced exactly; this is an approximation + over LIFT filenames, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in case. + """``candidate`` if it is a file, else one whose name differs only in spelling. LIFT folders are written on Windows, where the filesystem folds case, and read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. + this fallback, silently did not on a case-sensitive filesystem. See + :func:`_fold` for what counts as the same name. The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Where - several names fold together, the lexicographically first wins — arbitrary, - but stable across runs, which "whatever the directory yields first" would - not be. ``listings`` caches one directory read per folder. + case-folding filesystem never reaches it and nothing changes there. Only + the final path component is folded: a candidate under a *directory* spelled + in another case still does not resolve, which the hrefs this serves — bare + basenames, or relatives within the folder — do not need. Where several + names fold together, the first in code point order wins (so ``Dict.LIFT`` + ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, + which "whatever the directory yields first" would not be. ``listings`` + caches one directory read per folder. """ try: if candidate.is_file(): return candidate except OSError: - return None + pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent if folder not in listings: files: dict[str, Path] = {} try: - for path in sorted(folder.iterdir()): + # By name, not by Path: PurePath ordering is case-folded on Windows, + # which would leave the tie-break to directory order there. + for path in sorted(folder.iterdir(), key=lambda entry: entry.name): if path.is_file(): - files.setdefault(path.name.lower(), path) + files.setdefault(_fold(path.name), path) except OSError: pass # unreadable folder: no candidate resolves out of it listings[folder] = files - return listings[folder].get(candidate.name.lower()) + return listings[folder].get(_fold(candidate.name)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths differing only in spelling denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on + the latter one file reached under two spellings yields two distinct keys — + tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that + :func:`_fold` together are compared, since the inode check alone would + conflate distinct files on the filesystems that report ``st_ino`` as 0. + """ + if _fold(str(left)) != _fold(str(right)): + return False + try: + return left.samefile(right) + except OSError: + return False def _same_dir(left: Path, right: Path | None) -> bool: @@ -544,8 +582,9 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L hrefs are usually dangling absolute ``file://C:/...`` paths from the exporting machine, so the basename is what resolves locally). A candidate that no file matches exactly still resolves to a file whose - name differs only in case, so a folder authored on Windows loads the - same way on a case-sensitive filesystem. + name differs only in case or Unicode normalization, so a folder + authored on Windows loads the same way on a case-sensitive filesystem. + The ``.lift`` itself is never taken as its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -567,9 +606,16 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent + try: + own = self.path.resolve() + except OSError: + own = self.path candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) + # with_name, not with_suffix: they agree on every name that has an + # extension, but with_suffix rejects "-ranges" outright on a name + # without one, and nothing upstream requires the document to be named + # ``.lift`` — parse_document never looks at the extension. + candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: continue @@ -588,8 +634,15 @@ def _resolve_ranges(self) -> None: resolved = found.resolve() except OSError: continue - if resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(found) + # Skip a spelling of something already tracked (macOS keeps two + # keys for one file) and the .lift itself, which a header href + # naming it in another case now folds onto — RangesFile.load would + # reject its root and take the whole load down with it. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (own, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 39e0b25..9680eb9 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href from ._text import Multitext, Trait if TYPE_CHECKING: @@ -435,9 +435,12 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve # (they are resolved by basename when the companion is in the same folder) # and are not checked here; this catches an exporter that writes a relative - # href but not the file. + # href but not the file. Existence is the same notion load resolves + # companions by (_existing_file), so a companion spelled in another case is + # not reported missing on a case-sensitive filesystem. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, Path]] = {} for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -447,7 +450,7 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: resolved = all_ranges.get(range_.id) if resolved is not None and resolved.elements: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + if _existing_file(base / relative, listings) is None: yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 4869bc2..fc0d2b2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -226,7 +227,7 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: - """A loadable .lift plus companion under arbitrary filename casing. + """A loadable .lift plus companion under arbitrary filename spellings. Deliberately not named after the fixture, so the header's ``range/@href`` basename candidate finds nothing — only the sibling candidate resolves these. @@ -237,9 +238,22 @@ def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> return folder / lift_name +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + def _case_sensitive_filesystem(folder: Path) -> bool: - (folder / "CaseProbe").mkdir() - return not (folder / "caseprobe").exists() + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: @@ -274,6 +288,52 @@ def test_absent_companion_stays_absent(tmp_path: Path) -> None: assert sil_lift.load(folder / "Dict.lift").ranges_files == {} +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [ From 15109b5e38fc536af7089594aaef25db41d5c88f Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:23:53 -0400 Subject: [PATCH 05/11] Describe case- and normalization-tolerant companion discovery The 0.1.0 entry said companion names fold on case alone; they fold on Unicode normalization form as well. The folder guide listed the candidates tried but never mentioned the folding at all. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 15 ++++++++------- docs/en/guides/folder-media.md | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4ed83a..52d35bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,13 +49,14 @@ releases may contain breaking changes. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case, as Windows-authored folders often do), `save()` - writes companions together, `all_ranges()` merged view, `media_refs()` / - `missing_media()` helpers, build-from-scratch helpers - `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / - `Range.add_element()` (`save()` writes and header-references a new companion - beside the `.lift`); vendored `schemas/lift-ranges-0.13.rng` — the first - schema for standalone ranges documents. + the `.lift` only in case or Unicode normalization form, as Windows- and + FLEx-authored folders do), `save()` writes companions together, + `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, + build-from-scratch helpers `Lexicon.add_ranges_file()` / + `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and + header-references a new companion beside the `.lift`); vendored + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index 16497fb..ddbeb56 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: From 7cee80f9abece1abfb3263d6bdaaf96f2f7da181 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Tue, 18 Aug 2026 17:55:45 -0400 Subject: [PATCH 06/11] Trim the companion-folding prose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three helpers each stated a rule, defended it, then disclaimed it. What is left is the reasoning the code cannot show: casefold over lower, NFC, why only the final component folds, why code point order, and what the fold pre-check protects the inode comparison from. The folder guide drops the folding sentence outright — it describes behavior no reader acts on, in a paragraph otherwise about which candidate wins. The 0.1.0 entry keeps the fact and loses the justification, which now lives only in the docstrings. Co-Authored-By: Claude Opus 5 (1M context) --- CHANGELOG.md | 5 ++-- docs/en/guides/folder-media.md | 2 +- src/sil_lift/_model.py | 49 ++++++++++++++-------------------- 3 files changed, 23 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d35bb..4138872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,9 +48,8 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`, resolving a companion whose filename differs from - the `.lift` only in case or Unicode normalization form, as Windows- and - FLEx-authored folders do), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and diff --git a/docs/en/guides/folder-media.md b/docs/en/guides/folder-media.md index ddbeb56..16497fb 100644 --- a/docs/en/guides/folder-media.md +++ b/docs/en/guides/folder-media.md @@ -12,7 +12,7 @@ lex.all_ranges() # merged {id: Range} view lex.all_ranges()["grammatical-info"].elements ``` -Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem. +Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `.lift-ranges` sibling is picked up even when nothing references it. `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use: diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index b1e6155..0a7d42c 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -454,36 +454,28 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: - """A filename reduced to what a forgiving filesystem treats as one name. - - ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted - and dotless I), and NFC because normalization forms get mixed within a - single export — FLEx writes the ``.lift`` in NFC and its companion in NFD — - and macOS folds them together where Linux does not. Neither NTFS's nor - APFS's own folding table is reproduced exactly; this is an approximation - over LIFT filenames, not a general equivalence. + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC + because FLEx mixes normalization forms within one export. An approximation + of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name differs only in spelling. + """``candidate`` if it is a file, else one whose name folds onto it. LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in - case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before - this fallback, silently did not on a case-sensitive filesystem. See - :func:`_fold` for what counts as the same name. - - The fallback runs only where the exact name matched no file, so a - case-folding filesystem never reaches it and nothing changes there. Only - the final path component is folded: a candidate under a *directory* spelled - in another case still does not resolve, which the hrefs this serves — bare - basenames, or relatives within the folder — do not need. Where several - names fold together, the first in code point order wins (so ``Dict.LIFT`` - ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms, - which "whatever the directory yields first" would not be. ``listings`` - caches one directory read per folder. + read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there + and, before this fallback, silently did not on a case-sensitive filesystem. + + Only the final component folds — the hrefs this serves are basenames or + same-folder relatives — and only after the exact name misses, so a + case-folding filesystem never reaches this. Among names that fold together + the first in code point order wins: arbitrary, but stable across runs and + platforms, which directory order is not. ``listings`` caches one directory + read per folder. """ try: if candidate.is_file(): @@ -506,13 +498,12 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa def _same_file(left: Path, right: Path) -> bool: - """Whether two paths differing only in spelling denote one file. + """Whether two paths that fold together denote one file. - ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on - the latter one file reached under two spellings yields two distinct keys — - tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that - :func:`_fold` together are compared, since the inode check alone would - conflate distinct files on the filesystems that report ``st_ino`` as 0. + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode + comparison from conflating distinct files where ``st_ino`` is 0. """ if _fold(str(left)) != _fold(str(right)): return False From e6bd41a524b17bc239a00c9517cc7c5266cc32e7 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 14:11:36 -0400 Subject: [PATCH 07/11] Keep folder-shaped and self-referencing hrefs out of companion resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A candidate that exists as a directory now stops the lookup instead of falling through to the folded listing. An href of "" normalizes to the LIFT folder itself and one of "sub/" to a subfolder, and folding a folder's own name searches its *parent*: any file there spelled like the folder was returned as the companion, and RangesFile.load then rejected its root and failed the whole load. dangling-ranges-href also treats a match that is the .lift itself as no match. _resolve_ranges refuses to take the lexicon for its own companion, so a header href folding onto it resolves to nothing that supplies the range — the reference is as dangling as a missing file, and went unreported on a case-sensitive filesystem. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 4 ++++ src/sil_lift/_validate.py | 7 +++++-- tests/test_ranges_folder.py | 11 +++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 0a7d42c..36297c2 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -480,6 +480,10 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa try: if candidate.is_file(): return candidate + if candidate.is_dir(): + # An href of "" or "sub/" lands here; folding a folder's own name + # would search its parent and match anything spelled like it. + return None except OSError: pass # unstattable exact spelling: a case variant of it may still stat folder = candidate.parent diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 9680eb9..58a7feb 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError -from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href, _same_file from ._text import Multitext, Trait if TYPE_CHECKING: @@ -450,7 +450,10 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: resolved = all_ranges.get(range_.id) if resolved is not None and resolved.elements: continue # supplied by a sibling companion instead - if _existing_file(base / relative, listings) is None: + found = _existing_file(base / relative, listings) + # _resolve_ranges refuses the lexicon as its own companion, so an + # href folding onto it supplies nothing and dangles too. + if found is None or _same_file(found, lexicon.path): yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index fc0d2b2..9411afd 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -313,6 +313,17 @@ def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it # as one would raise on the root and take the whole load down. lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + +def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: + if not _case_sensitive_filesystem(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # An empty href names the folder itself; folding it would search the parent. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") + (tmp_path / "PKG").write_bytes(b"") assert sil_lift.load(lift).ranges_files == {} From 204df55b8aee9b48d5371acd88c6d1addd61ba2c Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 15:32:59 -0400 Subject: [PATCH 08/11] Resolve both sides before deciding two paths are one file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _same_file compared the spellings it was handed, so a caller had to canonicalize first to get a true answer. _resolve_ranges did; the dangling-ranges-href check did not, and an href reaching the .lift through a ".." segment or a symlink read there as some other file. The loader skipped that candidate as self-referential while validation counted it as a companion that exists, leaving the header range both unsupplied and unreported. Resolving inside _same_file makes the answer independent of how the caller spelled its arguments, and retires the loader's own pre-resolution — the duplicate that let the two drift apart. A path that will not resolve now compares false instead of falling back to the spelling as given; it would fail the samefile stat on the next line regardless. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 15 ++++++--------- tests/test_ranges_folder.py | 9 +++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 36297c2..ac3f6cd 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -506,12 +506,13 @@ def _same_file(left: Path, right: Path) -> bool: ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where one file reached under two spellings yields two keys — tracked twice, and - written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode - comparison from conflating distinct files where ``st_ino`` is 0. + written twice by :meth:`Lexicon.save`. Both sides resolve first, so an + href's ``..`` or a symlink compares alike; the fold pre-check then keeps + the inode comparison from conflating distinct files where ``st_ino`` is 0. """ - if _fold(str(left)) != _fold(str(right)): - return False try: + if _fold(str(left.resolve())) != _fold(str(right.resolve())): + return False return left.samefile(right) except OSError: return False @@ -601,10 +602,6 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent - try: - own = self.path.resolve() - except OSError: - own = self.path candidates: list[Path] = [] # with_name, not with_suffix: they agree on every name that has an # extension, but with_suffix rejects "-ranges" outright on a name @@ -634,7 +631,7 @@ def _resolve_ranges(self) -> None: # naming it in another case now folds onto — RangesFile.load would # reject its root and take the whole load down with it. if resolved in self.ranges_files or any( - _same_file(resolved, other) for other in (own, *self.ranges_files) + _same_file(resolved, other) for other in (self.path, *self.ranges_files) ): continue self.ranges_files[resolved] = RangesFile.load(found) diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9411afd..3669b07 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -318,6 +318,15 @@ def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] +def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> None: + # The ".." keeps the href from matching the lexicon's path as spelled, so + # both sides have to resolve before deciding what the reference supplies. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "../pkg/Dict.lift") + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files == {} + assert "dangling-ranges-href" in [p.code for p in lexicon.iter_problems()] + + def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: if not _case_sensitive_filesystem(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") From 9735644f9026a76ffe625b6089dffc751b496ed6 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Wed, 19 Aug 2026 16:28:29 -0400 Subject: [PATCH 09/11] Say what the folding rules are, not how they got there MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fold cited FLEx's mixed NFC/NFD content as its reason to normalize filenames. That is a different code path and no evidence for the names; decomposed filenames arrive from macOS, so say that instead. _existing_file described itself against the behavior it replaced, which reads oddly once nothing remembers that behavior. Lexicon.load's candidate list and its matching rules split into separate paragraphs, and "every one that exists is loaded" becomes "every distinct file among them" — candidates resolving to the lexicon, to a directory, or to a file already tracked all exist and are deliberately skipped. The dangling-ranges-href comment leads with what the check catches rather than closing with it, and loses a restatement of _existing_file's own docstring. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 29 ++++++++++++++--------------- src/sil_lift/_validate.py | 11 ++++------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index ac3f6cd..7a3adc0 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -456,9 +456,9 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: """A filename reduced to what a case-folding filesystem treats as one name. - ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC - because FLEx mixes normalization forms within one export. An approximation - of NTFS's and APFS's tables, not a general equivalence. + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC for + names that arrive decomposed, as ones written or zipped on macOS do. An + approximation of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() @@ -467,15 +467,14 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa """``candidate`` if it is a file, else one whose name folds onto it. LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there - and, before this fallback, silently did not on a case-sensitive filesystem. + read everywhere. ``Dict.LIFT`` beside ``Dict.lift-ranges`` is a pair there + and nowhere else. Only the final component folds — the hrefs this serves are basenames or same-folder relatives — and only after the exact name misses, so a case-folding filesystem never reaches this. Among names that fold together - the first in code point order wins: arbitrary, but stable across runs and - platforms, which directory order is not. ``listings`` caches one directory - read per folder. + the first in code point order wins: arbitrary, but stable, which directory + order is not. ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -571,16 +570,16 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L With ``resolve_ranges`` (the default), companion ``.lift-ranges`` files are loaded and tracked in :attr:`ranges_files`. Several - candidates are tried and every one that exists is loaded: the - conventional ``.lift-ranges`` sibling, and for each header + candidates are tried and every distinct file among them is loaded: + the conventional ``.lift-ranges`` sibling, and for each header ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). A - candidate that no file matches exactly still resolves to a file whose - name differs only in case or Unicode normalization, so a folder - authored on Windows loads the same way on a case-sensitive filesystem. - The ``.lift`` itself is never taken as its own companion. + exporting machine, so the basename is what resolves locally). + + A candidate matching no file exactly resolves across differences in + case or Unicode normalization, so a folder authored on Windows loads + the same way everywhere; the ``.lift`` is never its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 58a7feb..ee9d468 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -431,13 +431,10 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: file=file, ) - # Header references (relative) that resolve to no companion. - # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve - # (they are resolved by basename when the companion is in the same folder) - # and are not checked here; this catches an exporter that writes a relative - # href but not the file. Existence is the same notion load resolves - # companions by (_existing_file), so a companion spelled in another case is - # not reported missing on a case-sensitive filesystem. + # Header references that resolve to no companion — an exporter + # that wrote the href but not the file. Absolute and file:// hrefs are + # skipped: FLEx writes those knowing they will not resolve, and load reaches + # their companions by basename in the same folder instead. if lexicon.path is not None: base = lexicon.path.parent listings: dict[Path, dict[str, Path]] = {} From 98d1fe12377f1aeebfa741ff54c22f7690fab3ed Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 20 Aug 2026 08:18:45 -0400 Subject: [PATCH 10/11] Rework the companion-folding prose, and name a predicate like one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docstrings and comments only, plus one rename. _existing_file claimed a case-folding filesystem never reaches the fallback. That is false on NTFS, which folds case but not normalization: an NFD companion misses the exact stat and only the fallback finds it. The guarantee that does hold everywhere — an exact hit is returned unchanged — leads instead. Its summary named the argument rather than the return value, and its motivation read as though LIFT folders can only be written on Windows, when what matters is whether the authoring filesystem folds case, as macOS also does. The with_name comment justified a choice against with_suffix rather than warning about it. Naming the hazard is what stops someone reaching for the tidier call and reintroducing the raise it avoids. _case_sensitive_filesystem returned a bool under a noun phrase that promises a filesystem; _same_file and _same_dir are the house pattern for a predicate. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 48 ++++++++++++++++++++----------------- tests/test_ranges_folder.py | 6 ++--- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 7a3adc0..22fa836 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -456,25 +456,28 @@ def _normalize_href(href: str) -> Path | None: def _fold(text: str) -> str: """A filename reduced to what a case-folding filesystem treats as one name. - ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC for - names that arrive decomposed, as ones written or zipped on macOS do. An + ``casefold`` for what ``lower`` gets wrong (e.g. the Turkish dotless i); + NFC for names that arrive decomposed (e.g. ones zipped on macOS). An approximation of NTFS's and APFS's tables, not a general equivalence. """ return unicodedata.normalize("NFC", text).casefold() def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: - """``candidate`` if it is a file, else one whose name folds onto it. + """The file ``candidate`` names, matched exactly or by folded name. - LIFT folders are written on Windows, where the filesystem folds case, and - read everywhere. ``Dict.LIFT`` beside ``Dict.lift-ranges`` is a pair there - and nowhere else. + Where the authoring filesystem folds case, as on Windows and macOS, an + inconsistently spelled pair goes unnoticed: ``Dict.LIFT`` beside + ``Dict.lift-ranges`` is a pair there but not everywhere. - Only the final component folds — the hrefs this serves are basenames or - same-folder relatives — and only after the exact name misses, so a - case-folding filesystem never reaches this. Among names that fold together - the first in code point order wins: arbitrary, but stable, which directory - order is not. ``listings`` caches one directory read per folder. + An exact hit is always returned unchanged: folding runs only after the + exact name misses, and then only on the final component — the hrefs this + serves are basenames or same-folder relatives. + + Among names that fold together the first in code point order wins: + arbitrary, but stable, which directory order is not. + + ``listings`` caches one directory read per folder. """ try: if candidate.is_file(): @@ -505,9 +508,11 @@ def _same_file(left: Path, right: Path) -> bool: ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where one file reached under two spellings yields two keys — tracked twice, and - written twice by :meth:`Lexicon.save`. Both sides resolve first, so an - href's ``..`` or a symlink compares alike; the fold pre-check then keeps - the inode comparison from conflating distinct files where ``st_ino`` is 0. + written twice by :meth:`Lexicon.save`. + + Both sides resolve first, so ``..`` segments and symlinks compare alike; + the fold pre-check then keeps the inode comparison from conflating + distinct files where ``st_ino`` is 0. """ try: if _fold(str(left.resolve())) != _fold(str(right.resolve())): @@ -602,10 +607,9 @@ def _resolve_ranges(self) -> None: return base = self.path.parent candidates: list[Path] = [] - # with_name, not with_suffix: they agree on every name that has an - # extension, but with_suffix rejects "-ranges" outright on a name - # without one, and nothing upstream requires the document to be named - # ``.lift`` — parse_document never looks at the extension. + # with_name and with_suffix agree on every name that has an extension, + # but with_suffix would raise on a name that has none — which + # parse_document accepts, since it never inspects the extension. candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: @@ -625,10 +629,10 @@ def _resolve_ranges(self) -> None: resolved = found.resolve() except OSError: continue - # Skip a spelling of something already tracked (macOS keeps two - # keys for one file) and the .lift itself, which a header href - # naming it in another case now folds onto — RangesFile.load would - # reject its root and take the whole load down with it. + # A header href naming the .lift in another case folds onto it, and + # RangesFile.load rejects that root, failing the whole load; two + # spellings of one companion, which resolve() leaves distinct on + # macOS, would load and write it twice. if resolved in self.ranges_files or any( _same_file(resolved, other) for other in (self.path, *self.ranges_files) ): diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 3669b07..dde9835 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -248,7 +248,7 @@ def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: return folder / lift_name -def _case_sensitive_filesystem(folder: Path) -> bool: +def _case_sensitive(folder: Path) -> bool: probe = folder / "CaseProbe" probe.mkdir(exist_ok=True) sensitive = not (folder / "caseprobe").exists() @@ -269,7 +269,7 @@ def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) - def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: - if not _case_sensitive_filesystem(tmp_path): + if not _case_sensitive(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the # tie-break picks one: lexicographically first, the same one every run. @@ -328,7 +328,7 @@ def test_self_referencing_href_dangles_however_it_is_spelled(tmp_path: Path) -> def test_folder_shaped_href_stays_inside_the_folder(tmp_path: Path) -> None: - if not _case_sensitive_filesystem(tmp_path): + if not _case_sensitive(tmp_path): pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") # An empty href names the folder itself; folding it would search the parent. lift = _write_lift_with_href(tmp_path / "pkg", "Dict.lift", "") From 0f6fdcefe97927d305ae836582128658f641ccd8 Mon Sep 17 00:00:00 2001 From: Danny Rorabaugh Date: Thu, 20 Aug 2026 08:56:19 -0400 Subject: [PATCH 11/11] Call the tie-break deterministic rather than stable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Stable" names a specific property in sorting — preserving the relative order of equal elements — which is close enough to what is meant here to be read as a claim about the sort rather than about the outcome. The choice among fold-equal names is deterministic: same folder, same winner, on every platform and every run. Co-Authored-By: Claude Opus 5 (1M context) --- src/sil_lift/_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index 22fa836..8c197bd 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -475,7 +475,7 @@ def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Pa serves are basenames or same-folder relatives. Among names that fold together the first in code point order wins: - arbitrary, but stable, which directory order is not. + arbitrary, but deterministic, which directory order is not. ``listings`` caches one directory read per folder. """