From 721f62492df30d113fe6cfbc4e071598f64a32a2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 01:45:55 -0700 Subject: [PATCH 01/21] Make Role a StrEnum so members compare as their field names Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/_policy.py | 4 ++++ nameparser/_types.py | 9 +++++---- tests/v2/test_policy.py | 7 +++++++ tests/v2/test_types.py | 12 ++++++++++++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index c1124d0d..8ffc3b13 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -31,6 +31,7 @@ Release Log - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from + - Change ``Role`` to a ``StrEnum``: members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind`` - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 0be5db84..c8990d28 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -225,6 +225,10 @@ class Policy: def __post_init__(self) -> None: _reject_bare_string_order(self.name_order) order = tuple(_require_iterable(self.name_order, "name_order")) + # Sole rejection point for plain-string tuples: Role is a StrEnum, + # so the named-order membership check below compares EQUAL for + # ("given", "middle", "family") -- do not remove this loop as + # redundant. for element in order: if not isinstance(element, Role): raise TypeError( diff --git a/nameparser/_types.py b/nameparser/_types.py index d0c16648..a154b496 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -24,12 +24,13 @@ from nameparser._parser import Parser -class Role(Enum): +class Role(StrEnum): """The seven fields of a parsed name, one per :class:`Token`. Declaration order is the canonical field order everywhere - (``as_dict()``, ``comparison_key()``, rendering). Pass a member to - :meth:`ParsedName.tokens_for`; a member's ``.value`` is the field's - string name (``Role.GIVEN.value == "given"``).""" + (``as_dict()``, ``comparison_key()``, rendering). A StrEnum, like + :class:`AmbiguityKind`: members ARE their string values, so + ``token.role == "given"`` compares directly and + ``str(Role.GIVEN) == "given"``.""" # Declaration order IS the canonical field order (conventions §3): # every listing of the seven fields anywhere derives from this. diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index e088f5df..6f059e6a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -52,6 +52,13 @@ def test_name_order_rejects_non_role_elements_with_type_error() -> None: Policy(name_order=(1, 2, 3)) # type: ignore[arg-type] +def test_name_order_rejects_plain_string_tuples() -> None: + # Role is a StrEnum, so ("given", ...) == GIVEN_FIRST -- the + # isinstance loop is the only guard rejecting string elements. + with pytest.raises(TypeError, match="must be Role members"): + Policy(name_order=("given", "middle", "family")) # type: ignore[arg-type] + + def test_patronymic_rules_coerce_and_reject() -> None: p = Policy(patronymic_rules=frozenset({"east-slavic"})) # type: ignore[arg-type] assert p.patronymic_rules == frozenset({PatronymicRule.EAST_SLAVIC}) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 7df3c4d9..08a0e8e6 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -13,6 +13,18 @@ def test_role_declaration_order_is_canonical_field_order() -> None: ] +def test_role_members_are_their_string_values() -> None: + # Role is a StrEnum, like AmbiguityKind: members compare, hash, + # and stringify as their field names. + assert Role.GIVEN == "given" + assert str(Role.FAMILY) == "family" + d: dict[str, int] = {Role.GIVEN: 1} + assert d["given"] == 1 + assert isinstance(Role.GIVEN, str) + assert repr(Role.GIVEN) == "" # repr keeps .name form + assert Role("maiden") is Role.MAIDEN + + def test_token_construction_and_span_coercion() -> None: t = Token("Juan", (0, 4), Role.GIVEN) # type: ignore[arg-type] assert t.span == Span(0, 4) From 9aae04f427b8bea00966449503e2b591d3b904b7 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 02:01:47 -0700 Subject: [PATCH 02/21] Coerce tokens_for() roles instead of silently matching nothing Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + docs/usage.rst | 2 ++ nameparser/_types.py | 6 +++++- tests/v2/test_types.py | 17 +++++++++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 8ffc3b13..e3de6343 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -32,6 +32,7 @@ Release Log - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from - Change ``Role`` to a ``StrEnum``: members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind`` + - Change ``ParsedName.tokens_for()`` to accept a role's string name and to raise ``ValueError`` for an unknown role instead of silently returning no tokens - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/docs/usage.rst b/docs/usage.rst index 4faca1c8..d02ccbe1 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -385,6 +385,8 @@ so you can always get back to the text a field came from: >>> [t.text for t in name.tokens_for(Role.FAMILY)] ['de', 'la', 'Vega'] +A role's string name works too: ``name.tokens_for("family")``. + Correcting a parse -------------------- diff --git a/nameparser/_types.py b/nameparser/_types.py index a154b496..86ba3232 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -493,7 +493,11 @@ def given_names(self) -> str: # -- structured access ---------------------------------------------- - def tokens_for(self, role: Role) -> tuple[Token, ...]: + def tokens_for(self, role: Role | str) -> tuple[Token, ...]: + """The tokens of one field, in document order. Takes a Role + member or its string value; anything else raises ValueError + naming the valid roles.""" + role = _coerce_enum(role, Role, "Role", "roles") return tuple(t for t in self.tokens if t.role is role) def as_dict(self, include_empty: bool = True) -> dict[str, str]: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 08a0e8e6..9e8d24ea 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -2,6 +2,7 @@ import pytest +from nameparser import parse from nameparser._types import ( STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token, ) @@ -398,3 +399,19 @@ def test_setstate_rejects_layout_skew_on_frozen_types() -> None: bad_pn["zq_future"] = () with pytest.raises(ValueError, match="zq_future"): ParsedName.__new__(ParsedName).__setstate__(bad_pn) + + +def test_tokens_for_accepts_role_string() -> None: + name = parse("Juan de la Vega") + assert name.tokens_for("family") == name.tokens_for(Role.FAMILY) + assert name.tokens_for("given") == name.tokens_for(Role.GIVEN) + + +def test_tokens_for_unknown_role_raises_listing_roles() -> None: + with pytest.raises(ValueError, match=r"unknown Role 'last'.*valid roles: title, given, middle"): + parse("John Smith").tokens_for("last") + + +def test_tokens_for_non_coercible_raises() -> None: + with pytest.raises(ValueError, match="unknown Role 3"): + parse("John Smith").tokens_for(3) # type: ignore[arg-type] From 42142116a08bf67912ba072fff3f389e5b529a51 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 02:19:16 -0700 Subject: [PATCH 03/21] Give PolicyPatch the bounded deviation-only repr its siblings have Co-Authored-By: Claude Fable 5 --- nameparser/_policy.py | 51 ++++++++++++++++++++++++++++++++---------- tests/v2/test_reprs.py | 38 +++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/nameparser/_policy.py b/nameparser/_policy.py index c8990d28..1aabcf5a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -47,6 +47,29 @@ class PatronymicRule(StrEnum): _NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) +_ORDER_CONSTANT_NAMES: dict[tuple[Role, ...], str] = { + GIVEN_FIRST: "GIVEN_FIRST", + FAMILY_FIRST: "FAMILY_FIRST", + FAMILY_FIRST_GIVEN_LAST: "FAMILY_FIRST_GIVEN_LAST", +} + + +def _order_repr(value: tuple[Role, ...]) -> str: + # Unreachable via Policy's constructor (its __post_init__ restricts + # name_order to the three named orders) but REACHABLE via + # PolicyPatch, which defers name_order validation to apply time by + # design -- value may hold non-Role, even unhashable, elements. + # repr must never raise, so take the named-lookup path only once + # every element is confirmed a Role. (The annotation states the + # Policy-side truth; the PolicyPatch call site passes getattr-Any.) + if all(isinstance(r, Role) for r in value): + named = _ORDER_CONSTANT_NAMES.get(value) + if named is not None: + return named + return "(" + ", ".join(r.name for r in value) + ")" + return repr(value) + + # Single source for the migration hint raised by both Policy and # PolicyPatch when patronymic_rules gets a non-iterable (True is the # likeliest wrong value -- v1's flag was a bool that enabled BOTH rules). @@ -330,24 +353,13 @@ def __post_init__(self) -> None: def __repr__(self) -> str: # Bounded: only fields that deviate from the default are shown # (design rule, see nameparser._types module docstring). - constant_names = { - GIVEN_FIRST: "GIVEN_FIRST", - FAMILY_FIRST: "FAMILY_FIRST", - FAMILY_FIRST_GIVEN_LAST: "FAMILY_FIRST_GIVEN_LAST", - } parts = [] for f in dataclasses.fields(self): value = getattr(self, f.name) if value == f.default: continue if f.name == "name_order": - # __post_init__ restricts to the three named orders, so - # the fallback is unreachable via the constructor; kept - # because repr must never raise (e.g. a smuggled - # __setstate__ value -- layout is validated, values not). - order_repr = constant_names.get( - value, "(" + ", ".join(r.name for r in value) + ")") - parts.append(f"name_order={order_repr}") + parts.append(f"name_order={_order_repr(value)}") else: parts.append(f"{f.name}={value!r}") return f"Policy({', '.join(parts)})" @@ -436,6 +448,21 @@ def __post_init__(self) -> None: # class docstring ("Values are validated when the patch is # applied... not at patch construction"). + def __repr__(self) -> str: + # Bounded: only fields the patch actually sets are shown; UNSET + # fields are omitted (design rule, see nameparser._types module + # docstring -- the sibling of Policy's deviation-only repr). + parts = [] + for f in dataclasses.fields(self): + value = getattr(self, f.name) + if value is UNSET: + continue + if f.name == "name_order": + parts.append(f"name_order={_order_repr(value)}") + else: + parts.append(f"{f.name}={value!r}") + return f"PolicyPatch({', '.join(parts)})" + def apply_patch(policy: Policy, patch: PolicyPatch) -> Policy: """Fold a PolicyPatch onto a Policy. Policy.__post_init__ re-runs via diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 60b41ea0..5571701f 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -1,6 +1,8 @@ +import pytest from nameparser._lexicon import Lexicon from nameparser._locale import Locale +from nameparser._parser import Parser from nameparser._policy import FAMILY_FIRST, PatronymicRule, Policy, PolicyPatch from nameparser._types import Ambiguity, AmbiguityKind, ParsedName, Role, Span, Token @@ -66,3 +68,39 @@ def test_lexicon_repr_diffs_against_nearer_baseline() -> None: assert repr(Lexicon.empty()) == "Lexicon(empty)" lex = Lexicon.empty().add(titles={"zqx"}) assert repr(lex) == "Lexicon(empty + titles: +1)" + + +def test_policy_patch_repr_shows_only_set_fields() -> None: + assert repr(PolicyPatch()) == "PolicyPatch()" + assert repr(PolicyPatch(middle_as_family=True)) == \ + "PolicyPatch(middle_as_family=True)" + assert repr(PolicyPatch(name_order=FAMILY_FIRST)) == \ + "PolicyPatch(name_order=FAMILY_FIRST)" + + +@pytest.mark.parametrize("obj", [ + Policy(), + Policy(middle_as_family=True), + PolicyPatch(), + PolicyPatch(strip_emoji=False), + Locale("xx", Lexicon.empty()), + Locale("xx", Lexicon.empty(), PolicyPatch(middle_as_family=True)), + Parser(), +]) +def test_config_reprs_never_leak_the_unset_sentinel(obj: object) -> None: + # Guard the whole family: every config type's repr is bounded and + # deviation-only; the UNSET sentinel must never appear. + assert "UNSET" not in repr(obj) + assert "_Unset" not in repr(obj) + + +def test_policy_patch_repr_survives_unvalidated_name_order() -> None: + # PolicyPatch defers name_order validation to apply time, so a + # patch can hold a non-canonical or even garbage tuple -- its repr + # must describe it, never raise. + patch = PolicyPatch(name_order=(Role.GIVEN, Role.FAMILY, Role.MIDDLE)) + assert repr(patch) == "PolicyPatch(name_order=(GIVEN, FAMILY, MIDDLE))" + garbage = PolicyPatch(name_order=("given", "family", "middle")) # type: ignore[arg-type] + assert "given" in repr(garbage) # described, not crashed + unhashable = PolicyPatch(name_order=([1], "y", "z")) # type: ignore[arg-type] + repr(unhashable) # must not raise From 278adbd94910c8323c94ec29a2156418b07997f9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 02:38:27 -0700 Subject: [PATCH 04/21] Export STABLE_TAGS and write its docstring for users Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 11 +++++++++++ docs/release_log.rst | 1 + nameparser/__init__.py | 2 ++ nameparser/_types.py | 19 +++++++++++-------- tests/v2/test_types.py | 8 ++++++++ 5 files changed, 33 insertions(+), 8 deletions(-) diff --git a/docs/modules.rst b/docs/modules.rst index 8cec591a..88f44a0c 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -23,6 +23,17 @@ Results .. autoclass:: nameparser.Token :members: +.. py:data:: nameparser.STABLE_TAGS + :value: frozenset({"particle", "conjunction", "initial", "joined"}) + + The four :attr:`Token.tags ` values that are + stable API: ``particle`` (a family-name particle, "de"/"van"), + ``conjunction`` (a joining word, "and"/"y"), ``initial`` (an + initial-shaped word, "J."), and ``joined`` (a continuation of the + previous token within one merged piece, so the suffix view renders + "Ph. D." as one credential). Every other tag is namespaced + (``vocab:...``) and unstable — never match against those. + .. autoclass:: nameparser.Span :members: diff --git a/docs/release_log.rst b/docs/release_log.rst index e3de6343..420f6802 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -33,6 +33,7 @@ Release Log - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from - Change ``Role`` to a ``StrEnum``: members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind`` - Change ``ParsedName.tokens_for()`` to accept a role's string name and to raise ``ValueError`` for an unknown role instead of silently returning no tokens + - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/nameparser/__init__.py b/nameparser/__init__.py index 9e6e7011..621f1aec 100644 --- a/nameparser/__init__.py +++ b/nameparser/__init__.py @@ -20,6 +20,7 @@ PolicyPatch, ) from nameparser._types import ( + STABLE_TAGS, Ambiguity, AmbiguityKind, ParsedName, @@ -33,6 +34,7 @@ "HumanName", # v2 core "Span", "Role", "Token", "Ambiguity", "AmbiguityKind", "ParsedName", + "STABLE_TAGS", "Lexicon", "Policy", "PolicyPatch", "PatronymicRule", "UNSET", "GIVEN_FIRST", "FAMILY_FIRST", "FAMILY_FIRST_GIVEN_LAST", "DEFAULT_NICKNAME_DELIMITERS", "Locale", diff --git a/nameparser/_types.py b/nameparser/_types.py index 86ba3232..747b20c7 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -75,10 +75,13 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] ) -#: Stable, documented tag vocabulary (API). All other tags are -#: namespaced ("vocab:...", "patronymic:...") and unstable. "joined" -#: marks a continuation token of a merged piece (the ph-d merge) and -#: drives the suffix view's space-vs-comma join. +#: The four :attr:`Token.tags` values that are stable API. "particle" +#: marks a family-name particle ("de", "van"); "conjunction" a joining +#: word ("and", "y"); "initial" an initial-shaped word ("J.", "Q"); +#: "joined" a continuation of the previous token within one merged +#: piece ("Ph." + "D."), which the suffix view joins with a space +#: instead of ", ". Every other tag is namespaced ("vocab:...") and is +#: unstable debugging provenance -- never match against those. STABLE_TAGS = frozenset({"particle", "conjunction", "initial", "joined"}) #: The one sanctioned view-reorder marker (namespaced = unstable API). @@ -155,10 +158,10 @@ class Token: span: Span | None #: The field this token belongs to. role: Role - #: Classification labels. Exactly the four in STABLE_TAGS - #: ("particle", "conjunction", "initial", "joined") are API; - #: namespaced tags like "vocab:..." are unstable debugging - #: provenance -- never match against them. + #: Classification labels. Exactly the four members of + #: :data:`~nameparser.STABLE_TAGS` ("particle", "conjunction", + #: "initial", "joined") are API; namespaced tags like "vocab:..." + #: are unstable debugging provenance -- never match against them. tags: frozenset[str] = frozenset() # in the class body so @dataclass(slots=True) keeps them diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 9e8d24ea..76700d51 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -415,3 +415,11 @@ def test_tokens_for_unknown_role_raises_listing_roles() -> None: def test_tokens_for_non_coercible_raises() -> None: with pytest.raises(ValueError, match="unknown Role 3"): parse("John Smith").tokens_for(3) # type: ignore[arg-type] + + +def test_stable_tags_is_public_api() -> None: + import nameparser + assert "STABLE_TAGS" in nameparser.__all__ + # the documented stable tag vocabulary -- these four values are API + assert nameparser.STABLE_TAGS == frozenset( + {"particle", "conjunction", "initial", "joined"}) From c12c79580a84a1f3d9057e342685af8b1e1880ef Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 02:45:49 -0700 Subject: [PATCH 05/21] Make ParsedName.as_dict's include_empty keyword-only Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/_types.py | 2 +- tests/v2/test_types.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 420f6802..a91fd2c9 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -34,6 +34,7 @@ Release Log - Change ``Role`` to a ``StrEnum``: members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind`` - Change ``ParsedName.tokens_for()`` to accept a role's string name and to raise ``ValueError`` for an unknown role instead of silently returning no tokens - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) + - Change ``ParsedName.as_dict()`` to take ``include_empty`` as keyword-only (``HumanName.as_dict()`` is unchanged) - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/nameparser/_types.py b/nameparser/_types.py index 747b20c7..b5349847 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -503,7 +503,7 @@ def tokens_for(self, role: Role | str) -> tuple[Token, ...]: role = _coerce_enum(role, Role, "Role", "roles") return tuple(t for t in self.tokens if t.role is role) - def as_dict(self, include_empty: bool = True) -> dict[str, str]: + def as_dict(self, *, include_empty: bool = True) -> dict[str, str]: # _text_for handles the suffix ", "-join (single-role SUFFIX call) d = {role.value: self._text_for(role) for role in Role} if not include_empty: diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 76700d51..71eaeb7d 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -423,3 +423,8 @@ def test_stable_tags_is_public_api() -> None: # the documented stable tag vocabulary -- these four values are API assert nameparser.STABLE_TAGS == frozenset( {"particle", "conjunction", "initial", "joined"}) + + +def test_as_dict_include_empty_is_keyword_only() -> None: + with pytest.raises(TypeError): + parse("John Smith").as_dict(False) # type: ignore[misc] From 4b80a101232cd943a7ef2a8d39f7a482dd35f747 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 02:51:55 -0700 Subject: [PATCH 06/21] Add Policy.patched() so a PolicyPatch is applicable without a Locale Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 5 +++++ docs/locales.rst | 3 ++- docs/release_log.rst | 1 + nameparser/_policy.py | 15 +++++++++++++++ tests/v2/test_policy.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/customize.rst b/docs/customize.rst index 70d0da4a..83d1fb6a 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -260,6 +260,11 @@ listed below. - Excludes bidirectional control characters the same way. Defaults to ``True``. +To apply a :class:`PolicyPatch ` directly -- +without going through a locale pack -- call :meth:`Policy.patched() +`: +``Policy().patched(PolicyPatch(middle_as_family=True))``. + Family-first name order ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/locales.rst b/docs/locales.rst index e4897ef2..08152c51 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -149,7 +149,8 @@ You don't need to touch nameparser's registry to use your own pack — :class:`~nameparser.PolicyPatch` is a :class:`~nameparser.Policy`-shaped patch: every field defaults to :data:`~nameparser.UNSET` (leave it alone) instead of to a concrete value, so a pack only ever states what -it changes. +it changes. A patch can also be applied directly, without a pack — +see :meth:`Policy.patched() `. The ``policy`` half works that way, but the ``lexicon`` half does not. A pack's :class:`~nameparser.Lexicon` is a complete value in its own diff --git a/docs/release_log.rst b/docs/release_log.rst index a91fd2c9..f6fa6a94 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -35,6 +35,7 @@ Release Log - Change ``ParsedName.tokens_for()`` to accept a role's string name and to raise ``ValueError`` for an unknown role instead of silently returning no tokens - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) - Change ``ParsedName.as_dict()`` to take ``include_empty`` as keyword-only (``HumanName.as_dict()`` is unchanged) + - Add ``Policy.patched(patch)``, applying a ``PolicyPatch`` directly without wrapping it in a locale pack - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 1aabcf5a..6a358d26 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -364,6 +364,21 @@ def __repr__(self) -> str: parts.append(f"{f.name}={value!r}") return f"Policy({', '.join(parts)})" + # -- editing ------------------------------------------------------ + + def patched(self, patch: PolicyPatch) -> Policy: + """Fold a :class:`PolicyPatch` onto this Policy and return the + combined Policy. Set-valued fields union with the patch's; + scalar fields are overridden by the patch; UNSET fields are + left alone. Patch VALUES are validated here (Policy's + constructor re-runs on the result), not at patch construction + -- see PolicyPatch. The maiden-wins canonicalization applies + to the combined result exactly as if it had been constructed + directly.""" + if not isinstance(patch, PolicyPatch): + raise TypeError(f"patched() takes a PolicyPatch, got {patch!r}") + return apply_patch(self, patch) + class _Unset(Enum): UNSET = auto() diff --git a/tests/v2/test_policy.py b/tests/v2/test_policy.py index 6f059e6a..01a8554a 100644 --- a/tests/v2/test_policy.py +++ b/tests/v2/test_policy.py @@ -413,3 +413,36 @@ def test_every_field_rejects_a_buffer_with_a_decode_hint( # field the first pass missed. with pytest.raises(TypeError, match="decode"): cls(**{field: value}) + + +def test_patched_unions_set_valued_fields() -> None: + base = Policy(extra_suffix_delimiters={"|"}) # type: ignore[arg-type] + out = base.patched( + PolicyPatch(extra_suffix_delimiters={"/"})) # type: ignore[arg-type] + assert out.extra_suffix_delimiters == frozenset({"|", "/"}) + + +def test_patched_overrides_scalars_and_leaves_unset_alone() -> None: + base = Policy(middle_as_family=True, lenient_comma_suffixes=False) + out = base.patched(PolicyPatch(strip_emoji=False)) + assert out.strip_emoji is False + assert out.middle_as_family is True + assert out.lenient_comma_suffixes is False + + +def test_patched_with_empty_patch_returns_same_policy() -> None: + base = Policy(middle_as_family=True) + assert base.patched(PolicyPatch()) is base + + +def test_patched_validates_patch_values_at_apply_time() -> None: + # PolicyPatch scalars validate lazily by design; the error surfaces + # when the patch is applied. + patch = PolicyPatch(strip_emoji="off") # type: ignore[arg-type] + with pytest.raises(TypeError, match="strip_emoji must be a bool"): + Policy().patched(patch) + + +def test_patched_rejects_non_patch() -> None: + with pytest.raises(TypeError, match="takes a PolicyPatch"): + Policy().patched({"strip_emoji": False}) # type: ignore[arg-type] From 468561b11713b9da402d74b2ce6b94a66c649057 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 03:04:08 -0700 Subject: [PATCH 07/21] Add Parser.matches and Parser.capitalized for custom-config workflows Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/_parser.py | 25 +++++++++++++++++++++++++ nameparser/_types.py | 11 ++++++++--- tests/v2/test_parser.py | 41 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index f6fa6a94..5f1ebbf0 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -36,6 +36,7 @@ Release Log - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) - Change ``ParsedName.as_dict()`` to take ``include_empty`` as keyword-only (``HumanName.as_dict()`` is unchanged) - Add ``Policy.patched(patch)``, applying a ``PolicyPatch`` directly without wrapping it in a locale pack + - Add ``Parser.matches(a, b)`` and ``Parser.capitalized(name)``: the ``ParsedName`` methods of the same names fall back to the default configuration for str/omitted arguments, which is silently wrong for names parsed with a custom ``Parser`` - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 93b6545a..9ef399e6 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -75,6 +75,31 @@ def parse(self, text: str) -> ParsedName: policy=self.policy) return assemble(run(state)) + def matches(self, a: str | ParsedName, b: str | ParsedName) -> bool: + """Component-wise case-insensitive comparison of two names, + parsing str arguments with THIS parser. + :meth:`ParsedName.matches` parses its str argument with the + DEFAULT parser instead -- for names parsed with a custom + Parser, use this method.""" + if isinstance(a, str): + a = self.parse(a) + elif not isinstance(a, ParsedName): + raise TypeError(f"matches() takes str or ParsedName, got {a!r}") + if isinstance(b, str): + b = self.parse(b) + elif not isinstance(b, ParsedName): + raise TypeError(f"matches() takes str or ParsedName, got {b!r}") + return a.comparison_key() == b.comparison_key() + + def capitalized(self, name: ParsedName, *, + force: bool = False) -> ParsedName: + """:meth:`ParsedName.capitalized` under THIS parser's lexicon. + The no-argument form of that method uses the DEFAULT lexicon -- + for names parsed with a custom Parser, use this method.""" + if not isinstance(name, ParsedName): + raise TypeError(f"capitalized() takes a ParsedName, got {name!r}") + return name.capitalized(self.lexicon, force=force) + @functools.cache def _default_parser() -> Parser: diff --git a/nameparser/_types.py b/nameparser/_types.py index b5349847..ce9c34a5 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -565,7 +565,10 @@ def matches(self, other: str | ParsedName, *, parser: Parser | None = None) -> bool: """Component-wise case-insensitive comparison (the semantic layer; __eq__ stays strict). A str argument is parsed with - `parser`, or the default parser when None.""" + `parser`, or with the DEFAULT parser when None -- if this name + came from a custom Parser, pass that parser (or use + Parser.matches); otherwise the comparison silently runs under + the wrong configuration.""" if isinstance(other, str): import nameparser._parser as _parser active = parser if parser is not None else _parser._default_parser() @@ -608,7 +611,9 @@ def capitalized(self, lexicon: Lexicon | None = None, *, force: bool = False) -> ParsedName: """Case-fixing transform -> new ParsedName, same spans, new token texts. Needs a lexicon for capitalization_exceptions and - particle rules; None uses the default lexicon. force=False - preserves mixed-case input (v1 parity). Idempotent.""" + particle rules; None uses the DEFAULT lexicon -- if this name + came from a custom Parser, pass its lexicon or use + Parser.capitalized. force=False preserves mixed-case input + (v1 parity). Idempotent.""" import nameparser._render as _render return _render.capitalized(self, lexicon, force=force) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index da363ced..69676ecb 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -1,3 +1,4 @@ +import dataclasses import pickle import pytest @@ -315,3 +316,43 @@ def test_each_suffix_or_name_branch_describes_itself() -> None: numeral = parse("John Smith V").ambiguities[0].detail assert "periods" not in numeral assert "numeral" in numeral and "initial" in numeral + + +def test_parser_matches_uses_its_own_config() -> None: + p = Parser(lexicon=Lexicon.default().add(titles=["moff"])) + a = p.parse("Moff Tarkin") + # ParsedName.matches falls back to the DEFAULT parser for the str + # argument, which reads "Moff" as a given name -- mismatch. + assert not a.matches("Moff Tarkin") + # Parser.matches parses the str with the same config -- match. + assert p.matches(a, "Moff Tarkin") + assert p.matches("Moff Tarkin", "MOFF TARKIN") + + +def test_parser_matches_rejects_wrong_types() -> None: + p = Parser() + with pytest.raises(TypeError, match="takes str or ParsedName"): + p.matches(42, "John Smith") # type: ignore[arg-type] + with pytest.raises(TypeError, match="takes str or ParsedName"): + p.matches("John Smith", 42) # type: ignore[arg-type] + + +def test_parser_capitalized_uses_its_own_lexicon() -> None: + # pair-tuples, not a dict: the field is tuple-annotated and typed + # call sites pass canonical pairs (see _default_lexicon's note) + exceptions = dict( + Lexicon.default().capitalization_exceptions_map) | {"zqx": "ZqX"} + lex = dataclasses.replace( + Lexicon.default(), + capitalization_exceptions=tuple(sorted(exceptions.items()))) + p = Parser(lexicon=lex) + n = p.parse("john zqx") + assert p.capitalized(n).family == "ZqX" + # ParsedName.capitalized() with no argument uses the DEFAULT + # lexicon, which has no such exception. + assert n.capitalized().family == "Zqx" + + +def test_parser_capitalized_rejects_non_parsed_name() -> None: + with pytest.raises(TypeError, match="takes a ParsedName"): + Parser().capitalized("john smith") # type: ignore[arg-type] From 4a1dfbbe5433d9bc2fefa8ec96820c93c47aa3a0 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 03:11:00 -0700 Subject: [PATCH 08/21] Add Parser.revise, the tag-preserving field replacement Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + docs/usage.rst | 8 +++++ nameparser/_facade.py | 2 ++ nameparser/_parser.py | 33 +++++++++++++++++++- nameparser/_types.py | 60 +++++++++++++++++++++++------------ tests/v2/test_parser.py | 69 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 153 insertions(+), 20 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 5f1ebbf0..f7b4cc54 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -37,6 +37,7 @@ Release Log - Change ``ParsedName.as_dict()`` to take ``include_empty`` as keyword-only (``HumanName.as_dict()`` is unchanged) - Add ``Policy.patched(patch)``, applying a ``PolicyPatch`` directly without wrapping it in a locale pack - Add ``Parser.matches(a, b)`` and ``Parser.capitalized(name)``: the ``ParsedName`` methods of the same names fall back to the default configuration for str/omitted arguments, which is silently wrong for names parsed with a custom ``Parser`` + - Add ``Parser.revise(name, **fields)``: ``ParsedName.replace()`` with the replacement text classified by the parser's vocabulary, so particle/initial/suffix-join behavior survives the edit - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` diff --git a/docs/usage.rst b/docs/usage.rst index d02ccbe1..94b84d97 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -403,6 +403,14 @@ everything else carried over. >>> name.title '' +``replace()`` splits values on whitespace and its tokens carry no +vocabulary tags, so tag-driven views degrade (``family_particles`` +empties, particles regain their initials). When that matters, use +:meth:`Parser.revise() ` — the same +operation, with each value classified by the parser's vocabulary, +which also means delimiters and marker words in the value are consumed +as they would be in a parse. + Command line ------------ diff --git a/nameparser/_facade.py b/nameparser/_facade.py index bb7875c3..bc38d2c5 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -335,6 +335,8 @@ def _set_field(self, member: str, value: str | list[str] | None) -> None: else: raise TypeError( f"{member} must be a str, list, or None, got {value!r}") + # v1 setters stay on replace(): revise()'s vocabulary tags would + # change v1 parity self._parsed = self._parsed.replace( **{_V2_FIELD.get(member, member): joined}) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 9ef399e6..eef0090e 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -19,7 +19,10 @@ from nameparser._pipeline._assemble import assemble from nameparser._pipeline._state import ParseState from nameparser._policy import UNSET, Policy, PolicyPatch, apply_patch -from nameparser._types import ParsedName, _guarded_getstate, _guarded_setstate +from nameparser._types import ( + FOLDED_TAG, ParsedName, Token, _guarded_getstate, _guarded_setstate, + _validated_field_strings, +) @dataclass(frozen=True, slots=True) @@ -100,6 +103,34 @@ def capitalized(self, name: ParsedName, *, raise TypeError(f"capitalized() takes a ParsedName, got {name!r}") return name.capitalized(self.lexicon, force=force) + def revise(self, name: ParsedName, **fields: str) -> ParsedName: + """:meth:`ParsedName.replace` with this parser's vocabulary: + each value is tokenized and classified by a full sub-parse, so + the stable tags survive and the tag-driven views + (family_particles, initials(), the suffix join) behave as if + the text had been parsed. The value is classified ON ITS OWN, + though -- a word whose reading depends on surrounding context + may classify differently than it would in place (a standalone + "B. S." reads as initials, not a suffix run). The sub-parse's + role choices and ambiguities are discarded -- every harvested + token takes the named field's role -- and its structural + behavior applies: delimiter characters do not become tokens, + and a mid-value maiden marker is consumed as in parsing. + Tokens are synthetic (span=None); original is unchanged; a + value with no name content (empty, whitespace, or punctuation + only) clears the field; ambiguities referencing replaced + tokens are dropped.""" + if not isinstance(name, ParsedName): + raise TypeError(f"revise() takes a ParsedName, got {name!r}") + replaced = _validated_field_strings(fields) + harvested = { + role: tuple( + Token(t.text, None, role, t.tags - {FOLDED_TAG}) + for t in self.parse(value).tokens) + for role, value in replaced.items() + } + return name._with_field_tokens(harvested) + @functools.cache def _default_parser() -> Parser: diff --git a/nameparser/_types.py b/nameparser/_types.py index ce9c34a5..4d46afc2 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -321,12 +321,31 @@ def __repr__(self) -> str: return f"Ambiguity({self.kind.value!r}: {texts})" +def _validated_field_strings(fields: dict[str, str]) -> dict[Role, str]: + """Shared by ParsedName.replace and Parser.revise: validate a + **fields mapping of role names to replacement strings and key it + by Role. TypeErrors match replace()'s historical wording.""" + by_value = {role.value: role for role in Role} + for key, value in fields.items(): + if key not in by_value: + raise TypeError( + f"unknown field {key!r}; expected one of " + f"{', '.join(by_value)}" + ) + if not isinstance(value, str): + raise TypeError( + f"field {key!r} must be a str, got {value!r}" + ) + return {by_value[k]: v for k, v in fields.items()} + + @dataclass(frozen=True, slots=True) class ParsedName: """The immutable result of parsing one name string. Read the seven fields as strings (``.given``, ``.family``, ...); inspect structure through :attr:`tokens` / :meth:`tokens_for`; correct a parse with - :meth:`replace` (returns a new value); produce output with + :meth:`replace` (returns a new value; Parser.revise is the + tag-preserving form); produce output with :meth:`render`, :meth:`initials`, :meth:`capitalized`, or ``str()``. Constructor-enforced invariants: spans ascending, non-overlapping, @@ -517,35 +536,38 @@ def replace(self, **fields: str) -> ParsedName: synthetic tokens (span=None). Whitespace-splits each value; an empty value clears the field. original is unchanged (provenance). Ambiguities referencing replaced tokens are dropped. - """ - by_value = {role.value: role for role in Role} - for key, value in fields.items(): - if key not in by_value: - raise TypeError( - f"unknown field {key!r}; expected one of " - f"{', '.join(by_value)}" - ) - if not isinstance(value, str): - raise TypeError( - f"field {key!r} must be a str, got {value!r}" - ) - - def synthetic(value: str, role: Role) -> list[Token]: - return [Token(word, None, role) for word in value.split()] - replaced = {by_value[k]: v for k, v in fields.items()} + Replacement tokens carry NO tags, so tag-driven views degrade: + family_particles empties, particles regain their initials, and + a multi-word suffix is comma-joined. Parser.revise() is the + tag-preserving alternative. + """ + replaced = _validated_field_strings(fields) + synthetic = { + role: tuple(Token(word, None, role) for word in value.split()) + for role, value in replaced.items() + } + return self._with_field_tokens(synthetic) + + def _with_field_tokens( + self, replaced: Mapping[Role, tuple[Token, ...]], + ) -> ParsedName: + """Shared tail of replace()/Parser.revise(): splice each role's + replacement tokens in at the role's first position (appended in + canonical order when the role had no tokens); drop ambiguities + whose referents were replaced.""" new_tokens: list[Token] = [] emitted: set[Role] = set() for tok in self.tokens: if tok.role in replaced: if tok.role not in emitted: - new_tokens.extend(synthetic(replaced[tok.role], tok.role)) + new_tokens.extend(replaced[tok.role]) emitted.add(tok.role) continue new_tokens.append(tok) for role in Role: if role in replaced and role not in emitted: - new_tokens.extend(synthetic(replaced[role], role)) + new_tokens.extend(replaced[role]) kept = tuple( amb for amb in self.ambiguities if all(t in new_tokens for t in amb.tokens) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 69676ecb..81f53895 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -356,3 +356,72 @@ def test_parser_capitalized_uses_its_own_lexicon() -> None: def test_parser_capitalized_rejects_non_parsed_name() -> None: with pytest.raises(TypeError, match="takes a ParsedName"): Parser().capitalized("john smith") # type: ignore[arg-type] + + +def test_revise_preserves_particle_tags() -> None: + p = Parser() + n = p.parse("Juan de la Vega") + r = p.revise(n, family="de la Vega Smith") + assert r.family == "de la Vega Smith" + assert r.family_particles == "de la" + assert r.initials() == "J. V. S." # particles contribute no initial + + +def test_revise_keeps_multiword_suffix_one_credential() -> None: + p = Parser() + n = p.parse("John Smith Ph.D.") + r = p.revise(n, suffix="Ph. D.") + assert r.suffix == "Ph. D." # replace() would render "Ph., D." + + +def test_revise_views_match_a_fresh_parse() -> None: + p = Parser() + r = p.revise(p.parse("John Smith"), family="de la Vega") + f = p.parse("John de la Vega") + for view in ("given", "family", "family_particles", "family_base"): + assert getattr(r, view) == getattr(f, view) + assert r.initials() == f.initials() + + +def test_revise_replace_shared_semantics() -> None: + p = Parser() + n = p.parse("Dr. Juan de la Vega Jr.") + r = p.revise(n, given="José", suffix="") + assert r.given == "José" + assert r.suffix == "" # empty value clears the field + assert r.original == n.original # provenance unchanged + assert all(t.span is None for t in r.tokens_for("given")) + assert r.title == "Dr." # untouched fields keep spans + + +def test_revise_validation_matches_replace() -> None: + p = Parser() + n = p.parse("John Smith") + with pytest.raises(TypeError, match="takes a ParsedName"): + p.revise("John Smith", family="Doe") # type: ignore[arg-type] + with pytest.raises(TypeError, match="unknown field 'last'"): + p.revise(n, last="Doe") + with pytest.raises(TypeError, match="must be a str"): + p.revise(n, family=None) # type: ignore[arg-type] + + +def test_revise_strips_the_fold_marker() -> None: + # middle_as_family's fold tag must not survive the harvest: a + # carried tag would make the family view reorder the value. + p = Parser(policy=Policy(middle_as_family=True)) + r = p.revise(p.parse("Juan Perez"), family="Gabriel García Márquez") + assert r.family == "Gabriel García Márquez" + + +def test_revise_sub_parse_structural_behavior() -> None: + # the docstring's three structural promises, pinned: delimiters + # never become tokens, marker words are consumed as in parsing, + # and the sub-parse's ambiguities are discarded. + p = Parser() + n = p.parse("John Smith") + assert p.revise(n, family="Smith (Jones)").family == "Smith Jones" + revised = p.revise(n, family="Mary née Smith") + assert revised.family == "Mary Smith" + assert revised.maiden == "" + assert p.revise(n, given="J.R. 'Bob'").given == "J.R. Bob" + assert p.revise(n, family="Smith (Jones").ambiguities == () From a11d51e27c7ae7f55c865ec0f32cec584e51454e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 03:24:55 -0700 Subject: [PATCH 09/21] Repair the eight dead multi-word vocabulary entries Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 ++ nameparser/config/suffixes.py | 7 ------- nameparser/config/titles.py | 3 ++- tests/test_titles.py | 6 ++++++ 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index f7b4cc54..5f84eace 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -67,6 +67,8 @@ Release Log - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook. This now holds wherever the credential sits: 1.x healed the pair only when it trailed, so ``"Ph. D. John Smith"`` parsed as title ``Ph.`` / given ``D.`` with the real given name pushed to ``middle``; it now reads given ``John``, family ``Smith``, suffix ``Ph. D.`` + - Fix ``chargé d'affaires``: shipped as one unmatchable ``TITLES`` entry since it was added, it is now two chainable entries (``chargé``, ``d'affaires``), so the title is recognized + - Remove seven multi-word ``SUFFIX_ACRONYMS`` entries (``leed ap``, ``nicet i``–``nicet iv``, ``psm i``, ``psm ii``) that could never match in any release; splitting them would swallow real names ("John Leed", "Smith, A.P."), so they are dropped instead - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one diff --git a/nameparser/config/suffixes.py b/nameparser/config/suffixes.py index 05ae8f78..43d1f4b8 100644 --- a/nameparser/config/suffixes.py +++ b/nameparser/config/suffixes.py @@ -496,7 +496,6 @@ 'lcmt', 'lcpc', 'lcsw', - 'leed ap', 'lg', 'litk', 'litl', @@ -585,10 +584,6 @@ 'ncto', 'nd', 'ndtr', - 'nicet i', - 'nicet ii', - 'nicet iii', - 'nicet iv', 'nmd', 'np', 'np[18]', @@ -626,8 +621,6 @@ 'pp', 'pps', 'prm', - 'psm i', - 'psm ii', 'psm', 'psp', 'psyd', diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 35371299..1e260887 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -60,7 +60,8 @@ #: recognition titles like "Deputy Secretary of State". TITLES = FIRST_NAME_TITLES | { "attaché", - "chargé d'affaires", + "chargé", + "d'affaires", "king's", "marchioness", "marquess", diff --git a/tests/test_titles.py b/tests/test_titles.py index d01ae0bd..a73a7f6c 100644 --- a/tests/test_titles.py +++ b/tests/test_titles.py @@ -374,3 +374,9 @@ def test_leading_period_abbreviation_with_nickname(self) -> None: self.m(hn.title, "Xyz.", hn) self.m(hn.first, "Smith", hn) self.m(hn.nickname, "Bud", hn) + + def test_charge_daffaires_chains_as_title(self) -> None: + hn = HumanName("Chargé d'Affaires John Smith") + self.m(hn.title, "Chargé d'Affaires", hn) + self.m(hn.first, "John", hn) + self.m(hn.last, "Smith", hn) From 6597ad5f885e8332715ddb22fc30e8f335597462 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 03:34:31 -0700 Subject: [PATCH 10/21] Warn when a multi-word entry lands in a per-word Lexicon field Every Lexicon vocabulary field except given_name_titles is matched one word at a time, so a multi-word entry can never match -- the library itself shipped eight such dead entries for years (repaired in the previous commit). Per the inert-not-harmful convention a raise is forbidden (see the given_name_titles Gotcha in AGENTS.md); warn instead, from _normset (all per-word vocab fields) and _normpairs (capitalization_exceptions keys), through every entry path -- constructor, add(), union, unpickle re-validation, and the v1 shim's snapshot. Also fixes tests/v2/test_properties.py's shared _VOCAB hypothesis pool, which drew a multi-word phrase ("grand duke") into every Lexicon field even though its own comment says it's "legal only in given_name_titles" -- the fuzzer applied it everywhere, so it now tripped the new warning in every per-word field. Split it into a separate _TITLE_VOCAB used only for given_name_titles. Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 6 ++++ docs/release_log.rst | 1 + nameparser/_lexicon.py | 64 +++++++++++++++++++++++++++++++++---- tests/v2/test_lexicon.py | 63 +++++++++++++++++++++++++++++++++++- tests/v2/test_locales.py | 26 ++++++++++++++- tests/v2/test_properties.py | 20 +++++++++--- 6 files changed, 167 insertions(+), 13 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 83d1fb6a..628b6600 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -28,6 +28,12 @@ accepts a plain set of lowercase words, keyword by field name (``titles`` above; ``particles``, ``suffix_words``, and the rest work the same way) — see :doc:`modules` for the full field list. +Vocabulary entries are matched one word at a time (``given_name_titles`` +excepted), so a multi-word entry like ``titles={"grand moff"}`` can +never match; the constructor warns when it sees one +(``capitalization_exceptions`` keys included — they are looked up per +word too). + Removing works the same way, and drops the word from recognition: .. doctest:: diff --git a/docs/release_log.rst b/docs/release_log.rst index 5f84eace..b41d4c49 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -69,6 +69,7 @@ Release Log - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook. This now holds wherever the credential sits: 1.x healed the pair only when it trailed, so ``"Ph. D. John Smith"`` parsed as title ``Ph.`` / given ``D.`` with the real given name pushed to ``middle``; it now reads given ``John``, family ``Smith``, suffix ``Ph. D.`` - Fix ``chargé d'affaires``: shipped as one unmatchable ``TITLES`` entry since it was added, it is now two chainable entries (``chargé``, ``d'affaires``), so the title is recognized - Remove seven multi-word ``SUFFIX_ACRONYMS`` entries (``leed ap``, ``nicet i``–``nicet iv``, ``psm i``, ``psm ii``) that could never match in any release; splitting them would swallow real names ("John Leed", "Smith, A.P."), so they are dropped instead + - Add a ``UserWarning`` when a multi-word entry is stored in a per-word ``Lexicon`` field or as a ``capitalization_exceptions`` key -- such entries can never match - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy - Fold a leading never-given particle into the family name. Note that ``Lexicon.particles_ambiguous`` is the **complement** of v1's ``non_first_name_prefixes``, not a rename -- it lists the particles that *may* double as a given name, where v1 listed the ones that may not. Copying a v1 customization across without inverting it silently reverses the behavior; see :doc:`migrate` - Add ``ma`` and ``do`` to ``suffix_acronyms_ambiguous``, the set of post-nominals that are also ordinary surnames. An entry there is read as a credential when the name can spare it — written with periods (``"John Smith M.A."``), or when removing it still leaves a given *and* a family name (``"John Smith MA"`` → suffix ``MA``). With only two pieces to go around, the surname reading wins instead: ``"Jack Ma"`` and ``"Anh Do"`` keep their family names. As a side effect, a parenthesized or quoted ``"(MA)"``/``"(DO)"`` now falls through to nickname parsing rather than escaping to ``suffix``, since inside delimiters the nickname reading is the plausible one diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 1cd73668..dd8c84ad 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -9,9 +9,11 @@ import dataclasses import functools +import sys +import warnings from collections.abc import Iterable, Mapping from dataclasses import dataclass, field -from types import MappingProxyType +from types import FrameType, MappingProxyType from typing import cast #: Vocabulary set fields, in declaration order. add()/remove() operate @@ -112,7 +114,22 @@ def _reject_buffer(value: object, label: str, plural: str) -> None: ) -def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: +def _warn_dead_entry(message: str) -> None: + # A fixed stacklevel always lands on library internals: the call + # depth differs per entry point (Lexicon(), add(), unpickle, + # dataclasses.replace). Walk out of this module (and dataclasses' + # replace frames) so the warning points at the caller's own line. + level = 2 + frame: FrameType | None = sys._getframe(1) + while frame is not None and frame.f_globals.get("__name__") in ( + __name__, "dataclasses"): + frame, level = frame.f_back, level + 1 + warnings.warn(message, UserWarning, stacklevel=level) + + +def _normset( + entries: Iterable[str], field_name: str, warn: bool = True, +) -> frozenset[str]: # Reject a bare str before iterating: iterating "dr" would silently # yield the single characters {'d', 'r'} -- the set(str) footgun on # the primary customization surface. @@ -159,6 +176,24 @@ def _normset(entries: Iterable[str], field_name: str) -> frozenset[str]: f"Lexicon.{field_name} entry {w!r} normalizes to empty " f"(lowercase + strip periods/whitespace leaves nothing)" ) + # Every field but given_name_titles is matched one word at a + # time, so a multi-word entry can never match -- the library + # itself shipped eight such dead entries for years (repaired + # 2026-07-26). Warn, never raise: an inert entry produces + # nothing, and the given_name_titles precedent says a raise + # here costs working configurations (see __post_init__). + # warn=False is _edit()'s remove() path: a removal stores + # nothing, so warning there would name entries the caller is + # trying to get RID of, with "split it" advice that makes no + # sense for a no-op. add() still warns exactly once, via the + # new instance's __post_init__. + # interior whitespace test; split() covers all Unicode whitespace + if warn and field_name != "given_name_titles" \ + and n != "".join(n.split()): + _warn_dead_entry( + f"Lexicon.{field_name} entries are matched one word at " + f"a time; multi-word entry {w!r} can never match. " + f"Split it into separate entries") normalized.add(n) return frozenset(normalized) @@ -214,6 +249,14 @@ def _normpairs( f"empty (lowercase + strip periods/whitespace leaves " f"nothing)" ) + # capitalized() looks words up one at a time (the _WORD regex + # never yields spaces), so a multi-word key is unreachable. + # interior whitespace test; split() covers all Unicode whitespace + if normalized_key != "".join(normalized_key.split()): + _warn_dead_entry( + f"capitalization_exceptions keys are matched one word " + f"at a time; multi-word key {k!r} can never match. " + f"Split it into per-word entries") deduped[normalized_key] = v return tuple(sorted(deduped.items())) @@ -226,9 +269,12 @@ class Lexicon: :meth:`empty`, derive variants with :meth:`add` / :meth:`remove` / ``|`` (union), and pass the result to ``Parser(lexicon=...)``. Entries are normalized at construction -- lowercased, edge periods - stripped -- so matching is case-insensitive. Field docs below show - examples, not full contents; inspect any field's shipped vocabulary - directly, e.g. ``Lexicon.default().conjunctions``.""" + stripped -- so matching is case-insensitive. Vocabulary entries are + single words -- a multi-word entry warns at construction and can + never match (``given_name_titles``, matched as a space-joined run, + is the one exception). Field docs below show examples, not full + contents; inspect any field's shipped vocabulary directly, e.g. + ``Lexicon.default().conjunctions``.""" #: Pre-nominal titles ("dr", "sir", "capt", ...). Full default #: list: :data:`~nameparser.config.titles.TITLES`. @@ -479,7 +525,13 @@ def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: f"{', '.join(_VOCAB_FIELDS)}" ) current: frozenset[str] = getattr(self, name) - normalized = _normset(words, name) + # warn=False: this pass only computes the new set membership, + # never stores it directly. add() still warns exactly once, + # from the replaced instance's own __post_init__ -> _normset; + # remove() never reaches __post_init__ with the dead entry + # (it is subtracted out here), so it stays silent -- correct, + # since a removal stores nothing a warning could be about. + normalized = _normset(words, name, warn=False) updates[name] = (current | normalized if op == "add" else current - normalized) # mypy's dataclasses.replace() typing checks a **dict's single diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index a88b2f75..c894eaf9 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1,11 +1,14 @@ import dataclasses import pickle +import warnings from collections.abc import Callable import pytest from nameparser import Parser -from nameparser._lexicon import Lexicon, _normalize, _title_key +from nameparser._lexicon import ( + Lexicon, _VOCAB_FIELDS, _default_lexicon, _normalize, _title_key, +) def test_entries_are_normalized_at_construction() -> None: @@ -463,3 +466,61 @@ def test_every_lexicon_entry_point_rejects_a_buffer_with_a_decode_hint( # single most repeated defect on this branch. with pytest.raises(TypeError, match="decode"): build(value) + + +_PER_WORD_FIELDS = [f for f in _VOCAB_FIELDS if f != "given_name_titles"] + + +@pytest.mark.parametrize("field", _PER_WORD_FIELDS) +def test_multiword_entry_warns_per_field(field: str) -> None: + # Guard the whole family: every per-word field warns, not one. + with pytest.warns(UserWarning, match="matched one word at a time"): + if field in ("particles_ambiguous", "suffix_acronyms_ambiguous"): + # subset fields need their base populated to construct + base = {"particles_ambiguous": "particles", + "suffix_acronyms_ambiguous": "suffix_acronyms"}[field] + Lexicon.empty().add(**{base: ["zqx zqy"], field: ["zqx zqy"]}) + else: + Lexicon.empty().add(**{field: ["zqx zqy"]}) + + +def test_multiword_given_name_title_does_not_warn() -> None: + # given_name_titles matches space-joined runs -- multi-word entries + # are the point (see the __post_init__ note in _lexicon.py). + with warnings.catch_warnings(): + warnings.simplefilter("error") + Lexicon.empty().add(given_name_titles=["lt col"]) + + +def test_multiword_capitalization_key_warns() -> None: + with pytest.warns(UserWarning, match="matched one word at a time"): + dataclasses.replace( + Lexicon.empty(), + capitalization_exceptions=(("zqx zqy", "ZqXZqY"),)) + + +def test_default_lexicon_builds_warning_free() -> None: + # Would have caught the eight dead entries the previous commit + # repaired. + _default_lexicon.cache_clear() + try: + with warnings.catch_warnings(): + warnings.simplefilter("error") + Lexicon.default() + finally: + _default_lexicon.cache_clear() + + +def test_multiword_entry_warns_from_constructor_and_unpickle() -> None: + with pytest.warns(UserWarning, match="matched one word at a time"): + lex = Lexicon(titles=frozenset({"zqx zqy"})) + with pytest.warns(UserWarning, match="matched one word at a time"): + pickle.loads(pickle.dumps(lex)) + + +def test_remove_does_not_warn() -> None: + # remove() stores nothing; a dead-entry warning there would be a + # false positive with wrong advice. + with warnings.catch_warnings(): + warnings.simplefilter("error") + Lexicon.default().remove(titles=["zqx zqy"]) diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 16f1f961..344824b2 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -9,7 +9,7 @@ import pytest from nameparser import Locale, Parser, locales, parse, parser_for -from nameparser._lexicon import Lexicon +from nameparser._lexicon import _VOCAB_FIELDS, Lexicon from nameparser._policy import PatronymicRule, Policy from nameparser.locales import ru as _ru from nameparser.locales import tr_az as _tr_az @@ -67,6 +67,30 @@ def test_tr_az_pack_contents() -> None: assert locales.TR_AZ.lexicon == Lexicon.empty() +def test_pack_vocabulary_entries_are_single_words() -> None: + # The shipped-vocabulary guard (test_lexicon.py's + # test_default_lexicon_builds_warning_free), extended to the OTHER + # vocabulary source: a locale pack's lexicon fields are matched one + # word at a time same as Lexicon.default()'s, so a multi-word entry + # in any field but given_name_titles would be dead. Checked + # directly against field contents (not by capturing the + # construction-time warning) because a pack module is imported -- + # and its Locale built -- at most once per process; a later test in + # this file may already have forced the import, which would make a + # warning-capture version of this test pass even on a regression. + # Iterates every registered pack, not just ru/tr_az by name, so a + # future pack is covered automatically. + for code in locales.available(): + lexicon = locales.get(code).lexicon + for field in _VOCAB_FIELDS: + if field == "given_name_titles": + continue + for entry in getattr(lexicon, field): + assert entry == "".join(entry.split()), ( + f"{code}: {field} entry {entry!r} contains " + f"whitespace and can never match") + + def test_pack_marker_regexes_stay_in_sync_with_post_rules() -> None: # ru.py/tr_az.py hand-copy their DEVIATES marker regexes from # _pipeline._post_rules (layering forbids a pack importing the diff --git a/tests/v2/test_properties.py b/tests/v2/test_properties.py index 1c32c2fa..54440f26 100644 --- a/tests/v2/test_properties.py +++ b/tests/v2/test_properties.py @@ -182,15 +182,23 @@ def test_particle_fork_is_never_double_reported(text: str) -> None: # nobody tried. # Deliberately mixed: real vocabulary, one-letter words that collide -# with initials, an interior period, a multi-word phrase (legal only in -# given_name_titles), and non-Latin entries. Nothing here normalizes to -# empty, which Lexicon rejects outright. +# with initials, an interior period, and non-Latin entries. Nothing +# here normalizes to empty, which Lexicon rejects outright. No +# multi-word phrase: every field below except given_name_titles is +# matched one word at a time, so a multi-word draw in a per-word field +# would be a dead entry that trips Lexicon's multi-word warning. _VOCAB = st.sampled_from([ "van", "de", "la", "bin", "abdul", "abu", "dr", "sir", "prof", "md", "jr", "iii", "esq", "ma", "do", "and", "y", "née", "geb", - "a", "b", "ph.d", "grand duke", "عبد", "фон", "μεγα", + "a", "b", "ph.d", "عبد", "фон", "μεγα", ]) +# given_name_titles is the one field matched as a space-joined run, so +# a multi-word phrase there is meaningful (not dead) and must not warn. +# Derived from _VOCAB rather than duplicated, so the two pools cannot +# drift apart. +_TITLE_VOCAB = st.one_of(_VOCAB, st.just("grand duke")) + def _fix_invariants(**fields: frozenset[str]) -> dict[str, frozenset[str]]: """Repair a random draw into a legal Lexicon instead of generating @@ -219,7 +227,9 @@ def _fix_invariants(**fields: frozenset[str]) -> dict[str, frozenset[str]]: @st.composite def _lexicons(draw: st.DrawFn) -> Lexicon: - fields = {name: draw(st.frozensets(_VOCAB, max_size=5)) + fields = {name: draw(st.frozensets( + _TITLE_VOCAB if name == "given_name_titles" else _VOCAB, + max_size=5)) for name in _SET_FIELDS} caps = draw(st.lists(st.tuples(_VOCAB, _VOCAB), max_size=3)) return Lexicon(capitalization_exceptions=tuple(caps), From 90afe20dab5bc4b74870aaf81870a71606ad9296 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 03:54:10 -0700 Subject: [PATCH 11/21] Accept Role members as HumanName subscript keys Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 1 + nameparser/_facade.py | 5 ++++- tests/v2/test_facade.py | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index b41d4c49..d5ed0c90 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -91,6 +91,7 @@ Release Log - Reimplement ``HumanName`` as a facade over the 2.0 pipeline, and ``Constants`` as a shim resolving to a ``(Lexicon, Policy)`` snapshot with a shared parser cache. Fields, aggregates, mutation through ``name.C.titles.add(...)``, rendering defaults, ``capitalize()``, ``matches()``, ``comparison_key()``, iteration, ``as_dict()`` and pickling are all preserved, and ``nameparser.parser`` and ``nameparser.config`` remain importable. The compatibility layer ships through 2.x and is removed in 3.0 - Note that ``CONSTANTS.capitalize_name`` and ``force_mixed_case_capitalization`` are still honored through the facade, but the 2.0 API never capitalizes during ``parse()`` -- call ``capitalized()`` when you want it + - Add ``Role`` members (and their string values ``given``/``family``) as valid ``HumanName`` subscript keys: ``hn[Role.GIVEN]`` returns ``hn.first`` **Command line** diff --git a/nameparser/_facade.py b/nameparser/_facade.py index bc38d2c5..3f5e72f5 100644 --- a/nameparser/_facade.py +++ b/nameparser/_facade.py @@ -618,7 +618,10 @@ def __getitem__(self, key: str) -> str: "slicing a HumanName was removed in 2.0 (#258); access " "the named attributes instead" ) - return getattr(self, key) + # Role is a StrEnum, so Role members (and the plain 'given'/ + # 'family' strings) reach here too -- translate to the v1 + # spelling the facade actually exposes as attributes. + return getattr(self, _V1_SPELLING.get(key, key)) def as_dict(self, include_empty: bool = True) -> dict[str, str]: """The seven v1-named components as a dict; include_empty=False diff --git a/tests/v2/test_facade.py b/tests/v2/test_facade.py index d6658ac1..19beac82 100644 --- a/tests/v2/test_facade.py +++ b/tests/v2/test_facade.py @@ -7,6 +7,7 @@ from nameparser._config_shim import CONSTANTS, Constants from nameparser._facade import HumanName +from nameparser._types import Role _DATA_DIR = Path(__file__).parent / "data" @@ -221,6 +222,22 @@ def test_getitem_str_ok_slice_raises() -> None: # #258 n["first"] = "Jane" # type: ignore[index] # no __setitem__ in 2.0 +def test_getitem_accepts_role_members() -> None: + # Role is a StrEnum, so Role members reach __getitem__'s getattr; + # the v1 spellings (first/last) must resolve for given/family too. + hn = HumanName("Dr. John Quincy Smith Jr.") + assert hn[Role.TITLE] == hn.title + assert hn[Role.GIVEN] == hn.first + assert hn[Role.MIDDLE] == hn.middle + assert hn[Role.FAMILY] == hn.last + assert hn[Role.SUFFIX] == hn.suffix + assert hn[Role.NICKNAME] == hn.nickname + assert hn[Role.MAIDEN] == hn.maiden + # the plain strings work the same way + assert hn["given"] == hn.first + assert hn["family"] == hn.last + + def test_eq_and_hash_are_object_identity() -> None: # #223 a, b = HumanName("John Smith"), HumanName("John Smith") assert a != b and a == a From fbc81afd01b711412809cde0f2c15567fe4e27a2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 04:00:42 -0700 Subject: [PATCH 12/21] Sweep AGENTS.md and docs for the API-critique changes Co-Authored-By: Claude Fable 5 --- AGENTS.md | 8 +++++--- docs/concepts.rst | 4 +++- docs/release_log.rst | 15 ++++++++++----- docs/usage.rst | 10 +++++++++- nameparser/_lexicon.py | 17 +++++++++-------- nameparser/_policy.py | 2 -- tests/v2/test_locales.py | 6 +++++- 7 files changed, 41 insertions(+), 21 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f3a1b07a..4025622e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,15 +117,17 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 652-name corpus (two files: `corpus.jsonl` from the v1 test banks at a pinned ref, `corpus_issues.jsonl` harvested from the issue tracker; `compare.py` globs `corpus*.jsonl` and fails loudly if none match). `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy` only, and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. -- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. +- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. ``Role`` is a ``StrEnum``: members compare as their field-name strings, and ``tokens_for()`` coerces strings via ``_coerce_enum``. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Guard, hint, and emit for the WHOLE family, and parametrize the test over it**: a check added to one member of a set belongs on all of it, and the test must sweep the family, not one example. This session shipped `_reject_str_and_mapping` on `Policy` but not `PolicyPatch`, the bytes decode hint on three of five config entry points, and a regex-sync roster missing four of its copies — each a separate follow-up bug that a `{class} × {field} × {bad-value}` parametrization would have caught and a per-example test hid. When you find you're guarding member N, grep for the other members first. - **Ambiguities are emitted at the DECISION site**: an `Ambiguity` records a fork the parse had to call, not a token that sits in an ambiguous vocabulary. Emit where the branch is taken — the trailing-suffix peel in `_assign`, the delimiter escape's follow-up in `classify` — never by scanning for a `vocab:*-ambiguous` tag. The same tagged token is a genuine fork in one position and unremarkable in another (`do` mid-name in "Joao da Silva do Amaral de Souza" chooses nothing). **A branch that runs but changes nothing is not a decision either** -- the prefix chain's `merge(k, j)` executes even when `j == k + 1`, folding a piece into itself, and keying on "the code got here" reported a fork for all 39 ambiguous particles on "Dr. Van Jr.", where the particle stayed the GIVEN name and `_assign` reported the same token again. Check that the branch actually claimed something (`j > k + 1`) before recording. Structure also structure often settles the question before it arises, which is why `PARTICLE_OR_GIVEN` is deliberately not emitted on the `FAMILY_COMMA` path and `SUFFIX_OR_NAME` is not emitted for "Ma, Jack". The decision site also has the token index and the detail text in hand, which the tag scan would have to reconstruct. **If a fork's two branches are taken in DIFFERENT stages, every one of them needs the emitter** -- `PARTICLE_OR_GIVEN` is decided in `_assign` when the ambiguous particle stays a lone leading piece and in `_group` when a title shifts it off index 0 and the prefix chain claims it, so both report; for two years only the first did. The stage-ownership map in `tests/v2/pipeline/test_state.py` must list `ambiguities` for each such stage, and it passes vacuously until a case row exercises the path, so add the row too. Report BOTH directions of a two-way fork — "John Smith MA" (read as a suffix) and "Jack MA" (read as the family name) are equally guesses. Every kind needs a trigger in `tests/v2/test_contracts.py::_AMBIGUITY_TRIGGERS` (an explicit `None`, strict-xfail, while reserved), and case-table rows pin expected kinds exactly, so a new emitter shows up in both immediately. **Pin the decision, not the vocabulary**: the only titled-particle test used an UNAMBIGUOUS particle, so it walked the right code path and proved nothing about the branch under test -- two criticals passed 1539 tests. A row contrasting the two readings ("John Smith V" against "John Smith B") is what makes an emitter's absence meaningful. - **A kind is worth adding only if a reader would hesitate too**: the test is not "does the code take a branch" but whether a person reading that input would genuinely be unsure. "Smith, John V" reads as a middle initial to anyone -- the comma settles it -- so reporting it would be noise that teaches callers to ignore the field, which costs more than the missing report. Reachability of the second branch is necessary, not sufficient. Prefer leaving a fork silent and documenting the omission (see the comma paths in concepts.rst) over emitting on input nobody finds ambiguous. +- **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. +- **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. -- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). +- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). ``PolicyPatch``'s repr shows only set (non-UNSET) fields; ``_order_repr`` must never raise even on an unvalidated patch's garbage ``name_order`` (PolicyPatch defers validation to apply time); the sweep test in ``tests/v2/test_reprs.py`` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. @@ -174,7 +176,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`esq` is in both `SUFFIX_ACRONYMS` and `SUFFIX_NOT_ACRONYMS` on purpose — do not "deduplicate" it** — the two branches normalize differently (see the asymmetry gotcha above): the word test strips only edge periods, the acronym test strips all of them. `Esq` matches only through the word set, `E.S.Q.` only through the acronym set. Removing either membership drops a spelling and, for the acronym one, loses the family name (`"John Smith E.S.Q."` → `family='E.S.Q.'`). Pinned by the `suffix_acronym_multidot_spelling` case-table row. The disjointness that IS asserted is `SUFFIX_ACRONYMS_AMBIGUOUS ∩ SUFFIX_NOT_ACRONYMS`, which is a different claim: `suffix_as_written` ORs the branches, so a word membership bypasses the period gate the ambiguous set exists to impose. -**`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking (a property, a repr note, a warning), never a raise. +**`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking — the multi-word UserWarning in `_normset` is the shipped example; a raise remains wrong. **`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. **Anything built on `_normalize` must converge too** — `_title_key` joins per-word `_normalize` and DROPS words that fold away; keeping the empty slot stored `'lt .'` as `'lt '`, a key match-time can never rebuild (so the entry is silently inert) and `__setstate__` rejects on the next round-trip as "not written by this version". diff --git a/docs/concepts.rst b/docs/concepts.rst index fb01a0ec..0ee9ef5f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -38,7 +38,9 @@ relatives); a position cannot be confused with a look-alike. assignment, ever. If a parse is almost right and you want to fix one field, you call ``.replace()``, which returns a new :class:`~nameparser.ParsedName` with that field changed and everything -else — tokens, spans, the rest of the roles — carried over unchanged. +else — tokens, spans, the rest of the roles — carried over unchanged +(``replace()`` tokens carry no vocabulary tags; :meth:`Parser.revise +` is the tag-preserving form). ``str()`` renders the default view; nothing about calling it mutates the value you called it on. diff --git a/docs/release_log.rst b/docs/release_log.rst index d5ed0c90..5ad3791c 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -30,16 +30,13 @@ Release Log - Add ``name_order`` and the constants ``GIVEN_FIRST``, ``FAMILY_FIRST`` and ``FAMILY_FIRST_GIVEN_LAST``, so a family-first name can be parsed as written rather than reordered by hand (closes the configuration half of #270) - Add ``PatronymicRule`` with the members ``EAST_SLAVIC`` and ``TURKIC``. v1's single ``patronymic_name_order`` flag enabled both detectors at once; ``Policy(patronymic_rules=...)`` lets you enable either one alone - Add ``PolicyPatch`` and the ``UNSET`` sentinel for partial policy deltas that compose -- set-valued fields union, scalar fields override with later winning. This is the mechanism locale packs are built from, and ``UNSET`` is only needed when you must distinguish "not set" from a real ``False`` or ``None`` - - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from - - Change ``Role`` to a ``StrEnum``: members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind`` - - Change ``ParsedName.tokens_for()`` to accept a role's string name and to raise ``ValueError`` for an unknown role instead of silently returning no tokens + - Add ``Token``, ``Span`` and ``Role``: every field is backed by tokens carrying exact ``(start, end)`` offsets into the original string, reachable with ``tokens_for(Role.GIVEN)``. This replaces v1's ``*_list`` attributes and makes it possible to highlight or re-slice the input the parse came from. ``Role`` is a ``StrEnum``, so members compare and stringify as their field names (``Role.GIVEN == "given"``), matching ``AmbiguityKind``; ``tokens_for()`` accepts a role's string name too, and raises ``ValueError`` for an unknown role - Add ``STABLE_TAGS`` to the public API: the four documented ``Token.tags`` values (``particle``, ``conjunction``, ``initial``, ``joined``) - - Change ``ParsedName.as_dict()`` to take ``include_empty`` as keyword-only (``HumanName.as_dict()`` is unchanged) - Add ``Policy.patched(patch)``, applying a ``PolicyPatch`` directly without wrapping it in a locale pack - Add ``Parser.matches(a, b)`` and ``Parser.capitalized(name)``: the ``ParsedName`` methods of the same names fall back to the default configuration for str/omitted arguments, which is silently wrong for names parsed with a custom ``Parser`` - Add ``Parser.revise(name, **fields)``: ``ParsedName.replace()`` with the replacement text classified by the parser's vocabulary, so particle/initial/suffix-join behavior survives the edit - Add ``Ambiguity`` and the ``AmbiguityKind`` enum, so a parse reports what it had to guess at instead of guessing silently. The kinds emitted today are ``particle-or-given``, ``suffix-or-name``, ``suffix-or-nickname``, ``unbalanced-delimiter`` and ``comma-structure``; ``order`` is reserved and not yet emitted. The two suffix kinds cover the post-nominals that are also ordinary words: ``"John Smith MA"`` reports that ``MA`` was read as a credential rather than a surname, and ``"JEFFREY (JD) BRICKEN"`` that the delimited ``JD`` was read as a nickname rather than a suffix. A reading the vocabulary settles on its own — ``"John Smith M.A."``, ``"Andrew Perkins (MBA)"`` — is not a guess and reports nothing - - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()``, ``replace(**fields)``, ``matches()`` and ``comparison_key()`` + - Add ``ParsedName`` output and comparison methods: ``render(spec)``, ``initials()``, ``capitalized()`` (which returns a new value rather than mutating in place), ``as_dict()`` (whose ``include_empty`` flag is keyword-only, unlike ``HumanName.as_dict()``'s), ``replace(**fields)``, ``matches()`` and ``comparison_key()`` - Add ``Locale``, the public pack type. Writing your own needs no registration -- construct a ``Locale`` and pass it to ``parser_for()`` - Ship a fully typed public API (PEP 561): the core modules are checked under strict mypy settings, and nameparser 2.0 has no runtime dependencies @@ -109,6 +106,14 @@ Release Log tooling, not part of the installed package) -- see its README to reproduce the comparison against your own names. + **Changed since 2.0.0rc1** (for anyone who tested the release candidate) + + - ``Role`` became a ``StrEnum``; ``str(Role.GIVEN)`` is now ``"given"`` + - ``ParsedName.tokens_for()`` raises ``ValueError`` for unknown roles instead of returning no tokens; it also accepts role-name strings + - ``ParsedName.as_dict()``'s ``include_empty`` is keyword-only + - ``HumanName`` subscripting accepts ``Role`` members + - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns + * 1.4.0 - July 12, 2026 - Add ``Constants.copy()``, a detached deep copy that preserves the source instance's current customizations (unlike ``Constants()``, which always starts from library defaults) -- useful as ``CONSTANTS.copy()`` for a private snapshot of the shared config (#260) diff --git a/docs/usage.rst b/docs/usage.rst index 94b84d97..641b03f3 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -188,6 +188,11 @@ Fixing case >>> str(parse("juan de la vega").capitalized()) 'Juan de la Vega' +The no-argument form above uses the DEFAULT lexicon; for a name parsed +with a custom :class:`~nameparser.Parser`, call +:meth:`Parser.capitalized() ` so the +parser's own vocabulary decides the exceptions. + Nicknames and maiden names ---------------------------- @@ -297,7 +302,10 @@ Comparing names ``==`` is strict value equality — two :class:`~nameparser.ParsedName` instances are equal only if every field matches exactly. For "is this the same name, allowing for order and case?" use ``matches()`` or -``comparison_key()`` instead. +``comparison_key()`` instead. ``matches()`` parses a str argument with +the DEFAULT parser; to compare against a string using a custom +:class:`~nameparser.Parser`'s vocabulary, call +:meth:`Parser.matches() `. .. doctest:: diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index dd8c84ad..60f4c516 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -182,14 +182,15 @@ def _normset( # 2026-07-26). Warn, never raise: an inert entry produces # nothing, and the given_name_titles precedent says a raise # here costs working configurations (see __post_init__). - # warn=False is _edit()'s remove() path: a removal stores - # nothing, so warning there would name entries the caller is - # trying to get RID of, with "split it" advice that makes no - # sense for a no-op. add() still warns exactly once, via the - # new instance's __post_init__. - # interior whitespace test; split() covers all Unicode whitespace - if warn and field_name != "given_name_titles" \ - and n != "".join(n.split()): + # warn=False is _edit()'s pass (both ops): add() warns via the + # new instance's __post_init__; remove() stores nothing, so + # warning there would name entries the caller is trying to get + # RID of, with "split it" advice that makes no sense for a + # no-op. + if (warn and field_name != "given_name_titles" + # interior whitespace test; split() covers all Unicode + # whitespace + and n != "".join(n.split())): _warn_dead_entry( f"Lexicon.{field_name} entries are matched one word at " f"a time; multi-word entry {w!r} can never match. " diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 6a358d26..00964259 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -45,8 +45,6 @@ class PatronymicRule(StrEnum): #: ``Policy(name_order=...)`` values. FAMILY_FIRST_GIVEN_LAST = (Role.FAMILY, Role.MIDDLE, Role.GIVEN) -_NAME_ROLES = frozenset({Role.GIVEN, Role.MIDDLE, Role.FAMILY}) - _ORDER_CONSTANT_NAMES: dict[tuple[Role, ...], str] = { GIVEN_FIRST: "GIVEN_FIRST", FAMILY_FIRST: "FAMILY_FIRST", diff --git a/tests/v2/test_locales.py b/tests/v2/test_locales.py index 344824b2..9fdb05e1 100644 --- a/tests/v2/test_locales.py +++ b/tests/v2/test_locales.py @@ -79,7 +79,11 @@ def test_pack_vocabulary_entries_are_single_words() -> None: # this file may already have forced the import, which would make a # warning-capture version of this test pass even on a regression. # Iterates every registered pack, not just ru/tr_az by name, so a - # future pack is covered automatically. + # future pack is covered automatically. The field loop is dormant + # until a pack actually ships vocabulary -- both current packs build + # on Lexicon.empty() -- so the registry-non-empty assert below is + # what keeps the test from passing vacuously today. + assert locales.available() for code in locales.available(): lexicon = locales.get(code).lexicon for field in _VOCAB_FIELDS: From 86157d6ef6bb80ecd0a67c45106ec528f5abccaa Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 04:13:06 -0700 Subject: [PATCH 13/21] Drop the eight legacy dead entries when restoring a pre-2.0 pickle Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 +- nameparser/_config_shim.py | 29 +++++++++++++++++++++++++++-- tests/v2/test_config_shim.py | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index 5ad3791c..fb34aea2 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -112,7 +112,7 @@ Release Log - ``ParsedName.tokens_for()`` raises ``ValueError`` for unknown roles instead of returning no tokens; it also accepts role-name strings - ``ParsedName.as_dict()``'s ``include_empty`` is keyword-only - ``HumanName`` subscripting accepts ``Role`` members - - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns + - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns; those eight are dropped silently, not warned about, when a pre-2.0 ``Constants`` pickle that froze them is restored * 1.4.0 - July 12, 2026 diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index 05451c11..c8fb6848 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -33,6 +33,22 @@ from nameparser.util import lc +#: The eight multi-word entries the pre-2.0 DEFAULT vocabulary shipped. +#: Provably inert in every release (they can never match; see the +#: 2.0.0 release log), so dropping them from a restored legacy pickle +#: changes no parse -- and keeps the multi-word warning from firing +#: eight times, with wrong advice, at library-internal lines, on the +#: first parse after a supported 1.3/1.4 pickle upgrade. A multi-word +#: entry the USER added still warns: only these eight exact entries +#: are dropped. +_LEGACY_DEAD_ENTRIES = { + "titles": frozenset({"chargé d'affaires"}), + "suffix_acronyms": frozenset({ + "leed ap", "nicet i", "nicet ii", "nicet iii", "nicet iv", + "psm i", "psm ii"}), +} + + def _reject_bare_str_or_bytes(value: object, expected: str) -> None: # A bare string is an iterable of its characters, so e.g. SetManager('dr') # would silently shred it into {'d', 'r'} instead of raising -- shared by @@ -1105,8 +1121,17 @@ def __setstate__(self, state: dict[str, object]) -> None: # instance, not whatever produced the incoming state) for name in _SET_FIELDS: if name in state: - object.__setattr__(self, name, SetManager( - state[name], _on_change=self._bump)) # type: ignore[arg-type] + manager = SetManager( + state[name], _on_change=self._bump) # type: ignore[arg-type] + # SetManager normalized on construction, so the frozen + # 1.3/1.4 vocabulary's dead entries are matchable in + # their normalized spelling here. Reach past the public + # discard() deliberately: this is part of restoring the + # state, not a mutation of it, and must not bump the + # generation of an instance that is still being built. + manager._elements -= _LEGACY_DEAD_ENTRIES.get( + name, frozenset()) + object.__setattr__(self, name, manager) if "capitalization_exceptions" in state: object.__setattr__( self, "capitalization_exceptions", TupleManager( diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index c033b4aa..3e8e845a 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -357,6 +357,23 @@ def test_v14_constants_blob_unpickles_into_shim() -> None: assert loaded.string_format == "{title} {first} {middle} {last} {suffix} ({nickname})" +def test_v14_pickle_restores_and_parses_warning_free() -> None: + # A 1.4 pickle froze the then-shipped vocabulary, including the + # eight dead multi-word entries 2.0 removed; restoring it must not + # spray warnings about entries the user never added. A multi-word + # entry the user DID add still warns (second half) -- at PARSE + # time, not add() time, since the shim's Lexicon snapshot is built + # lazily on the first parse after a mutation. + with open(_DATA_DIR / "constants_v14.pickle", "rb") as f: + c = pickle.load(f) + with warnings.catch_warnings(): + warnings.simplefilter("error") + HumanName("John Smith", constants=c) + c.titles.add("grand moff") + with pytest.warns(UserWarning, match="matched one word at a time"): + HumanName("Jane Roe", constants=c) + + def test_snapshot_keeps_a_multi_word_first_name_title() -> None: # v1 looks first_name_titles up on the joined title string, so a # multi-word entry is reachable with only its WORDS in titles. From d0235792978eeb146f47dda6bf39b9faf6e21c65 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 04:14:21 -0700 Subject: [PATCH 14/21] Polish from the final bundle review: rc1-delta notes, doc links, repr sweep Co-Authored-By: Claude Fable 5 --- AGENTS.md | 6 +++--- docs/concepts.rst | 3 ++- docs/release_log.rst | 2 ++ tests/v2/test_reprs.py | 2 ++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4025622e..331c82fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -117,7 +117,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Module layout**: every new module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_pipeline/`, `_parser.py`, plus the facade layer: `_facade.py`, `_config_shim.py`). The public import surface is exactly `nameparser` and `nameparser.locales`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. Since the M11 swap, the old paths are import-path-preserving re-exports: `nameparser.parser` re-exports the `_facade` `HumanName`, `nameparser.config` re-exports the `_config_shim` names (`Constants`, `CONSTANTS`, `SetManager`, `TupleManager`, `RegexTupleManager`); the `config/` DATA modules stay the vocabulary source through 2.x. The whole facade layer is deleted in 3.0. - **Facade layer** (`_facade.py`, `_config_shim.py`): the v1-compat `HumanName`/`Constants` over the core. Key mechanisms: `Constants._generation` dirty-tracking (every mutation bumps; facades resolve their `Parser` lazily via `_cached_parser(lexicon, policy)`); `Constants._snapshot()` mirrors `_lexicon._default_lexicon()` (equality-pinned); the facade pickles v1-SHAPED state (component lists, one `__setstate__` path for 1.4 and 2.x blobs; components rebuild via `replace()`, never a re-parse); `_V1_HOOKS` overrides warn once per subclass (#280). The compat contract is the migration spec's promise: warning-free 1.4 code behaves identically except release-log-classified fixes — `tools/differential/` (dev-only, not shipped) verifies this against 1.4-on-PyPI over a 652-name corpus (two files: `corpus.jsonl` from the v1 test banks at a pinned ref, `corpus_issues.jsonl` harvested from the issue tracker; `compare.py` globs `corpus*.jsonl` and fails loudly if none match). `parser.py:NNNN` citations throughout the 2.0 code refer to the PRE-swap v1 file, deleted at the M11 swap; resolve them with `git show 2d5d8c2:nameparser/parser.py`. - **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules only — vocabulary is single-sourced from the v1 data modules through 2.x, e.g. `config/maiden_markers.py`); `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin); `_render` imports `_types` and `_lexicon` (for `Lexicon.default()` and `_normalize`); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers; `_parser` sits on everything except `_render`; the facade layer (`_facade`, `_config_shim`, `parser`, `config/__init__`, `__main__`) may import anything public plus `_render`; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy` only, and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. -- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. ``Role`` is a ``StrEnum``: members compare as their field-name strings, and ``tokens_for()`` coerces strings via ``_coerce_enum``. +- **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. Sanctioned deviation, facade layer only: `HumanName` and the shim `Constants` organize by v1 concern groups (`# -- render defaults --`, `# -- config / parsing --`, `# -- fields --`, ..., dunders and pickle last) — the classes mirror v1's own surface and die in 3.0; the canonical order still binds every core type. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). - **Guard, hint, and emit for the WHOLE family, and parametrize the test over it**: a check added to one member of a set belongs on all of it, and the test must sweep the family, not one example. This session shipped `_reject_str_and_mapping` on `Policy` but not `PolicyPatch`, the bytes decode hint on three of five config entry points, and a regex-sync roster missing four of its copies — each a separate follow-up bug that a `{class} × {field} × {bad-value}` parametrization would have caught and a per-example test hid. When you find you're guarding member N, grep for the other members first. @@ -127,7 +127,7 @@ The 2.0 rewrite lands as underscore-private modules alongside the v1 code. These - **Per-word vocabulary fields warn on multi-word entries** (`_normset`/`_normpairs` via `_warn_dead_entry`, UserWarning, never a raise — see the given_name_titles Gotcha for why raising is wrong). `given_name_titles` is the one multi-word-matched field and is exempt; `_edit` passes `warn=False` (add() warns once via the new instance's `__post_init__`; remove() stores nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). - **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. `suffix_acronyms_ambiguous ∩ suffix_words` is guarded because the overlap loses a family name; `given_name_titles` is not, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. - **The shim TRANSLATES; it never raises on a config v1 accepted, and never silently changes the parse**: `Constants._snapshot()` is a translation boundary between v1's model and v2's invariants, and every transformation there carries its v1-reachability argument in a comment. Four exist today — `first_name_titles` re-folded per word (v1 joins-then-`lc`, v2 normalizes-then-joins), `suffix_acronyms_ambiguous ∩ acronyms` (a provable no-op), `suffix_words − ambiguous` (v1 already accepts the word via the acronym branch, so the addition is inert there), and `particles_ambiguous ∪ (bound ∩ particles)` (a pinned deviation, `test_bound_never_given_prefix_deviates_on_two_pieces`). When a v1 config cannot satisfy a v2 invariant, work out what v1 actually *does* with it — usually nothing — and reproduce that; weakening the invariant or letting the raise through are both wrong. **Test the case the translation decides**, not one where both branches agree: a test using an input v1 parses identically with and without the config pins nothing. -- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). ``PolicyPatch``'s repr shows only set (non-UNSET) fields; ``_order_repr`` must never raise even on an unvalidated patch's garbage ``name_order`` (PolicyPatch defers validation to apply time); the sweep test in ``tests/v2/test_reprs.py`` pins that no config repr leaks the UNSET sentinel. +- **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **Pickling**: v2 types must round-trip (`Parser` will be picklable by construction, and it holds a `Lexicon`). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, and values are deliberately NOT re-validated (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` keeps its own copy of the guard (layering) plus the `mappingproxy` slot rebuild; a new unpicklable slot type needs the same treatment plus a round-trip test. - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. Sanctioned exceptions, facade layer only: `_config_shim.CONSTANTS` (the v1 shared singleton, mutable by design) and `_facade._WARNED_SUBCLASSES` (the once-per-subclass hook-warning dedup set) — both deleted with the layer in 3.0. @@ -176,7 +176,7 @@ Add a dedicated `copy.deepcopy()` round-trip test for it too (see `test_regexes_ **`esq` is in both `SUFFIX_ACRONYMS` and `SUFFIX_NOT_ACRONYMS` on purpose — do not "deduplicate" it** — the two branches normalize differently (see the asymmetry gotcha above): the word test strips only edge periods, the acronym test strips all of them. `Esq` matches only through the word set, `E.S.Q.` only through the acronym set. Removing either membership drops a spelling and, for the acronym one, loses the family name (`"John Smith E.S.Q."` → `family='E.S.Q.'`). Pinned by the `suffix_acronym_multidot_spelling` case-table row. The disjointness that IS asserted is `SUFFIX_ACRONYMS_AMBIGUOUS ∩ SUFFIX_NOT_ACRONYMS`, which is a different claim: `suffix_as_written` ORs the branches, so a word membership bypasses the period gate the ambiguous set exists to impose. -**`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking — the multi-word UserWarning in `_normset` is the shipped example; a raise remains wrong. +**`Lexicon.given_name_titles` is deliberately unvalidated against `titles` — do not add a check** — the lookup key is the space-joined run of `Role.TITLE` tokens, built by the parse, and a conjunction inside a run is itself tagged `Role.TITLE`, so `"sir and dame"` is a matchable key whose middle word lives in `conjunctions`. A whole-entry check rejected multi-word entries; a per-word check rejected that one. No static relation over the vocabulary sets decides reachability. An unreachable entry is inert — nothing consults it and nothing misparses — so the condition being "guarded" costs the user nothing while each guard cost a working configuration. If a diagnostic is wanted, it must be non-blocking — the multi-word UserWarning in `_normset` is the shipped example; a raise remains wrong. Note the SHIPPED data cannot carry a multi-word given-name title without a spurious warning: `FIRST_NAME_TITLES ⊆ TITLES` puts the entry in `titles` too, which is per-word warned; user-supplied v2 Lexicons are unaffected (`add(given_name_titles=...)` alone is silent). **`_normalize` must reach a fixed point** — storage and match-time share the one fold, and `Lexicon.__setstate__` re-validates, so a value that changes on re-normalization changes under its owner. `strip().strip(".")` alone is not idempotent (`'. a .'` → `' a '` → `'a'`). The loop is the fix; keep any new stripping inside it. **Anything built on `_normalize` must converge too** — `_title_key` joins per-word `_normalize` and DROPS words that fold away; keeping the empty slot stored `'lt .'` as `'lt '`, a key match-time can never rebuild (so the entry is silently inert) and `__setstate__` rejects on the next round-trip as "not written by this version". diff --git a/docs/concepts.rst b/docs/concepts.rst index 0ee9ef5f..7704a251 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -176,7 +176,8 @@ signal to act on; do not read an empty one as a guarantee. :class:`Tokens ` also carry tags — a second, independent label alongside their role, recording how a token was classified rather than what part of the name it belongs to — but only a handful of them are part of the -stable API: ``particle``, ``conjunction``, ``initial``, and ``joined``. +stable API, collected in :data:`~nameparser.STABLE_TAGS`: ``particle``, +``conjunction``, ``initial``, and ``joined``. Any tag written with a namespace prefix, like ``vocab:...``, is provenance information for debugging how a token got classified — it can change shape between releases and isn't something to match against diff --git a/docs/release_log.rst b/docs/release_log.rst index fb34aea2..babc9b42 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -113,6 +113,8 @@ Release Log - ``ParsedName.as_dict()``'s ``include_empty`` is keyword-only - ``HumanName`` subscripting accepts ``Role`` members - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns; those eight are dropped silently, not warned about, when a pre-2.0 ``Constants`` pickle that froze them is restored + - ``PolicyPatch``'s repr shows only the fields a patch sets, instead of all nine with UNSET sentinels + - New since rc1: ``STABLE_TAGS``, ``Policy.patched()``, ``Parser.matches()``, ``Parser.capitalized()``, and ``Parser.revise()`` -- see the API section above * 1.4.0 - July 12, 2026 diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index 5571701f..f31fd7ee 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -85,6 +85,8 @@ def test_policy_patch_repr_shows_only_set_fields() -> None: PolicyPatch(strip_emoji=False), Locale("xx", Lexicon.empty()), Locale("xx", Lexicon.empty(), PolicyPatch(middle_as_family=True)), + Lexicon.default(), + Lexicon.empty(), Parser(), ]) def test_config_reprs_never_leak_the_unset_sentinel(obj: object) -> None: From d556dae50d0b5de446ca066e9eb6e763a9377a2f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 11:20:17 -0700 Subject: [PATCH 15/21] =?UTF-8?q?Ship=20the=20unaccented=20'charge'=20titl?= =?UTF-8?q?e=20spelling,=20like=20attach=C3=A9/attache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 --- docs/release_log.rst | 2 +- nameparser/config/titles.py | 1 + tests/test_titles.py | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/release_log.rst b/docs/release_log.rst index babc9b42..553843bd 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -64,7 +64,7 @@ Release Log - Fix the pre-comma piece being routed to ``first`` when everything after the comma is a suffix or title: ``"Andrews, M.D."`` now reads family ``Andrews`` / suffix ``M.D.`` where 1.x read given ``M.D.`` / family ``Andrews``, and ``"Smith, Dr."`` moves ``Smith`` from ``first`` to ``family`` (the title was already correct in 1.x). The piece before a comma is definitionally the family name - Fix a lone recognized trailing suffix with no comma being routed to ``first``/``last``: ``"Johnson PhD"`` and ``"Mr. Johnson PhD"`` now keep the suffix in ``suffix`` - Fix a split ``"Ph. D."`` credential being read as two tokens; it now classifies as one suffix, replacing v1's ``fix_phd`` hook. This now holds wherever the credential sits: 1.x healed the pair only when it trailed, so ``"Ph. D. John Smith"`` parsed as title ``Ph.`` / given ``D.`` with the real given name pushed to ``middle``; it now reads given ``John``, family ``Smith``, suffix ``Ph. D.`` - - Fix ``chargé d'affaires``: shipped as one unmatchable ``TITLES`` entry since it was added, it is now two chainable entries (``chargé``, ``d'affaires``), so the title is recognized + - Fix ``chargé d'affaires``: shipped as one unmatchable ``TITLES`` entry since it was added, it is now two chainable entries (``chargé``, ``d'affaires``), so the title is recognized; the unaccented spelling ``charge`` ships too, like ``attaché``/``attache`` - Remove seven multi-word ``SUFFIX_ACRONYMS`` entries (``leed ap``, ``nicet i``–``nicet iv``, ``psm i``, ``psm ii``) that could never match in any release; splitting them would swallow real names ("John Leed", "Smith, A.P."), so they are dropped instead - Add a ``UserWarning`` when a multi-word entry is stored in a per-word ``Lexicon`` field or as a ``capitalization_exceptions`` key -- such entries can never match - Parse an input with no alphanumeric character to an empty name in both APIs. 1.x kept pure punctuation as a name part, so ``"."`` gave ``first="."`` and ``bool()`` was ``True``; 2.0 empties it, keeping ``bool(parse(x))`` an honest "did I get a name?" test. The check is Unicode-aware, so names in any script are unaffected; only inputs that are entirely punctuation or symbols (``"."``, ``"- -"``) change. Junk embedded in a name with real content -- the stray dot in ``"John . Smith"`` -- is still kept, since that parse is already truthy diff --git a/nameparser/config/titles.py b/nameparser/config/titles.py index 1e260887..726d69b5 100644 --- a/nameparser/config/titles.py +++ b/nameparser/config/titles.py @@ -190,6 +190,7 @@ 'chairs', 'chancellor', 'chaplain', + 'charge', # unaccented 'chargé', like attaché/'attache' 'chef', 'chemist', 'chief', diff --git a/tests/test_titles.py b/tests/test_titles.py index a73a7f6c..7bfee722 100644 --- a/tests/test_titles.py +++ b/tests/test_titles.py @@ -380,3 +380,10 @@ def test_charge_daffaires_chains_as_title(self) -> None: self.m(hn.title, "Chargé d'Affaires", hn) self.m(hn.first, "John", hn) self.m(hn.last, "Smith", hn) + + def test_unaccented_charge_daffaires_chains_as_title(self) -> None: + # both spellings ship, like attaché/attache + hn = HumanName("Charge d'Affaires John Smith") + self.m(hn.title, "Charge d'Affaires", hn) + self.m(hn.first, "John", hn) + self.m(hn.last, "Smith", hn) From 76cc09bf2c73531221ce012d62508b00fb9bd1c1 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 12:00:32 -0700 Subject: [PATCH 16/21] Harden the review seams: repr smuggle-proofing, tag docs, warning attribution Co-Authored-By: Claude Fable 5 --- docs/modules.rst | 4 ++- docs/release_log.rst | 2 +- nameparser/_config_shim.py | 39 ++++++++++++++++++-------- nameparser/_lexicon.py | 11 ++++++-- nameparser/_parser.py | 54 ++++++++++++++++++++---------------- nameparser/_policy.py | 14 ++++++---- nameparser/_types.py | 9 ++++-- tests/v2/test_config_shim.py | 22 +++++++++++++++ tests/v2/test_lexicon.py | 11 ++++++++ tests/v2/test_reprs.py | 25 +++++++++++++++++ tests/v2/test_types.py | 7 +++++ 11 files changed, 150 insertions(+), 48 deletions(-) diff --git a/docs/modules.rst b/docs/modules.rst index 88f44a0c..025acadc 100644 --- a/docs/modules.rst +++ b/docs/modules.rst @@ -27,7 +27,9 @@ Results :value: frozenset({"particle", "conjunction", "initial", "joined"}) The four :attr:`Token.tags ` values that are - stable API: ``particle`` (a family-name particle, "de"/"van"), + stable API: ``particle`` (a word from the particle vocabulary, + "de"/"van", wherever it lands — combine with ``Role.FAMILY`` for + actual family particles), ``conjunction`` (a joining word, "and"/"y"), ``initial`` (an initial-shaped word, "J."), and ``joined`` (a continuation of the previous token within one merged piece, so the suffix view renders diff --git a/docs/release_log.rst b/docs/release_log.rst index 553843bd..3bf09a68 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -112,7 +112,7 @@ Release Log - ``ParsedName.tokens_for()`` raises ``ValueError`` for unknown roles instead of returning no tokens; it also accepts role-name strings - ``ParsedName.as_dict()``'s ``include_empty`` is keyword-only - ``HumanName`` subscripting accepts ``Role`` members - - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns; those eight are dropped silently, not warned about, when a pre-2.0 ``Constants`` pickle that froze them is restored + - The eight multi-word vocabulary entries that could never match were repaired (``chargé d'affaires`` split; seven credential acronyms removed), and storing a new multi-word entry now warns; those eight are dropped silently, not warned about, when a restored ``Constants`` pickle carries all eight of them (the pre-2.0 signature) - ``PolicyPatch``'s repr shows only the fields a patch sets, instead of all nine with UNSET sentinels - New since rc1: ``STABLE_TAGS``, ``Policy.patched()``, ``Parser.matches()``, ``Parser.capitalized()``, and ``Parser.revise()`` -- see the API section above diff --git a/nameparser/_config_shim.py b/nameparser/_config_shim.py index c8fb6848..f0a2ca85 100644 --- a/nameparser/_config_shim.py +++ b/nameparser/_config_shim.py @@ -38,9 +38,15 @@ #: 2.0.0 release log), so dropping them from a restored legacy pickle #: changes no parse -- and keeps the multi-word warning from firing #: eight times, with wrong advice, at library-internal lines, on the -#: first parse after a supported 1.3/1.4 pickle upgrade. A multi-word -#: entry the USER added still warns: only these eight exact entries -#: are dropped. +#: first parse after a supported 1.3/1.4 pickle upgrade. +#: +#: Gated on ALL EIGHT being present across the two fields -- the +#: signature of a pre-2.0 blob, which froze the complete shipped set. +#: A round-trip of 2.0-era state that carries FEWER than all eight +#: keeps them (a user who deliberately re-added one or two does not +#: match the signature, and .copy() never subtracts either). The trade: +#: a 2.0 user who re-adds ALL eight exact strings is indistinguishable +#: from a legacy blob and loses them on the next unpickle. _LEGACY_DEAD_ENTRIES = { "titles": frozenset({"chargé d'affaires"}), "suffix_acronyms": frozenset({ @@ -1119,19 +1125,30 @@ def __setstate__(self, state: dict[str, object]) -> None: self.__init__() # type: ignore[misc] # defaults, then overlay # (managers re-wrapped below so _on_change points at THIS # instance, not whatever produced the incoming state) + managers: dict[str, SetManager] = {} for name in _SET_FIELDS: if name in state: - manager = SetManager( + managers[name] = SetManager( state[name], _on_change=self._bump) # type: ignore[arg-type] - # SetManager normalized on construction, so the frozen - # 1.3/1.4 vocabulary's dead entries are matchable in - # their normalized spelling here. Reach past the public - # discard() deliberately: this is part of restoring the - # state, not a mutation of it, and must not bump the - # generation of an instance that is still being built. + # SetManager normalized on construction, so the frozen 1.3/1.4 + # vocabulary's dead entries are matchable in their normalized + # spelling here. Subtract only when ALL EIGHT are present -- + # the pre-2.0 signature; a 2.0 user who re-added one or two + # keeps them through a round-trip (see _LEGACY_DEAD_ENTRIES). + legacy = all( + name in managers and entry in managers[name] + for name, entries in _LEGACY_DEAD_ENTRIES.items() + for entry in entries + ) + for name, manager in managers.items(): + if legacy: + # Reach past the public discard() deliberately: this is + # part of restoring the state, not a mutation of it, and + # must not bump the generation of an instance that is + # still being built. manager._elements -= _LEGACY_DEAD_ENTRIES.get( name, frozenset()) - object.__setattr__(self, name, manager) + object.__setattr__(self, name, manager) if "capitalization_exceptions" in state: object.__setattr__( self, "capitalization_exceptions", TupleManager( diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 60f4c516..b64dcf70 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -117,12 +117,17 @@ def _reject_buffer(value: object, label: str, plural: str) -> None: def _warn_dead_entry(message: str) -> None: # A fixed stacklevel always lands on library internals: the call # depth differs per entry point (Lexicon(), add(), unpickle, - # dataclasses.replace). Walk out of this module (and dataclasses' - # replace frames) so the warning points at the caller's own line. + # dataclasses.replace, and the v1 shim's lazy snapshot -- built on + # the first parse after a Constants mutation, several facade frames + # below the user's own add()). Walk out of this module (and + # dataclasses' replace frames, and the facade layer that builds + # lexicons on the caller's behalf) so the warning points at the + # caller's own line. level = 2 frame: FrameType | None = sys._getframe(1) while frame is not None and frame.f_globals.get("__name__") in ( - __name__, "dataclasses"): + __name__, "dataclasses", + "nameparser._config_shim", "nameparser._facade"): frame, level = frame.f_back, level + 1 warnings.warn(message, UserWarning, stacklevel=level) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index eef0090e..6389ce6b 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -78,30 +78,7 @@ def parse(self, text: str) -> ParsedName: policy=self.policy) return assemble(run(state)) - def matches(self, a: str | ParsedName, b: str | ParsedName) -> bool: - """Component-wise case-insensitive comparison of two names, - parsing str arguments with THIS parser. - :meth:`ParsedName.matches` parses its str argument with the - DEFAULT parser instead -- for names parsed with a custom - Parser, use this method.""" - if isinstance(a, str): - a = self.parse(a) - elif not isinstance(a, ParsedName): - raise TypeError(f"matches() takes str or ParsedName, got {a!r}") - if isinstance(b, str): - b = self.parse(b) - elif not isinstance(b, ParsedName): - raise TypeError(f"matches() takes str or ParsedName, got {b!r}") - return a.comparison_key() == b.comparison_key() - - def capitalized(self, name: ParsedName, *, - force: bool = False) -> ParsedName: - """:meth:`ParsedName.capitalized` under THIS parser's lexicon. - The no-argument form of that method uses the DEFAULT lexicon -- - for names parsed with a custom Parser, use this method.""" - if not isinstance(name, ParsedName): - raise TypeError(f"capitalized() takes a ParsedName, got {name!r}") - return name.capitalized(self.lexicon, force=force) + # -- editing ---------------------------------------------------------- def revise(self, name: ParsedName, **fields: str) -> ParsedName: """:meth:`ParsedName.replace` with this parser's vocabulary: @@ -131,6 +108,35 @@ def revise(self, name: ParsedName, **fields: str) -> ParsedName: } return name._with_field_tokens(harvested) + # -- comparison ------------------------------------------------------- + + def matches(self, a: str | ParsedName, b: str | ParsedName) -> bool: + """Component-wise case-insensitive comparison of two names, + parsing str arguments with THIS parser. + :meth:`ParsedName.matches` parses its str argument with the + DEFAULT parser instead -- for names parsed with a custom + Parser, use this method.""" + if isinstance(a, str): + a = self.parse(a) + elif not isinstance(a, ParsedName): + raise TypeError(f"matches() takes str or ParsedName, got {a!r}") + if isinstance(b, str): + b = self.parse(b) + elif not isinstance(b, ParsedName): + raise TypeError(f"matches() takes str or ParsedName, got {b!r}") + return a.comparison_key() == b.comparison_key() + + # -- rendering delegates ---------------------------------------------- + + def capitalized(self, name: ParsedName, *, + force: bool = False) -> ParsedName: + """:meth:`ParsedName.capitalized` under THIS parser's lexicon. + The no-argument form of that method uses the DEFAULT lexicon -- + for names parsed with a custom Parser, use this method.""" + if not isinstance(name, ParsedName): + raise TypeError(f"capitalized() takes a ParsedName, got {name!r}") + return name.capitalized(self.lexicon, force=force) + @functools.cache def _default_parser() -> Parser: diff --git a/nameparser/_policy.py b/nameparser/_policy.py index 00964259..e750798a 100644 --- a/nameparser/_policy.py +++ b/nameparser/_policy.py @@ -56,11 +56,15 @@ def _order_repr(value: tuple[Role, ...]) -> str: # Unreachable via Policy's constructor (its __post_init__ restricts # name_order to the three named orders) but REACHABLE via # PolicyPatch, which defers name_order validation to apply time by - # design -- value may hold non-Role, even unhashable, elements. - # repr must never raise, so take the named-lookup path only once - # every element is confirmed a Role. (The annotation states the - # Policy-side truth; the PolicyPatch call site passes getattr-Any.) - if all(isinstance(r, Role) for r in value): + # design -- value may hold non-Role, even unhashable, elements. A + # value smuggled in through __setstate__ (which validates layout, + # not values) can also be a non-tuple container or not iterable at + # all. repr must never raise, so the named-lookup path is taken + # only for a TUPLE whose every element is confirmed a Role; + # everything else renders via repr(value). (The annotation states + # the Policy-side truth; the PolicyPatch call site passes + # getattr-Any.) + if isinstance(value, tuple) and all(isinstance(r, Role) for r in value): named = _ORDER_CONSTANT_NAMES.get(value) if named is not None: return named diff --git a/nameparser/_types.py b/nameparser/_types.py index 4d46afc2..b04af901 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -75,9 +75,12 @@ def __add__(self, other: object) -> NoReturn: # type: ignore[override] ) -#: The four :attr:`Token.tags` values that are stable API. "particle" -#: marks a family-name particle ("de", "van"); "conjunction" a joining -#: word ("and", "y"); "initial" an initial-shaped word ("J.", "Q"); +#: The four :attr:`Token.tags` values that are stable API. +#: "particle" marks a word from the particle vocabulary ("de", "van") +#: wherever it lands -- including a given-name "Van" -- so combine it +#: with Role.FAMILY (as family_particles does) to get actual family +#: particles; "conjunction" a joining word ("and", "y"); "initial" an +#: initial-shaped word ("J.", "Q"); #: "joined" a continuation of the previous token within one merged #: piece ("Ph." + "D."), which the suffix view joins with a space #: instead of ", ". Every other tag is namespaced ("vocab:...") and is diff --git a/tests/v2/test_config_shim.py b/tests/v2/test_config_shim.py index 3e8e845a..194fc6ed 100644 --- a/tests/v2/test_config_shim.py +++ b/tests/v2/test_config_shim.py @@ -792,3 +792,25 @@ def test_constants_shared_flag_is_read_only() -> None: with pytest.raises(AttributeError, match="read-only"): CONSTANTS._shared = False assert CONSTANTS._shared is True and c._shared is False + + +def test_multiword_warning_through_shim_points_at_caller() -> None: + c = Constants() + c.titles.add("zqx zqy") + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + HumanName("John Smith", constants=c) # snapshot builds lazily here + multi = [x for x in w if "matched one word at a time" in str(x.message)] + assert multi and all(x.filename == __file__ for x in multi) + + +def test_2x_pickle_roundtrip_keeps_a_readded_dead_entry() -> None: + # the all-eight gate: a 2.0 user's deliberate re-add of ONE legacy + # string survives a round-trip (only a full pre-2.0 blob, which + # froze all eight, is subtracted) + c = Constants() + with pytest.warns(UserWarning): + c.suffix_acronyms.add("leed ap") + HumanName("John Smith", constants=c) # snapshot builds lazily here + c2 = pickle.loads(pickle.dumps(c)) + assert "leed ap" in c2.suffix_acronyms diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index c894eaf9..14259a1b 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -524,3 +524,14 @@ def test_remove_does_not_warn() -> None: with warnings.catch_warnings(): warnings.simplefilter("error") Lexicon.default().remove(titles=["zqx zqy"]) + + +def test_multiword_warning_points_at_caller_line() -> None: + # the frame walk is the feature: both entry depths must attribute + # the warning to THIS file, not library internals + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + Lexicon(titles=frozenset({"zqx zqy"})) + Lexicon.empty().add(titles=["zqx zqy"]) + assert len(w) >= 2 + assert all(x.filename == __file__ for x in w) diff --git a/tests/v2/test_reprs.py b/tests/v2/test_reprs.py index f31fd7ee..71ede1f5 100644 --- a/tests/v2/test_reprs.py +++ b/tests/v2/test_reprs.py @@ -1,3 +1,6 @@ +import dataclasses +from typing import cast + import pytest from nameparser._lexicon import Lexicon @@ -106,3 +109,25 @@ def test_policy_patch_repr_survives_unvalidated_name_order() -> None: assert "given" in repr(garbage) # described, not crashed unhashable = PolicyPatch(name_order=([1], "y", "z")) # type: ignore[arg-type] repr(unhashable) # must not raise + + +def test_order_repr_survives_smuggled_setstate_values() -> None: + # __setstate__ validates layout, not values (the pickling rule), so + # the reprs must survive shapes no constructor can produce: an + # all-Role list (passes an element check, unhashable), a + # non-iterable, and a tuple of Role-EQUAL plain strings (StrEnum + # equality would hit the named lookup and lie). + for garbage in ([Role.GIVEN, Role.MIDDLE, Role.FAMILY], 5, + ("given", "middle", "family")): + for cls in (Policy, PolicyPatch): + obj = cls() + state = {f.name: getattr(obj, f.name) + for f in dataclasses.fields(obj)} + state["name_order"] = garbage + smuggled = cast("Policy | PolicyPatch", cls.__new__(cls)) + smuggled.__setstate__(state) + rendered = repr(smuggled) # must not raise + if isinstance(garbage, tuple): + # a smuggled string tuple must not render as the + # named constant it merely compares equal to + assert "GIVEN_FIRST" not in rendered diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index 71eaeb7d..cb9fe335 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -428,3 +428,10 @@ def test_stable_tags_is_public_api() -> None: def test_as_dict_include_empty_is_keyword_only() -> None: with pytest.raises(TypeError): parse("John Smith").as_dict(False) # type: ignore[misc] + + +def test_particle_tag_marks_vocabulary_membership_not_role() -> None: + # the tag follows the WORD, not the field: a leading ambiguous + # particle read as a given name still carries it (STABLE_TAGS docs) + tok = parse("Van Johnson").tokens_for(Role.GIVEN)[0] + assert "particle" in tok.tags From 3bf981fdb1a83eda0841ebb5f4cee853369d6539 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 12:02:24 -0700 Subject: [PATCH 17/21] Pin the review's test gaps: warning counts, role forcing, enum disjointness Co-Authored-By: Claude Fable 5 --- nameparser/_lexicon.py | 4 ++++ nameparser/_types.py | 15 ++++++++++++++- tests/v2/test_lexicon.py | 21 +++++++++++++++++++++ tests/v2/test_parser.py | 13 ++++++++++++- tests/v2/test_types.py | 17 +++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index b64dcf70..6c025008 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -537,6 +537,10 @@ def _edit(self, op: str, entries: Mapping[str, Iterable[str]]) -> Lexicon: # remove() never reaches __post_init__ with the dead entry # (it is subtracted out here), so it stays silent -- correct, # since a removal stores nothing a warning could be about. + # That silence covers the entry BEING REMOVED only: a + # different multi-word entry still stored re-warns from the + # derived instance's __post_init__, since the warning is + # per-construction by design. normalized = _normset(words, name, warn=False) updates[name] = (current | normalized if op == "add" else current - normalized) diff --git a/nameparser/_types.py b/nameparser/_types.py index b04af901..4c818d67 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -30,7 +30,9 @@ class Role(StrEnum): (``as_dict()``, ``comparison_key()``, rendering). A StrEnum, like :class:`AmbiguityKind`: members ARE their string values, so ``token.role == "given"`` compares directly and - ``str(Role.GIVEN) == "given"``.""" + ``str(Role.GIVEN) == "given"``. Members order as strings, so + ``sorted()`` yields alphabetical order -- iterate ``Role`` itself + for the canonical order.""" # Declaration order IS the canonical field order (conventions §3): # every listing of the seven fields anywhere derives from this. @@ -559,6 +561,17 @@ def _with_field_tokens( replacement tokens in at the role's first position (appended in canonical order when the role had no tokens); drop ambiguities whose referents were replaced.""" + # Private contract, made self-enforcing: a token filed under a + # key that is not its own role is the one way this shared tail + # could build a semantically wrong ParsedName (the splice keys + # on the mapping, the views key on the token). + for role, toks in replaced.items(): + for tok in toks: + if tok.role is not role: + raise ValueError( + f"replacement token {tok.text!r} has role " + f"{tok.role.value}, not {role.value}" + ) new_tokens: list[Token] = [] emitted: set[Role] = set() for tok in self.tokens: diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 14259a1b..19eb60f5 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -535,3 +535,24 @@ def test_multiword_warning_points_at_caller_line() -> None: Lexicon.empty().add(titles=["zqx zqy"]) assert len(w) >= 2 assert all(x.filename == __file__ for x in w) + + +def test_add_warns_exactly_once() -> None: + # the warn=False plumbing exists to prevent the double warning + # (_edit's pass + the new instance's __post_init__); count it + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + Lexicon.empty().add(titles=["zqx zqy"]) + assert len([x for x in w if "matched one word" in str(x.message)]) == 1 + + +def test_remove_of_a_stored_dead_entry_is_silent() -> None: + # the scenario the design argues for: user got the warning, now + # removes the dead entry -- silently. (An UNRELATED edit on a + # lexicon still holding one re-warns; that is per-construction + # warning, by design.) + with pytest.warns(UserWarning): + dirty = Lexicon.empty().add(titles=["zqx zqy"]) + with warnings.catch_warnings(): + warnings.simplefilter("error") + dirty.remove(titles=["zqx zqy"]) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 81f53895..152cdaa5 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -7,7 +7,7 @@ from nameparser._policy import ( FAMILY_FIRST, FAMILY_FIRST_GIVEN_LAST, PatronymicRule, ) -from nameparser._types import AmbiguityKind +from nameparser._types import AmbiguityKind, Role def test_parser_defaults_and_properties() -> None: @@ -389,6 +389,7 @@ def test_revise_replace_shared_semantics() -> None: r = p.revise(n, given="José", suffix="") assert r.given == "José" assert r.suffix == "" # empty value clears the field + assert p.revise(n, suffix="()").suffix == "" # punctuation-only too assert r.original == n.original # provenance unchanged assert all(t.span is None for t in r.tokens_for("given")) assert r.title == "Dr." # untouched fields keep spans @@ -425,3 +426,13 @@ def test_revise_sub_parse_structural_behavior() -> None: assert revised.maiden == "" assert p.revise(n, given="J.R. 'Bob'").given == "J.R. Bob" assert p.revise(n, family="Smith (Jones").ambiguities == () + + +def test_revise_forces_the_named_role_on_every_harvested_token() -> None: + # the sub-parse reads "Dr." as a title and "Jr." as a suffix; the + # named field's role must win for every token or the family view + # silently drops them + p = Parser() + r = p.revise(p.parse("John Smith"), family="Dr. Vega Jr.") + assert r.family == "Dr. Vega Jr." + assert all(t.role is Role.FAMILY for t in r.tokens_for(Role.FAMILY)) diff --git a/tests/v2/test_types.py b/tests/v2/test_types.py index cb9fe335..8390fc5e 100644 --- a/tests/v2/test_types.py +++ b/tests/v2/test_types.py @@ -435,3 +435,20 @@ def test_particle_tag_marks_vocabulary_membership_not_role() -> None: # particle read as a given name still carries it (STABLE_TAGS docs) tok = parse("Van Johnson").tokens_for(Role.GIVEN)[0] assert "particle" in tok.tags + + +def test_str_enum_value_sets_are_pairwise_disjoint() -> None: + # three StrEnums cross-compare by value; a future collision would + # make unrelated members equal + from nameparser import AmbiguityKind, PatronymicRule + role = {m.value for m in Role} + kind = {m.value for m in AmbiguityKind} + rule = {m.value for m in PatronymicRule} + assert not (role & kind) and not (role & rule) and not (kind & rule) + + +def test_with_field_tokens_rejects_mismatched_roles() -> None: + name = parse("John Smith") + with pytest.raises(ValueError, match="has role family, not given"): + name._with_field_tokens( + {Role.GIVEN: (Token("x", None, Role.FAMILY),)}) From 20ca3bfebed5a3887794e2ec061b7b6abbf66398 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 12:30:17 -0700 Subject: [PATCH 18/21] Show the replace() degradation and the revise() fix as runnable examples Co-Authored-By: Claude Fable 5 --- docs/customize.rst | 7 ++++++- docs/usage.rst | 38 +++++++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/docs/customize.rst b/docs/customize.rst index 628b6600..940acb7a 100644 --- a/docs/customize.rst +++ b/docs/customize.rst @@ -269,7 +269,12 @@ listed below. To apply a :class:`PolicyPatch ` directly -- without going through a locale pack -- call :meth:`Policy.patched() `: -``Policy().patched(PolicyPatch(middle_as_family=True))``. + +.. doctest:: + + >>> from nameparser import Policy, PolicyPatch + >>> Policy().patched(PolicyPatch(middle_as_family=True)) + Policy(middle_as_family=True) Family-first name order ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/usage.rst b/docs/usage.rst index 641b03f3..47b0a144 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -411,13 +411,37 @@ everything else carried over. >>> name.title '' -``replace()`` splits values on whitespace and its tokens carry no -vocabulary tags, so tag-driven views degrade (``family_particles`` -empties, particles regain their initials). When that matters, use -:meth:`Parser.revise() ` — the same -operation, with each value classified by the parser's vocabulary, -which also means delimiters and marker words in the value are consumed -as they would be in a parse. +``replace()`` splits values on whitespace into plain, untagged +tokens — the vocabulary knowledge a parse would have about the new +text is not there. The views that depend on tags degrade: the parser +no longer knows ``de la`` are particles, so ``family_particles`` +empties and the particles start contributing initials. + +.. doctest:: + + >>> name.family_particles + 'de la' + >>> replaced = name.replace(family="de la Vega Smith") + >>> replaced.family_particles + '' + >>> replaced.initials() + 'J. d. l. V. S.' + +:meth:`Parser.revise() ` is the same +operation with each value classified by the parser's vocabulary, so +the correction behaves like a fresh parse of the corrected name +(which also means delimiters and marker words in the value are +consumed as they would be in a parse): + +.. doctest:: + + >>> from nameparser import Parser + >>> parser = Parser() + >>> revised = parser.revise(name, family="de la Vega Smith") + >>> revised.family_particles + 'de la' + >>> revised.initials() + 'J. V. S.' Command line ------------ From fd43eac9c31634f1a5235c608f59ed01f2463067 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 13:05:28 -0700 Subject: [PATCH 19/21] Cross-reference the Parser convenience trio from the revise() section Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/usage.rst b/docs/usage.rst index 47b0a144..c418e9ae 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -443,6 +443,13 @@ consumed as they would be in a parse): >>> revised.initials() 'J. V. S.' +``revise()`` has two siblings on :class:`~nameparser.Parser`: +:meth:`Parser.matches() ` and +:meth:`Parser.capitalized() `. Those +two matter when you have built a custom parser — the +:class:`~nameparser.ParsedName` methods of the same names fall back +to the *default* configuration for str or omitted arguments. + Command line ------------ From f5c74e73a70ff69f3475e99df88950292ed78e1c Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 13:16:20 -0700 Subject: [PATCH 20/21] Teach the particle ambiguity with the Van Johnson / Van Buren pair Co-Authored-By: Claude Fable 5 --- docs/concepts.rst | 8 ++++---- docs/usage.rst | 14 ++++++++------ nameparser/_lexicon.py | 3 ++- nameparser/_types.py | 7 ++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 7704a251..fd575468 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -150,10 +150,10 @@ Some calls are irreducibly ambiguous — both readings are legitimate, and no amount of rule-tuning resolves them without breaking some other name. Those surface as entries on ``ParsedName.ambiguities`` instead of being silently guessed away. The canonical example: a leading "Van" -in "Van Johnson" reads as a given name (that's the common case for -that shape), but "Van" is also a family-name particle in plenty of -other names, so the parse records a ``particle-or-given`` ambiguity -alongside its answer. You can inspect ``ambiguities`` to decide, case +reads as a given name — the right call for the actor Van Johnson, the +wrong one for a bare "Van Buren", and nothing in the two-word shape +distinguishes them — so the parse records a ``particle-or-given`` +ambiguity alongside its answer. You can inspect ``ambiguities`` to decide, case by case, whether your data needs a second look. An ambiguity records a *decision*, not a word. The same token in a diff --git a/docs/usage.rst b/docs/usage.rst index c418e9ae..0031407b 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -317,16 +317,18 @@ the DEFAULT parser; to compare against a string using a custom When the parser had to guess ----------------------------- -Some names have no single correct reading. ``"Van Johnson"`` could be -the given name ``Van``, or the family-name particle ``van``. 2.0 takes -the more likely reading and *records* the choice on ``ambiguities`` -rather than deciding silently: +Some names have no single correct reading. A leading ``Van`` could be +a given name — it really is for the actor Van Johnson — or the start +of a family name, as it is for President Van Buren. Both are the same +shape, so no rule can tell them apart. 2.0 takes the more likely +reading and *records* the choice on ``ambiguities`` rather than +deciding silently: .. doctest:: - >>> name = parse("Van Johnson") + >>> name = parse("Van Buren") >>> name.given, name.family - ('Van', 'Johnson') + ('Van', 'Buren') >>> for a in name.ambiguities: ... print(a.kind.value, "-", a.detail) particle-or-given - leading 'Van' may be a family-name particle; read as a given name diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 6c025008..cde2c75a 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -308,7 +308,8 @@ class Lexicon: particles: frozenset[str] = frozenset() #: Subset of particles that can also BE a given name: a leading #: one reads as given and records a particle-or-given ambiguity - #: ("Van Johnson"). No constant of its own -- the default derives + #: ("Van Johnson", but also "Van Buren"). No constant of its own + #: -- the default derives #: as particles minus #: :data:`~nameparser.config.prefixes.NON_FIRST_NAME_PREFIXES` #: (which marks the opposite, never-given subset). diff --git a/nameparser/_types.py b/nameparser/_types.py index 4c818d67..ff2346a1 100644 --- a/nameparser/_types.py +++ b/nameparser/_types.py @@ -262,9 +262,10 @@ class AmbiguityKind(StrEnum): #: Smith B"). Which name part was declined depends on position and #: ``name_order``, so ``detail`` names it rather than the kind. SUFFIX_OR_NAME = "suffix-or-name" - #: A leading ambiguous particle was read as a given name -- "Van - #: Johnson" parses given="Van", but "Van" is also a family-name - #: particle in other names. + #: A leading ambiguous particle was read as a given name -- the + #: right call for "Van Johnson" (the actor's given name), the + #: wrong one for a bare "Van Buren" (the presidential surname); + #: the two-word shape cannot distinguish them. PARTICLE_OR_GIVEN = "particle-or-given" #: A nickname/maiden delimiter opened without closing (or closed #: without opening); the text was kept as literal name content, so From 9d42aaa4802e1a5c10b8f705b0fa7d2f42935478 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Sun, 26 Jul 2026 13:24:28 -0700 Subject: [PATCH 21/21] Say what replace() actually carries over, and what it deliberately drops Co-Authored-By: Claude Fable 5 --- docs/usage.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 0031407b..ef06085a 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -401,8 +401,11 @@ Correcting a parse -------------------- :class:`~nameparser.ParsedName` is immutable, so a correction is a new -value: ``replace()`` returns a copy with the given fields changed and -everything else carried over. +value: ``replace()`` returns a copy with the given fields changed. +Untouched fields keep their tokens (and ``original`` is preserved), +with one deliberate exception: an ambiguity that pointed into a +replaced field is dropped — correcting the field that was flagged +clears the flag, while correcting an unrelated field keeps it. .. doctest:: @@ -412,6 +415,11 @@ everything else carried over. 'Dr. Juan de la Vega' >>> name.title '' + >>> flagged = parse("Van Buren") + >>> flagged.replace(given="Martin").ambiguities + () + >>> [a.kind.value for a in flagged.replace(family="Harrison").ambiguities] + ['particle-or-given'] ``replace()`` splits values on whitespace into plain, untagged tokens — the vocabulary knowledge a parse would have about the new