diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 0d651b1..dc8d7d2 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -531,7 +531,8 @@ def test_cjk_corpus_matches_the_case_table() -> None: #: exactly one entry. The full issue lists are the keys, not a bare #: '#308': both 2.0 rules cite #308 while copying different constants. _HONORIFIC_SOURCES: dict[str, frozenset[str]] = { - "cjk-honorific-suffix": SUFFIX_WORDS, # 1.4 + "cjk-honorific-suffix": SUFFIX_WORDS, # 1.4, spaced + "cjk-glued-honorific-peel": GLUED_HONORIFICS, # 1.4, glued (#372) "#307/#308/#320": SUFFIX_WORDS, # 2.0, spaced "#308/#312/#319/#320": GLUED_HONORIFICS, # 2.0, glued } @@ -1017,6 +1018,8 @@ def _claim(rule: dict) -> _Claim: _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), + "fix(comma-precomma-family) pre-comma run reads as family, not given": + _Claim(215, ('family', 'given'), "f16a0e79cba3"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -1035,8 +1038,10 @@ def _claim(rule: dict) -> _Claim: _Claim(20, ('given', 'suffix'), "b2ea8fa59eea"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"), + "fix(cjk-glued-honorific-peel) glued honorific peels into suffix": + _Claim(34, ('family', 'given', 'suffix'), "877ab3246d33"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": - _Claim(14, ('family', 'given', 'middle', 'suffix'), "d49ce901bdce"), + _Claim(19, ('family', 'given', 'middle', 'suffix'), "aa475ddd4745"), "feat(#269) non-Latin titles/conjunctions recognized": _Claim(2, ('given', 'middle', 'title'), "c14187bb08f8"), "fix(leading-credential) a split 'Ph. D.' before the name stays one unit": @@ -1166,6 +1171,35 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: ("田中さん, PhD", ("family", "given", "suffix")): "fix(cjk-comma-compound)", ("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)", + # #372's suffix-routing split. 'Bob Jones, author' moves NO + # suffix, which is what disqualifies it from the routing rule; + # 'Smith Jr.' is the Latin shape that rule is named for; the two + # glued honorifics are the majority it also legitimately takes, + # and cannot be given a rule of their own -- '김민준씨' and the + # given name '김지양' are the same string shape. + ("Bob Jones, author", ("family", "given")): + "fix(comma-precomma-family)", + ("MD, PHD", ("family", "given")): "fix(comma-precomma-family)", + ("Smith Jr.", ("family", "suffix")): "fix(suffix-routing)", + # the glued/spaced boundary. 'Andersonさん' and '김민준씨' left + # suffix-routing for a rule that names them; '김민준 씨.' is + # spaced and stays on the spaced rule, which #372 taught to + # tolerate the trailing period. + # + # '김지양' is why the glued rule copies GLUED_HONORIFICS and not + # SUFFIX_WORDS: 양 is absent from the glued set, so no rule + # NAMED for honorifics can claim a suffix diff on a given name + # that merely ends in one. The fields-only fix(suffix-routing) + # still would -- measured -- which is unchanged by #372 and is + # the residual cost of having a last-resort tier at all. Being + # absorbed by the catch-all is recoverable; being labelled + # 'recognized honorific' by a specific rule is not. + ("Andersonさん", ("given", "suffix")): + "fix(cjk-glued-honorific-peel)", + ("김민준씨", ("family", "given", "suffix")): + "fix(cjk-glued-honorific-peel)", + ("김민준 씨.", ("family", "given", "suffix")): + "fix(cjk-honorific-suffix)", }, } @@ -1252,7 +1286,15 @@ class _Excluded(NamedTuple): _EXCLUSION_EFFECT: dict[str, _Excluded] = { "(?i)^[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$": _Excluded(3, "5a12a8117651", - ("fix(comma-family)", "fix(suffix-routing)")), + # fix(comma-precomma-family) JOINED this tuple in #372, + # it did not replace anything: it claims the {given, + # family} readings, which it legitimately describes for a + # Latin comma name, while fix(suffix-routing) still + # claims the readings outside its two fields. The + # exclusion refuses the name before any of the three is + # reached; this records what would happen without it. + ("fix(comma-family)", "fix(comma-precomma-family)", + "fix(suffix-routing)")), '(^|[\\w.]\\s+)[("\'][^)"\']+[)"\'](\\s+\\w|\\s*$)': _Excluded(34, "9ea55c4c4382", ()), } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 80cfed7..407c034 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -159,17 +159,44 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "title", "suffix"] +[[change]] +issue = "fix(comma-precomma-family) pre-comma run reads as family, not given" +# 'Bob Jones, author' / 'MD, PHD': the post-comma piece is a title on +# BOTH sides -- it does not move, and no suffix moves either. What +# changes is the pre-comma run: 1.4 read it as `first`, 2.x reads it +# as `last`, because pre-comma is definitionally family. Diff is +# exactly {given, family}. +# +# Its own rule because these have nothing to do with suffix routing: +# they were falling to the fields-only fix(suffix-routing) below, on a +# rule whose every other name moves a trailing token into `suffix`, +# while these move no suffix at all (#372). +# +# Distinct from fix(comma-family) above by fields, not by order: that +# rule is the lone post-comma piece moving into suffix/title and its +# fields exclude `family`, which every name here moves. Latin-anchored +# for the same reason it is -- a bare comma reaches every script. +name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" +fields = ["given", "family"] + [[change]] issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" -# 'Johnson PhD' / 'Mr. Johnson PhD': v1 routed a lone trailing suffix -# to family/first (no comma present); 2.0 keeps recognized suffixes in -# `suffix`. -# #312: also claims comma-less glued-honorific names like -# '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any -# glued Latin honorific. Its comma-bearing siblings never reach here: -# they split by whether the diff moves `family` -- -# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel) -# when it does not -- both above, both keyed on a comma. +# 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone +# trailing suffix to family/first (no comma present); 2.0 keeps +# recognized suffixes in `suffix`. Four corpus names, and the prose +# fits all four. +# +# It claimed 25 until #372, on nothing but being the only fields-only +# rule in any ledger: it sorts after every name_regex rule, so it +# takes whatever nothing narrower named. The other 21 have their own +# rules now -- fix(cjk-glued-honorific-peel) and the widened +# fix(cjk-honorific-suffix) below for #312's honorifics, +# fix(comma-precomma-family) above for the three names that move no +# suffix at all. +# +# Being last is the point of this rule and worth keeping. What is not +# worth keeping is it being the only explanation a name ever gets: +# when it grows, the question is which narrower rule is missing. fields = ["given", "family", "suffix"] [[change]] @@ -349,6 +376,40 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "title", "suffix"] +[[change]] +issue = "fix(cjk-glued-honorific-peel) glued honorific peels into suffix" +# '김민준씨' -> 김/민준/씨, '田中さん' -> 田中/さん, 'Andersonさん' -> +# Anderson/さん: #312's peel, on a name with no space before the +# honorific and no comma anywhere. 17 corpus names, which fell to the +# fields-only fix(suffix-routing) above until #372 gave them a rule +# that names them. +# +# The regex is this file's first hand copy of GLUED_HONORIFICS rather +# than SUFFIX_WORDS, and that narrower vocabulary is the whole reason +# this rule can exist. '김민준씨' and the given name '김지양' are the +# same string shape -- hangul ending in a character the spaced list +# calls an honorific -- so a rule keyed on SUFFIX_WORDS could not tell +# them apart, and unanchoring the spaced rule below to reach the glued +# forms would let a future suffix regression on '김지양' classify as a +# recognized honorific. GLUED_HONORIFICS settles it in the config +# instead: 양, 군, 氏, 殿 and 博士 are deliberately absent from it, +# because a glued match on those collides with real given names. Same +# copy, same pin (_HONORIFIC_SOURCES) as the 2.0 ledger's twin, whose +# regex this is verbatim. +# +# `\.?(?=$|[ ,])` rather than `$`: the honorific ends a TOKEN, not +# necessarily the string. '田中さん II' and '김민준씨 Jr.' carry a +# Latin suffix after it, '김민준씨 (Jimmy)' a nickname. +# +# One spaced name lands here and is left that way, exactly as the 2.0 +# twin records: '김민준 박사님' matches on its 님, glued to 사 inside +# 박사님. The label stays true of it -- the peel is what routes it to +# `suffix` -- and closing the overlap needs a lookbehind asserting the +# match is not interior to a longer listed honorific, which is more +# machinery than a correct label is worth. +name_regex = "(?<=[^\\s,])(?:박사님|선생님|교수님|박사|씨|님|先生|女士|小姐|教授|様|さん|さま|くん|ちゃん)\\.?(?=$|[ ,])" +fields = ["given", "family", "suffix"] + [[change]] issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip" # '王小明 先生', '김민준 씨', '田中 太郎 様': #307 ships the spaced CJK @@ -380,7 +441,7 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # 2026-07-30 run, not assumed: #308's peel put a dozen glued names # in the corpus (Latin+glued like Andersonさん included) and every # one classified there -- none carried a middle-field diff. -name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" +name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)\\.?$" fields = ["given", "middle", "family", "suffix"] [[change]]