Skip to content

Split fix(suffix-routing) into the rules that name its shapes (#372) #376

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
derek73 merged 2 commits into from
Aug 14, 2026
+116 −13
Merged

Split fix(suffix-routing) into the rules that name its shapes (#372) #376

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
49bd8c3
Stop fix(suffix-routing) explaining what it cannot name
derek73 Aug 13, 2026
0125804
Give the glued honorifics the rule I said was impossible
derek73 Aug 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 45 additions & 3 deletions tests/v2/test_ledger_guards.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,8 @@ def test_cjk_corpus_matches_the_case_table() -> None:
#: exactly one entry. The full issue lists are the keys, not a bare
#: '#308': both 2.0 rules cite #308 while copying different constants.
_HONORIFIC_SOURCES: dict[str, frozenset[str]] = {
"cjk-honorific-suffix": SUFFIX_WORDS, # 1.4
"cjk-honorific-suffix": SUFFIX_WORDS, # 1.4, spaced
"cjk-glued-honorific-peel": GLUED_HONORIFICS, # 1.4, glued (#372)
"#307/#308/#320": SUFFIX_WORDS, # 2.0, spaced
"#308/#312/#319/#320": GLUED_HONORIFICS, # 2.0, glued
}
Expand Down Expand Up @@ -1017,6 +1018,8 @@ def _claim(rule: dict) -> _Claim:
_Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"),
"fix(comma-family) lone post-comma piece routes to suffix/title, not first":
_Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"),
"fix(comma-precomma-family) pre-comma run reads as family, not given":
_Claim(215, ('family', 'given'), "f16a0e79cba3"),
"fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix":
_Claim(751, ('family', 'given', 'suffix'), "231640fc7535"),
"fix(suffix-delimiter-rendering) no-space delimiter core token kept whole":
Expand All @@ -1035,8 +1038,10 @@ def _claim(rule: dict) -> _Claim:
_Claim(20, ('given', 'suffix'), "b2ea8fa59eea"),
"fix(cjk-comma-compound) comma routing compounds with the CJK order flip":
_Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"),
"fix(cjk-glued-honorific-peel) glued honorific peels into suffix":
_Claim(34, ('family', 'given', 'suffix'), "877ab3246d33"),
"fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip":
_Claim(14, ('family', 'given', 'middle', 'suffix'), "d49ce901bdce"),
_Claim(19, ('family', 'given', 'middle', 'suffix'), "aa475ddd4745"),
"feat(#269) non-Latin titles/conjunctions recognized":
_Claim(2, ('given', 'middle', 'title'), "c14187bb08f8"),
"fix(leading-credential) a split 'Ph. D.' before the name stays one unit":
Expand Down Expand Up @@ -1166,6 +1171,35 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None:
("田中さん, PhD", ("family", "given", "suffix")):
"fix(cjk-comma-compound)",
("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)",
# #372's suffix-routing split. 'Bob Jones, author' moves NO
# suffix, which is what disqualifies it from the routing rule;
# 'Smith Jr.' is the Latin shape that rule is named for; the two
# glued honorifics are the majority it also legitimately takes,
# and cannot be given a rule of their own -- '김민준씨' and the
# given name '김지양' are the same string shape.
("Bob Jones, author", ("family", "given")):
"fix(comma-precomma-family)",
("MD, PHD", ("family", "given")): "fix(comma-precomma-family)",
("Smith Jr.", ("family", "suffix")): "fix(suffix-routing)",
# the glued/spaced boundary. 'Andersonさん' and '김민준씨' left
# suffix-routing for a rule that names them; '김민준 씨.' is
# spaced and stays on the spaced rule, which #372 taught to
# tolerate the trailing period.
#
# '김지양' is why the glued rule copies GLUED_HONORIFICS and not
# SUFFIX_WORDS: 양 is absent from the glued set, so no rule
# NAMED for honorifics can claim a suffix diff on a given name
# that merely ends in one. The fields-only fix(suffix-routing)
# still would -- measured -- which is unchanged by #372 and is
# the residual cost of having a last-resort tier at all. Being
# absorbed by the catch-all is recoverable; being labelled
# 'recognized honorific' by a specific rule is not.
("Andersonさん", ("given", "suffix")):
"fix(cjk-glued-honorific-peel)",
("김민준씨", ("family", "given", "suffix")):
"fix(cjk-glued-honorific-peel)",
("김민준 씨.", ("family", "given", "suffix")):
"fix(cjk-honorific-suffix)",
},
}

Expand Down Expand Up @@ -1252,7 +1286,15 @@ class _Excluded(NamedTuple):
_EXCLUSION_EFFECT: dict[str, _Excluded] = {
"(?i)^[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$":
_Excluded(3, "5a12a8117651",
("fix(comma-family)", "fix(suffix-routing)")),
# fix(comma-precomma-family) JOINED this tuple in #372,
# it did not replace anything: it claims the {given,
# family} readings, which it legitimately describes for a
# Latin comma name, while fix(suffix-routing) still
# claims the readings outside its two fields. The
# exclusion refuses the name before any of the three is
# reached; this records what would happen without it.
("fix(comma-family)", "fix(comma-precomma-family)",
"fix(suffix-routing)")),
'(^|[\\w.]\\s+)[("\'][^)"\']+[)"\'](\\s+\\w|\\s*$)':
_Excluded(34, "9ea55c4c4382", ()),
}
Expand Down
81 changes: 71 additions & 10 deletions tools/differential/expected_since_1.4.0.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,17 +159,44 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir
name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$"
fields = ["given", "title", "suffix"]

[[change]]
issue = "fix(comma-precomma-family) pre-comma run reads as family, not given"
# 'Bob Jones, author' / 'MD, PHD': the post-comma piece is a title on
# BOTH sides -- it does not move, and no suffix moves either. What
# changes is the pre-comma run: 1.4 read it as `first`, 2.x reads it
# as `last`, because pre-comma is definitionally family. Diff is
# exactly {given, family}.
#
# Its own rule because these have nothing to do with suffix routing:
# they were falling to the fields-only fix(suffix-routing) below, on a
# rule whose every other name moves a trailing token into `suffix`,
# while these move no suffix at all (#372).
#
# Distinct from fix(comma-family) above by fields, not by order: that
# rule is the lone post-comma piece moving into suffix/title and its
# fields exclude `family`, which every name here moves. Latin-anchored
# for the same reason it is -- a bare comma reaches every script.
name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$"
fields = ["given", "family"]

[[change]]
issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix"
# 'Johnson PhD' / 'Mr. Johnson PhD': v1 routed a lone trailing suffix
# to family/first (no comma present); 2.0 keeps recognized suffixes in
# `suffix`.
# #312: also claims comma-less glued-honorific names like
# '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any
# glued Latin honorific. Its comma-bearing siblings never reach here:
# they split by whether the diff moves `family` --
# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel)
# when it does not -- both above, both keyed on a comma.
# 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone
# trailing suffix to family/first (no comma present); 2.0 keeps
# recognized suffixes in `suffix`. Four corpus names, and the prose
# fits all four.
#
# It claimed 25 until #372, on nothing but being the only fields-only
# rule in any ledger: it sorts after every name_regex rule, so it
# takes whatever nothing narrower named. The other 21 have their own
# rules now -- fix(cjk-glued-honorific-peel) and the widened
# fix(cjk-honorific-suffix) below for #312's honorifics,
# fix(comma-precomma-family) above for the three names that move no
# suffix at all.
#
# Being last is the point of this rule and worth keeping. What is not
# worth keeping is it being the only explanation a name ever gets:
# when it grows, the question is which narrower rule is missing.
fields = ["given", "family", "suffix"]

[[change]]
Expand Down Expand Up @@ -349,6 +376,40 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip
name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])"
fields = ["given", "middle", "family", "title", "suffix"]

[[change]]
issue = "fix(cjk-glued-honorific-peel) glued honorific peels into suffix"
# '김민준씨' -> 김/민준/씨, '田中さん' -> 田中/さん, 'Andersonさん' ->
# Anderson/さん: #312's peel, on a name with no space before the
# honorific and no comma anywhere. 17 corpus names, which fell to the
# fields-only fix(suffix-routing) above until #372 gave them a rule
# that names them.
#
# The regex is this file's first hand copy of GLUED_HONORIFICS rather
# than SUFFIX_WORDS, and that narrower vocabulary is the whole reason
# this rule can exist. '김민준씨' and the given name '김지양' are the
# same string shape -- hangul ending in a character the spaced list
# calls an honorific -- so a rule keyed on SUFFIX_WORDS could not tell
# them apart, and unanchoring the spaced rule below to reach the glued
# forms would let a future suffix regression on '김지양' classify as a
# recognized honorific. GLUED_HONORIFICS settles it in the config
# instead: 양, 군, 氏, 殿 and 博士 are deliberately absent from it,
# because a glued match on those collides with real given names. Same
# copy, same pin (_HONORIFIC_SOURCES) as the 2.0 ledger's twin, whose
# regex this is verbatim.
#
# `\.?(?=$|[ ,])` rather than `$`: the honorific ends a TOKEN, not
# necessarily the string. '田中さん II' and '김민준씨 Jr.' carry a
# Latin suffix after it, '김민준씨 (Jimmy)' a nickname.
#
# One spaced name lands here and is left that way, exactly as the 2.0
# twin records: '김민준 박사님' matches on its 님, glued to 사 inside
# 박사님. The label stays true of it -- the peel is what routes it to
# `suffix` -- and closing the overlap needs a lookbehind asserting the
# match is not interior to a longer listed honorific, which is more
# machinery than a correct label is worth.
name_regex = "(?<=[^\\s,])(?:박사님|선생님|교수님|박사|씨|님|先生|女士|小姐|教授|様|さん|さま|くん|ちゃん)\\.?(?=$|[ ,])"
fields = ["given", "family", "suffix"]

[[change]]
issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip"
# '王小明 先生', '김민준 씨', '田中 太郎 様': #307 ships the spaced CJK
Expand Down Expand Up @@ -380,7 +441,7 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin
# 2026-07-30 run, not assumed: #308's peel put a dozen glued names
# in the corpus (Latin+glued like Andersonさん included) and every
# one classified there -- none carried a middle-field diff.
name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$"
name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)\\.?$"
fields = ["given", "middle", "family", "suffix"]

[[change]]
Expand Down