From 49bd8c3a5cb1da5cb7de65f1c6f4386ed5e956a3 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 12 Aug 2026 21:31:22 -0700 Subject: [PATCH 1/2] Stop fix(suffix-routing) explaining what it cannot name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It claimed 25 names in four shapes; its prose -- "two-token name with unambiguous trailing suffix stays suffix" -- described four of them. Three of the 25 move no suffix at all. 'Bob Jones, author' and 'MD, PHD' keep an identical title on both sides and move the pre-comma run from `first` to `last`, because pre-comma is definitionally family. Diff is exactly {given, family}. They now have their own rule, fix(comma-precomma-family), separated from fix(comma-family) by fields rather than order -- that rule's exclude `family`, which every name here moves -- and Latin-anchored for the same reason it is. The other 18 are #312's comma-less glued honorifics, and they stay. They cannot be given a name_regex rule: '김민준씨' and the given name '김지양' are the same string shape, hangul ending in a character that is also an honorific, and only vocabulary plus segmentation tells them apart. That is why fix(cjk-honorific-suffix) is anchored to a whole trailing token. Unanchoring that rule to reach the glued forms was the first design here, and measuring it is what rejected it: it would make a future suffix regression on '김지양' classify as a recognized honorific. A specific rule confidently claiming a name it does not describe is worse than this rule's honest breadth, and the differential run cannot see the difference -- 108/0 either way -- because nothing is being absorbed yet. So the issue text becomes what all 22 remaining names share: a trailing token routes to suffix. The Latin two-token case is four of them; the comment now says the glued honorifics are the majority, and why this is the last-resort tier rather than a gap left for later. Gate unmoved: 108/0 at 1.4.0, 90/0 at 2.0.0, 1/0 at 2.1.0. Three rosters recorded. _CORPUS_CLAIMS gains the new rule and rekeys suffix-routing on its new issue text. _EXCLUSION_EFFECT's Ph. D. entry gains fix(comma-precomma-family) in absorbed_by -- it JOINED, it did not replace fix(suffix-routing), which still claims the readings outside the new rule's two fields. _CROSS_RULE_WINNERS gains the five names that now sit on either side of the split; measured, narrowing the new rule so it can no longer claim them fails that pin. Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 26 ++++++++- tools/differential/expected_since_1.4.0.toml | 57 ++++++++++++++++---- 2 files changed, 71 insertions(+), 12 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 0d651b1..ab39f3b 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1017,7 +1017,9 @@ def _claim(rule: dict) -> _Claim: _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), - "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": + "fix(comma-precomma-family) pre-comma run reads as family, not given": + _Claim(215, ('family', 'given'), "f16a0e79cba3"), + "fix(suffix-routing) a trailing token routes to suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": _Claim(0, ('suffix',), "e3b0c44298fc"), @@ -1166,6 +1168,18 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: ("田中さん, PhD", ("family", "given", "suffix")): "fix(cjk-comma-compound)", ("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)", + # #372's suffix-routing split. 'Bob Jones, author' moves NO + # suffix, which is what disqualifies it from the routing rule; + # 'Smith Jr.' is the Latin shape that rule is named for; the two + # glued honorifics are the majority it also legitimately takes, + # and cannot be given a rule of their own -- '김민준씨' and the + # given name '김지양' are the same string shape. + ("Bob Jones, author", ("family", "given")): + "fix(comma-precomma-family)", + ("MD, PHD", ("family", "given")): "fix(comma-precomma-family)", + ("Smith Jr.", ("family", "suffix")): "fix(suffix-routing)", + ("Andersonさん", ("given", "suffix")): "fix(suffix-routing)", + ("김민준씨", ("family", "given", "suffix")): "fix(suffix-routing)", }, } @@ -1252,7 +1266,15 @@ class _Excluded(NamedTuple): _EXCLUSION_EFFECT: dict[str, _Excluded] = { "(?i)^[\\u0000-\\u024f]*\\bph\\.\\s*d\\.\\s*$": _Excluded(3, "5a12a8117651", - ("fix(comma-family)", "fix(suffix-routing)")), + # fix(comma-precomma-family) JOINED this tuple in #372, + # it did not replace anything: it claims the {given, + # family} readings, which it legitimately describes for a + # Latin comma name, while fix(suffix-routing) still + # claims the readings outside its two fields. The + # exclusion refuses the name before any of the three is + # reached; this records what would happen without it. + ("fix(comma-family)", "fix(comma-precomma-family)", + "fix(suffix-routing)")), '(^|[\\w.]\\s+)[("\'][^)"\']+[)"\'](\\s+\\w|\\s*$)': _Excluded(34, "9ea55c4c4382", ()), } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 80cfed7..7fe0446 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -160,16 +160,53 @@ name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "title", "suffix"] [[change]] -issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" -# 'Johnson PhD' / 'Mr. Johnson PhD': v1 routed a lone trailing suffix -# to family/first (no comma present); 2.0 keeps recognized suffixes in -# `suffix`. -# #312: also claims comma-less glued-honorific names like -# '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any -# glued Latin honorific. Its comma-bearing siblings never reach here: -# they split by whether the diff moves `family` -- -# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel) -# when it does not -- both above, both keyed on a comma. +issue = "fix(comma-precomma-family) pre-comma run reads as family, not given" +# 'Bob Jones, author' / 'MD, PHD': the post-comma piece is a title on +# BOTH sides -- it does not move, and no suffix moves either. What +# changes is the pre-comma run: 1.4 read it as `first`, 2.x reads it +# as `last`, because pre-comma is definitionally family. Diff is +# exactly {given, family}. +# +# Its own rule because these have nothing to do with suffix routing: +# they were falling to the fields-only fix(suffix-routing) below, on a +# rule whose every other name moves a trailing token into `suffix`, +# while these move no suffix at all (#372). +# +# Distinct from fix(comma-family) above by fields, not by order: that +# rule is the lone post-comma piece moving into suffix/title and its +# fields exclude `family`, which every name here moves. Latin-anchored +# for the same reason it is -- a bare comma reaches every script. +name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" +fields = ["given", "family"] + +[[change]] +issue = "fix(suffix-routing) a trailing token routes to suffix" +# 'Johnson PhD' / 'Smith Jr.' / 'John V': v1 routed a lone trailing +# suffix to family/first (no comma present); 2.0 keeps recognized +# suffixes in `suffix`. +# +# The issue text says "a trailing token", not "a two-token Latin +# name", because #312's comma-less glued honorifics land here too and +# they are the majority: '田中さん' -> 田中/さん, 'Andersonさん' -> +# Anderson/さん, '김민준씨' -> 김/민준/씨. Every name this rule claims +# moves a trailing token into `suffix`; the Latin two-token case is +# four of them. +# +# That is deliberate, not a gap left for later. The glued honorifics +# CANNOT be given a name_regex rule of their own: '김민준씨' and the +# given name '김지양' are the same string shape -- hangul ending in a +# character that is also an honorific -- and only vocabulary plus +# segmentation tells them apart, neither of which a regex over the +# name string can reach. fix(cjk-honorific-suffix) below is anchored +# to a whole trailing token for exactly that reason, and unanchoring +# it to reach the glued forms was measured (#372) to make a future +# suffix regression on '김지양' classify as a recognized honorific -- +# a specific rule confidently claiming a name it does not describe, +# which is worse than this rule's honest breadth. +# +# So this is the last-resort tier and says so: it is the only +# fields-only rule in any ledger, it sorts after every name_regex +# rule, and it takes what nothing narrower could name. fields = ["given", "family", "suffix"] [[change]] From 0125804f231d61b5245e945494cf101d928bcacc Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Thu, 13 Aug 2026 01:48:24 -0700 Subject: [PATCH 2/2] Give the glued honorifics the rule I said was impossible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit claimed 18 names could not have a name_regex rule -- that '김민준씨' and the given name '김지양' are the same string shape and only vocabulary plus segmentation separates them, neither reachable by a regex. Review falsified it, and the answer was in the config: GLUED_HONORIFICS deliberately omits 양, 군, 氏, 殿 and 博士, exactly the characters whose glued match collides with real given names. The parser had already solved this by curating a narrower vocabulary; a regex reaches vocabulary by hand-copying it, which this ledger does routinely and pins with a sync test. The 2.0.0 ledger has had such a rule since #308. This adds its twin to the 1.4 ledger, regex verbatim, enrolled in _HONORIFIC_SOURCES against the same GLUED_HONORIFICS source. 17 names. The other two were never glued: '김민준 씨.' and '김민준 양.' are spaced, and the spaced rule missed them only because its `$` would not tolerate a trailing period. Relaxed to `\.?$`. 양 stays safe there -- it is in the SPACED vocabulary, where a whole-token anchor already separates it from 지양. So fix(suffix-routing) is back to the four Latin names its original prose describes, and that prose is restored. It claimed 25 not because it was too broad but because it is the only fields-only rule and sorts last: when it grows, the question is which narrower rule is missing. Measured, all shapes accounted for and the gate unmoved at 108/0, 90/0, 1/0: fix(cjk-glued-honorific-peel) 17 new fix(cjk-honorific-suffix) 11 was 10, +2 periods, -1 to glued fix(comma-precomma-family) 3 unchanged from the last commit fix(suffix-routing) 4 was 25 Three rosters fired and were recorded: _HONORIFIC_SOURCES (the new vocabulary copy), _CORPUS_CLAIMS, _CROSS_RULE_WINNERS. One claim in this commit's first draft was also wrong and is corrected here rather than shipped: a suffix regression on '김지양' is still absorbed, by the fields-only catch-all. What #372 buys is that no rule NAMED for honorifics will claim it. Being taken by the catch-all is recoverable; being labelled 'recognized honorific' is not. Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 30 ++++++-- tools/differential/expected_since_1.4.0.toml | 80 +++++++++++++------- 2 files changed, 77 insertions(+), 33 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index ab39f3b..dc8d7d2 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -531,7 +531,8 @@ def test_cjk_corpus_matches_the_case_table() -> None: #: exactly one entry. The full issue lists are the keys, not a bare #: '#308': both 2.0 rules cite #308 while copying different constants. _HONORIFIC_SOURCES: dict[str, frozenset[str]] = { - "cjk-honorific-suffix": SUFFIX_WORDS, # 1.4 + "cjk-honorific-suffix": SUFFIX_WORDS, # 1.4, spaced + "cjk-glued-honorific-peel": GLUED_HONORIFICS, # 1.4, glued (#372) "#307/#308/#320": SUFFIX_WORDS, # 2.0, spaced "#308/#312/#319/#320": GLUED_HONORIFICS, # 2.0, glued } @@ -1019,7 +1020,7 @@ def _claim(rule: dict) -> _Claim: _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), "fix(comma-precomma-family) pre-comma run reads as family, not given": _Claim(215, ('family', 'given'), "f16a0e79cba3"), - "fix(suffix-routing) a trailing token routes to suffix": + "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": _Claim(0, ('suffix',), "e3b0c44298fc"), @@ -1037,8 +1038,10 @@ def _claim(rule: dict) -> _Claim: _Claim(20, ('given', 'suffix'), "b2ea8fa59eea"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"), + "fix(cjk-glued-honorific-peel) glued honorific peels into suffix": + _Claim(34, ('family', 'given', 'suffix'), "877ab3246d33"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": - _Claim(14, ('family', 'given', 'middle', 'suffix'), "d49ce901bdce"), + _Claim(19, ('family', 'given', 'middle', 'suffix'), "aa475ddd4745"), "feat(#269) non-Latin titles/conjunctions recognized": _Claim(2, ('given', 'middle', 'title'), "c14187bb08f8"), "fix(leading-credential) a split 'Ph. D.' before the name stays one unit": @@ -1178,8 +1181,25 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: "fix(comma-precomma-family)", ("MD, PHD", ("family", "given")): "fix(comma-precomma-family)", ("Smith Jr.", ("family", "suffix")): "fix(suffix-routing)", - ("Andersonさん", ("given", "suffix")): "fix(suffix-routing)", - ("김민준씨", ("family", "given", "suffix")): "fix(suffix-routing)", + # the glued/spaced boundary. 'Andersonさん' and '김민준씨' left + # suffix-routing for a rule that names them; '김민준 씨.' is + # spaced and stays on the spaced rule, which #372 taught to + # tolerate the trailing period. + # + # '김지양' is why the glued rule copies GLUED_HONORIFICS and not + # SUFFIX_WORDS: 양 is absent from the glued set, so no rule + # NAMED for honorifics can claim a suffix diff on a given name + # that merely ends in one. The fields-only fix(suffix-routing) + # still would -- measured -- which is unchanged by #372 and is + # the residual cost of having a last-resort tier at all. Being + # absorbed by the catch-all is recoverable; being labelled + # 'recognized honorific' by a specific rule is not. + ("Andersonさん", ("given", "suffix")): + "fix(cjk-glued-honorific-peel)", + ("김민준씨", ("family", "given", "suffix")): + "fix(cjk-glued-honorific-peel)", + ("김민준 씨.", ("family", "given", "suffix")): + "fix(cjk-honorific-suffix)", }, } diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 7fe0446..407c034 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -180,33 +180,23 @@ name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "family"] [[change]] -issue = "fix(suffix-routing) a trailing token routes to suffix" -# 'Johnson PhD' / 'Smith Jr.' / 'John V': v1 routed a lone trailing -# suffix to family/first (no comma present); 2.0 keeps recognized -# suffixes in `suffix`. -# -# The issue text says "a trailing token", not "a two-token Latin -# name", because #312's comma-less glued honorifics land here too and -# they are the majority: '田中さん' -> 田中/さん, 'Andersonさん' -> -# Anderson/さん, '김민준씨' -> 김/민준/씨. Every name this rule claims -# moves a trailing token into `suffix`; the Latin two-token case is -# four of them. -# -# That is deliberate, not a gap left for later. The glued honorifics -# CANNOT be given a name_regex rule of their own: '김민준씨' and the -# given name '김지양' are the same string shape -- hangul ending in a -# character that is also an honorific -- and only vocabulary plus -# segmentation tells them apart, neither of which a regex over the -# name string can reach. fix(cjk-honorific-suffix) below is anchored -# to a whole trailing token for exactly that reason, and unanchoring -# it to reach the glued forms was measured (#372) to make a future -# suffix regression on '김지양' classify as a recognized honorific -- -# a specific rule confidently claiming a name it does not describe, -# which is worse than this rule's honest breadth. -# -# So this is the last-resort tier and says so: it is the only -# fields-only rule in any ledger, it sorts after every name_regex -# rule, and it takes what nothing narrower could name. +issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix" +# 'Johnson PhD' / 'Smith Jr.' / 'John V' / 'QC MP': v1 routed a lone +# trailing suffix to family/first (no comma present); 2.0 keeps +# recognized suffixes in `suffix`. Four corpus names, and the prose +# fits all four. +# +# It claimed 25 until #372, on nothing but being the only fields-only +# rule in any ledger: it sorts after every name_regex rule, so it +# takes whatever nothing narrower named. The other 21 have their own +# rules now -- fix(cjk-glued-honorific-peel) and the widened +# fix(cjk-honorific-suffix) below for #312's honorifics, +# fix(comma-precomma-family) above for the three names that move no +# suffix at all. +# +# Being last is the point of this rule and worth keeping. What is not +# worth keeping is it being the only explanation a name ever gets: +# when it grows, the question is which narrower rule is missing. fields = ["given", "family", "suffix"] [[change]] @@ -386,6 +376,40 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "title", "suffix"] +[[change]] +issue = "fix(cjk-glued-honorific-peel) glued honorific peels into suffix" +# '김민준씨' -> 김/민준/씨, '田中さん' -> 田中/さん, 'Andersonさん' -> +# Anderson/さん: #312's peel, on a name with no space before the +# honorific and no comma anywhere. 17 corpus names, which fell to the +# fields-only fix(suffix-routing) above until #372 gave them a rule +# that names them. +# +# The regex is this file's first hand copy of GLUED_HONORIFICS rather +# than SUFFIX_WORDS, and that narrower vocabulary is the whole reason +# this rule can exist. '김민준씨' and the given name '김지양' are the +# same string shape -- hangul ending in a character the spaced list +# calls an honorific -- so a rule keyed on SUFFIX_WORDS could not tell +# them apart, and unanchoring the spaced rule below to reach the glued +# forms would let a future suffix regression on '김지양' classify as a +# recognized honorific. GLUED_HONORIFICS settles it in the config +# instead: 양, 군, 氏, 殿 and 博士 are deliberately absent from it, +# because a glued match on those collides with real given names. Same +# copy, same pin (_HONORIFIC_SOURCES) as the 2.0 ledger's twin, whose +# regex this is verbatim. +# +# `\.?(?=$|[ ,])` rather than `$`: the honorific ends a TOKEN, not +# necessarily the string. '田中さん II' and '김민준씨 Jr.' carry a +# Latin suffix after it, '김민준씨 (Jimmy)' a nickname. +# +# One spaced name lands here and is left that way, exactly as the 2.0 +# twin records: '김민준 박사님' matches on its 님, glued to 사 inside +# 박사님. The label stays true of it -- the peel is what routes it to +# `suffix` -- and closing the overlap needs a lookbehind asserting the +# match is not interior to a longer listed honorific, which is more +# machinery than a correct label is worth. +name_regex = "(?<=[^\\s,])(?:박사님|선생님|교수님|박사|씨|님|先生|女士|小姐|教授|様|さん|さま|くん|ちゃん)\\.?(?=$|[ ,])" +fields = ["given", "family", "suffix"] + [[change]] issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip" # '王小明 先生', '김민준 씨', '田中 太郎 様': #307 ships the spaced CJK @@ -417,7 +441,7 @@ issue = "fix(cjk-honorific-suffix) postnominal honorifics recognized, compoundin # 2026-07-30 run, not assumed: #308's peel put a dozen glued names # in the corpus (Latin+glued like Andersonさん included) and every # one classified there -- none carried a middle-field diff. -name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)$" +name_regex = "(?:^| )(?:씨|박사|박사님|선생님|교수님|군|양|님|先生|女士|小姐|博士|教授|様|氏|殿|さん|さま|くん|ちゃん)\\.?$" fields = ["given", "middle", "family", "suffix"] [[change]]