From 34f573dd0b17ad293871b69eaed612c29e95a3b9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 12 Aug 2026 16:07:39 -0700 Subject: [PATCH 1/3] Stop fix(comma-family) explaining seven CJK names Its regex was a bare comma, which reaches every comma in every script, so seven glued-honorific CJK names classified under a rule whose prose is "lone post-comma piece routes to suffix/title, not first". Of the eight names it explained, that description fit exactly one. They diff as {given, suffix}, a subset of this rule's fields, and classify() takes the first rule whose regex matches and whose fields are a superset. fix(cjk-comma-compound) -- the rule named for that shape -- also matched, but both carry a name_regex, so they share a tier and file order decided. This rule was written first. Anchor it to Latin, at the U+0250 threshold _is_latin_only already uses, anchored at both ends so a mixed name cannot match on its Latin half. Measured: reach 236 -> 215, this rule now claims one name ('Andrews, M.D.'), and all seven moved to fix(cjk-comma-compound), which grew 10 -> 17. The gate is unmoved -- 108 intentional / 0 unexplained at 1.4.0, 90/0 at 2.0.0, 1/0 at 2.1.0. No name lost an explanation; only which rule gives it changed. That is the whole point: #372 is about rules explaining diffs they do not describe, not about diffs going unexplained. _CORPUS_CLAIMS re-recorded, which is the one guard that fired -- the forcing function working as designed. The rule's comment kept the history of how the CJK rows got here and now says how they left. Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 2 +- tools/differential/expected_since_1.4.0.toml | 67 ++++++++------------ 2 files changed, 29 insertions(+), 40 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 5c68306..e1f4403 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1015,7 +1015,7 @@ def _claim(rule: dict) -> _Claim: "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip": _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(236, ('given', 'suffix', 'title'), "3416f69d0ce4"), + _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 899b06a..2a98af6 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -121,45 +121,34 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # (family/`last` is unchanged either way -- "pre-comma is definitionally # family"). # -# #312 (the glued-honorific peel site move) sent two of its five -# DIFFING corpus rows here instead of the CJK-comma rule below, and it -# is worth recording why, since this rule's prose has nothing to do -# with CJK. Diffing, not new: four of the five are rows #312 added, -# while '田中さん, PhD' was already in the corpus and #312 changed only -# what it parses to. #312 added a fifth row that does NOT appear here, -# '田中さん, 太郎' -- it agrees with 1.4.0, so it never reaches -# classify() at all; it is the pinned guard for the case a naive fix -# breaks (the peel site there is 太郎, so nothing peels). -# '김, 민준씨' and '田中, 太郎さん' diff as {first, suffix}: the glued -# honorific peels off the post-comma given name into `suffix` -# (민준씨->민준/씨, 太郎さん->太郎/さん), and that field pair happens to -# be a subset of this rule's [first, title, suffix]. No title moved and -# the peeled token is CJK, not a Latin credential -- this rule doesn't -# know or care, because compare.py's classify() matches purely on -# regex-then-field-superset, first rule to satisfy both wins, and this -# rule sits earlier in file order (name_regex tier, written-order tie- -# break) than fix(cjk-comma-compound) below. The other two comma rows, -# 'Dr 김민준씨, V.' and '田中さん, PhD', diff with `last` in the mix -# (v1 read the whole pre-comma run as `first`; 2.0 splits family into -# `last`), which is outside this rule's fields, so they correctly fall -# through to fix(cjk-comma-compound). The fifth row, '威廉·莎士比亚さん', -# has no comma at all -- its dot is U+00B7 (间隔号), not a comma -- so -# it skips every comma-keyed rule and the honorific-suffix rule too (no -# space precedes its glued さん), landing on the fields-only -# fix(suffix-routing) tier by its plain {first, last, suffix} shape, -# same as any glued Latin honorific. Verified by the 2026-08-01 run, -# not assumed: traced through compare.py's classify(), which tries -# name_regex rules before fields-only ones and, within a tier, file -# order, returning the first rule whose regex (if any) matches the -# string and whose fields are a superset of the diff. No rule here -# inspects the honorific token itself -- the field shape alone decides. -# The coverage pass that followed added one more row here, '김, 민준씨 -# (Jimmy)': same {first, suffix} diff as '김, 민준씨' -- the extracted -# nickname matches on both sides and so is not part of the diff -- and -# it lands on this rule for the same reason. The limit row recorded -# next to it, '田中さん, V.', appears under no rule at all: it is -# parity with 1.4.0, so it never reaches classify(). -name_regex = "," +# LATIN-ANCHORED, and that is what keeps this rule describing what it +# claims. The regex was a bare comma until #372, which reaches every +# comma in every script -- so seven CJK glued-honorific names landed +# here, on a rule whose prose has nothing to do with CJK. They diff as +# {given, suffix} (민준씨->민준/씨, 太郎さん->太郎/さん), which is a +# subset of this rule's fields, and classify() takes the first rule +# whose regex matches and whose fields are a superset. Both this rule +# and fix(cjk-comma-compound) carry a name_regex, so they share a tier +# and file order broke the tie -- in favour of the one written first, +# which was this one. +# +# Measured, narrowing it: corpus reach 236 -> 215, and this rule now +# claims exactly one name, 'Andrews, M.D.', which is the shape named +# above. All seven moved to fix(cjk-comma-compound), which grew 10 -> +# 17 and is the rule named for that shape. The gate is unmoved at 108 +# intentional / 0 unexplained -- nothing became unexplained, so no name +# lost an explanation, it only changed which rule gave it. +# +# The cut is at U+0250, the threshold _is_latin_only uses in +# compare.py, matching the trailing-'Ph. D.' exclusion. Anchored at +# both ends because a name is Latin or it is not: an unanchored class +# would match the Latin part of a mixed name and take it back. +# +# One row nearby is not here and never was: '田中さん, 太郎' agrees +# with 1.4.0, so it never reaches classify() at all. It is the pinned +# guard for the case a naive #312 fix breaks (the peel site is 太郎, so +# nothing peels). Same for '田中さん, V.'. +name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "title", "suffix"] [[change]] From e30898768567c3b01ebedb23e96408debd3ad15b Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 12 Aug 2026 19:02:24 -0700 Subject: [PATCH 2/3] Give the peeled post-comma names a rule that describes them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the previous commit found it relocated the over-claiming rather than fixing it. Measured: all seven names diff as exactly {given, suffix} with `family` byte-identical -- no order flip happens, because the comma already fixes family/given order on both sides. But fix(cjk-comma-compound) exists for the UNION, comma routing AND the flip together, which is why its fields include `family`. Its own comment says so. The seven landed there on a fields superset and file order: the same field-shape coincidence #372 was filed over, moved one rule along. Add fix(cjk-comma-honorific-peel) for what actually happens -- #312's honorific peeling off the post-comma given name into `suffix`, family untouched -- with fields = ["given", "suffix"]. The field list is what separates it from the compound rule, not its position: the union rows move `family`, which is outside these fields, so they cannot match it however early it sits. It is written first anyway so the narrower rule is also the earlier one. Now each of the three comma rules explains only what it describes: fix(comma-family) 1 'Andrews, M.D.' fix(cjk-comma-honorific-peel) 7 the peels fix(cjk-comma-compound) 10 the unions, back to its original set Gate unmoved at 108/0, 90/0, 1/0. Two rosters fired and were recorded: _SPAN_BEARING_RULES (the rule carries the script class) and _CORPUS_CLAIMS. Three prose sites corrected, all found by review. Two were invalidated by the previous commit: fix(cjk-comma-compound) still said the {given,suffix} rows match fix(comma-family) "purely on file order", and fix(suffix-routing) still pointed at a note about a split that no longer exists. The third was inherited and had been wrong before this branch: '田中さん, V.' is not a parity row. It diffs {family, suffix} and reaches classify(); it reads as parity only under Policy(lenient_comma_suffixes=False), which the corpora never use. Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 5 +- tools/differential/expected_since_1.4.0.toml | 61 +++++++++++++++++--- 2 files changed, 56 insertions(+), 10 deletions(-) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index e1f4403..bbb07df 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -310,9 +310,10 @@ def test_script_ranges_membership_is_decided() -> None: _SPAN_BEARING_RULES: dict[str, frozenset[str]] = { "expected_since_1.4.0.toml": frozenset({ "fix(#271/#272/#298)", # the canonical class - "fix(cjk-delimited-nickname)", # the three compounds, whose + "fix(cjk-delimited-nickname)", # the four compounds, whose "fix(cjk-fullwidth-paren-nickname)", # lookaheads each carry "fix(cjk-comma-compound)", # their own copy + "fix(cjk-comma-honorific-peel)", }), "expected_since_2.0.0.toml": frozenset({ "fix(#271/#272/#298)", # the canonical class @@ -1030,6 +1031,8 @@ def _claim(rule: dict) -> _Claim: _Claim(6, ('family', 'given', 'nickname'), "ae1dffa01608"), "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition compounds with the CJK order flip": _Claim(1, ('family', 'given', 'middle', 'nickname'), "cf370e856ae7"), + "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name": + _Claim(20, ('given', 'suffix'), "b2ea8fa59eea"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 2a98af6..80cfed7 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -147,7 +147,15 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # One row nearby is not here and never was: '田中さん, 太郎' agrees # with 1.4.0, so it never reaches classify() at all. It is the pinned # guard for the case a naive #312 fix breaks (the peel site is 太郎, so -# nothing peels). Same for '田中さん, V.'. +# nothing peels). +# +# '田中さん, V.' is NOT that case, though this comment said it was +# until #372 checked. It diffs {family, suffix} and lands on +# fix(cjk-comma-compound) below, before and after this change. It +# reads as parity only under Policy(lenient_comma_suffixes=False) -- +# the knob tests/v2/cases.py exercises in +# ja_honorific_glued_family_comma_strict_knob -- and the corpora run +# under the default policy, so here it is a real diff. name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "title", "suffix"] @@ -158,9 +166,10 @@ issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix sta # `suffix`. # #312: also claims comma-less glued-honorific names like # '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any -# glued Latin honorific -- see the note on fix(comma-family) above for -# why its four comma-bearing siblings split between that rule and -# fix(cjk-comma-compound) instead. +# glued Latin honorific. Its comma-bearing siblings never reach here: +# they split by whether the diff moves `family` -- +# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel) +# when it does not -- both above, both keyed on a comma. fields = ["given", "family", "suffix"] [[change]] @@ -280,6 +289,38 @@ issue = "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition com name_regex = "(?s)(?=.*[（）])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "nickname"] +[[change]] +issue = "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name" +# '김, 민준씨' / '田中, 太郎さん': the comma already fixes family/given +# order on both sides, so NO order flip happens -- `family` is +# byte-identical. The only change is #312's peel: the honorific glued +# to the post-comma given name comes off into `suffix` (민준씨 -> +# 민준/씨, 太郎さん -> 太郎/さん). Diff is exactly {given, suffix} on +# all seven names this rule claims. +# +# Its own rule because neither neighbour describes it, and #372 is +# about rules explaining what they describe. fix(comma-family) above +# is the Latin lone-post-comma-piece routing and is now anchored to +# Latin, so it cannot reach these at all. fix(cjk-comma-compound) +# below is for the UNION -- comma routing AND the family-first flip +# together -- which is why its fields include `family`; these names +# move no family, so claiming them there would be the same field-shape +# coincidence #372 was filed over, relocated rather than fixed. +# +# `fields` is what separates the two, not file order. The union rows +# ('Dr 김민준씨, V.', '田中さん, PhD') diff with `family` in the mix, +# which is outside this rule's [given, suffix], so they cannot match +# here however early it sits and still fall through to the compound +# rule. Written above it anyway, so the narrower rule is also the +# earlier one and the tie never has to be reasoned about. +# +# The regex is the same comma-plus-classified-codepoint pair the +# compound rule carries, hand-copied from _SCRIPT_RANGES and pinned by +# the same sync test -- a comma alone matches every Latin 'Smith, Jr.' +# in the corpus. +name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" +fields = ["given", "suffix"] + [[change]] issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip" # '威廉·莎士比亚, PhD': one name, two intended changes at once -- the @@ -298,11 +339,13 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip # #271/#272 substrings it avoids were a constraint of a selector # retired in #333. # -# #312 sends only the two glued-honorific comma rows whose diff -# includes `last` here ('Dr 김민준씨, V.', '田中さん, PhD'); the two -# whose diff is only {first, suffix} ('김, 민준씨', '田中, 太郎さん') -# match fix(comma-family) first instead, purely on file order -- see -# the note there for why. +# #312 sends only the glued-honorific comma rows whose diff includes +# `family` here ('Dr 김민준씨, V.', '田中さん, PhD') -- the union this +# rule is for. The rows whose diff is only {given, suffix} peel +# without any order flip and belong to fix(cjk-comma-honorific-peel) +# above, which names that shape and whose fields exclude `family` so +# the two cannot compete. Until #372 they landed on fix(comma-family) +# instead, on nothing but file order. name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "title", "suffix"] From 45f74c8accdfefe7a08655f15174e29810c94c0f Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Wed, 12 Aug 2026 19:49:56 -0700 Subject: [PATCH 3/3] Pin which rule wins each contested comma name Every other guard here measures a rule ALONE. _CORPUS_CLAIMS records what a regex reaches; the gate counts names. Neither can see which rule wins a contest -- which is the property #372 is about, and why seven names sat on fix(comma-family), whose prose describes none of them, for months with every guard green and the gate reporting 108/0 throughout. _CROSS_RULE_WINNERS records classify()'s answer for the ten names whose boundaries this file argues about, keyed by (name, diff shape). The diff shapes are measured against the 1.4.0 wheel, not guessed. 42 corpus names are reachable by two or more same-tier rules; these are the ones with an argument attached, and pinning the argument is cheaper than re-deriving it. Proven by mutation, both directions the reviewers named: drop Hangul from the peel rule's class -> 1 failed (three Korean names fall through to fix(suffix-routing), a fields-only catch-all about two-token Latin names; the gate stays 108/0 and reports nothing) move the peel rule after the compound rule -> 1 failed a pure reorder, no regex or fields change. Measured across the whole suite: this is the ONLY test that fails. _CORPUS_CLAIMS is order-blind by construction, so before this pin a reorder could silently reattribute names with 3217 tests green. Note 'Andrews, M.D.' diffs on the same {given, suffix} shape as the seven peels -- only the CJK lookahead separates them, so it is the row that fails if that lookahead is ever dropped. A failure here is not automatically a regression: a rule narrowed correctly may send its names somewhere better. It means someone has to look, which is the point. Co-Authored-By: Claude Opus 5 --- tests/v2/test_ledger_guards.py | 77 ++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index bbb07df..0d651b1 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -1128,6 +1128,83 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: f"{sorted(set(_CORPUS_CLAIMS) - {L.name for L in _LEDGERS})}") +#: Which rule classify() actually picks, for names several rules could +#: claim. Keyed by (name, the diff it produces against that baseline). +#: +#: Every other guard here measures a rule ALONE: _CORPUS_CLAIMS records +#: what a regex reaches, and the gate's total counts names. Neither can +#: see which rule wins a contest, and that is the property #372 is +#: about -- seven of these names spent months on fix(comma-family), +#: whose prose describes none of them, with every guard green and the +#: gate reporting 108/0 throughout. +#: +#: The comma family is recorded because it is where that went wrong. +#: 42 corpus names are reachable by two or more rules in the same tier; +#: these ten are the ones whose boundaries this file argues about, and +#: pinning the argument is cheaper than re-deriving it. Note +#: 'Andrews, M.D.' diffs on the SAME {given, suffix} shape as the seven +#: peels -- only the CJK lookahead separates them, so it is the row +#: that fails if that lookahead is ever dropped. +#: +#: The diff shapes are measured against the 1.4.0 wheel, not guessed. +#: Re-measure rather than adjust them if a parser change moves one: +#: a diff shape that shifted is a finding, not a number to update. +_CROSS_RULE_WINNERS: dict[str, dict[tuple[str, tuple[str, ...]], str]] = { + "expected_since_1.4.0.toml": { + ("Andrews, M.D.", ("given", "suffix")): "fix(comma-family)", + ("田中, 太郎さん", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김, 민준씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김, 민준씨 (Jimmy)", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김민준, 씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김민준, 씨.", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("선생님, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("이, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + # the union rows: they move `family`, so the peel rule's fields + # exclude them and they must stay on the compound rule + ("Dr 김민준씨, V.", ("family", "given", "suffix")): + "fix(cjk-comma-compound)", + ("田中さん, PhD", ("family", "given", "suffix")): + "fix(cjk-comma-compound)", + ("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)", + }, +} + + +def test_the_recorded_rule_still_wins_each_contested_name() -> None: + """Who explains what, which nothing else here asks. + + Measured: narrowing fix(cjk-comma-compound)'s script class sends + three of these names to fix(suffix-routing) -- a fields-only + catch-all whose prose is about two-token Latin names -- and the + gate still reports 108 intentional / 0 unexplained. Reach is + per-rule, the total is per-corpus; neither notices a name changing + hands. + + A failure here is not necessarily a regression: it can equally mean + a rule was narrowed correctly and its names found a better home. It + means someone has to look, which is the point. + """ + compare = load_tool("compare") + by_name = {led.name: led for led in _LEDGERS} + checked = 0 + for ledger_name, winners in _CROSS_RULE_WINNERS.items(): + ledger = by_name[ledger_name] + rules = compare._sorted_rules(_rules(ledger)) + never = _exclusions(ledger) + for (name, fields), expected in winners.items(): + got = compare.classify(name, set(fields), rules, never) + assert got is not None and got.startswith(expected), ( + f"{ledger_name}: {name!r} diffing {list(fields)} is now " + f"explained by {got!r}, not {expected!r}. Check the new " + f"rule's prose actually describes this name before " + f"recording it -- a rule claiming a diff it does not " + f"describe is #372, and it stays green everywhere else") + checked += 1 + assert checked, "no contested name was checked, so this pin is vacuous" + assert set(_CROSS_RULE_WINNERS) <= {led.name for led in _LEDGERS}, ( + f"_CROSS_RULE_WINNERS names ledgers that do not exist: " + f"{sorted(set(_CROSS_RULE_WINNERS) - {L.name for L in _LEDGERS})}") + class _Excluded(NamedTuple): """What a [[never]] entry silences, in the two dimensions that can