Make the comma rules explain what they describe (#372)

tests/v2/test_ledger_guards.py

@@ -310,9 +310,10 @@ def test_script_ranges_membership_is_decided() -> None:
 _SPAN_BEARING_RULES: dict[str, frozenset[str]] = {
     "expected_since_1.4.0.toml": frozenset({
         "fix(#271/#272/#298)",              # the canonical class
-        "fix(cjk-delimited-nickname)",      # the three compounds, whose
+        "fix(cjk-delimited-nickname)",      # the four compounds, whose
         "fix(cjk-fullwidth-paren-nickname)",  # lookaheads each carry
         "fix(cjk-comma-compound)",          # their own copy
+        "fix(cjk-comma-honorific-peel)",
     }),
     "expected_since_2.0.0.toml": frozenset({
         "fix(#271/#272/#298)",              # the canonical class
@@ -1015,7 +1016,7 @@ def _claim(rule: dict) -> _Claim:
         "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip":
             _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"),
         "fix(comma-family) lone post-comma piece routes to suffix/title, not first":
-            _Claim(236, ('given', 'suffix', 'title'), "3416f69d0ce4"),
+            _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"),
         "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix":
             _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"),
         "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole":
@@ -1030,6 +1031,8 @@ def _claim(rule: dict) -> _Claim:
             _Claim(6, ('family', 'given', 'nickname'), "ae1dffa01608"),
         "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition compounds with the CJK order flip":
             _Claim(1, ('family', 'given', 'middle', 'nickname'), "cf370e856ae7"),
+        "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name":
+            _Claim(20, ('given', 'suffix'), "b2ea8fa59eea"),
         "fix(cjk-comma-compound) comma routing compounds with the CJK order flip":
             _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"),
         "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip":
@@ -1125,6 +1128,83 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None:
         f"{sorted(set(_CORPUS_CLAIMS) - {L.name for L in _LEDGERS})}")
 
 
+#: Which rule classify() actually picks, for names several rules could
+#: claim. Keyed by (name, the diff it produces against that baseline).
+#:
+#: Every other guard here measures a rule ALONE: _CORPUS_CLAIMS records
+#: what a regex reaches, and the gate's total counts names. Neither can
+#: see which rule wins a contest, and that is the property #372 is
+#: about -- seven of these names spent months on fix(comma-family),
+#: whose prose describes none of them, with every guard green and the
+#: gate reporting 108/0 throughout.
+#:
+#: The comma family is recorded because it is where that went wrong.
+#: 42 corpus names are reachable by two or more rules in the same tier;
+#: these ten are the ones whose boundaries this file argues about, and
+#: pinning the argument is cheaper than re-deriving it. Note
+#: 'Andrews, M.D.' diffs on the SAME {given, suffix} shape as the seven
+#: peels -- only the CJK lookahead separates them, so it is the row
+#: that fails if that lookahead is ever dropped.
+#:
+#: The diff shapes are measured against the 1.4.0 wheel, not guessed.
+#: Re-measure rather than adjust them if a parser change moves one:
+#: a diff shape that shifted is a finding, not a number to update.
+_CROSS_RULE_WINNERS: dict[str, dict[tuple[str, tuple[str, ...]], str]] = {
+    "expected_since_1.4.0.toml": {
+        ("Andrews, M.D.", ("given", "suffix")): "fix(comma-family)",
+        ("田中, 太郎さん", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("김, 민준씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("김, 민준씨 (Jimmy)", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("김민준, 씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("김민준, 씨.", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("선생님, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        ("이, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)",
+        # the union rows: they move `family`, so the peel rule's fields
+        # exclude them and they must stay on the compound rule
+        ("Dr 김민준씨, V.", ("family", "given", "suffix")):
+            "fix(cjk-comma-compound)",
+        ("田中さん, PhD", ("family", "given", "suffix")):
+            "fix(cjk-comma-compound)",
+        ("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)",
+    },
+}
+
+
+def test_the_recorded_rule_still_wins_each_contested_name() -> None:
+    """Who explains what, which nothing else here asks.
+
+    Measured: narrowing fix(cjk-comma-compound)'s script class sends
+    three of these names to fix(suffix-routing) -- a fields-only
+    catch-all whose prose is about two-token Latin names -- and the
+    gate still reports 108 intentional / 0 unexplained. Reach is
+    per-rule, the total is per-corpus; neither notices a name changing
+    hands.
+
+    A failure here is not necessarily a regression: it can equally mean
+    a rule was narrowed correctly and its names found a better home. It
+    means someone has to look, which is the point.
+    """
+    compare = load_tool("compare")
+    by_name = {led.name: led for led in _LEDGERS}
+    checked = 0
+    for ledger_name, winners in _CROSS_RULE_WINNERS.items():
+        ledger = by_name[ledger_name]
+        rules = compare._sorted_rules(_rules(ledger))
+        never = _exclusions(ledger)
+        for (name, fields), expected in winners.items():
+            got = compare.classify(name, set(fields), rules, never)
+            assert got is not None and got.startswith(expected), (
+                f"{ledger_name}: {name!r} diffing {list(fields)} is now "
+                f"explained by {got!r}, not {expected!r}. Check the new "
+                f"rule's prose actually describes this name before "
+                f"recording it -- a rule claiming a diff it does not "
+                f"describe is #372, and it stays green everywhere else")
+            checked += 1
+    assert checked, "no contested name was checked, so this pin is vacuous"
+    assert set(_CROSS_RULE_WINNERS) <= {led.name for led in _LEDGERS}, (
+        f"_CROSS_RULE_WINNERS names ledgers that do not exist: "
+        f"{sorted(set(_CROSS_RULE_WINNERS) - {L.name for L in _LEDGERS})}")
+
 
 class _Excluded(NamedTuple):
     """What a [[never]] entry silences, in the two dimensions that can

tools/differential/expected_since_1.4.0.toml

@@ -121,45 +121,42 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir
 # (family/`last` is unchanged either way -- "pre-comma is definitionally
 # family").
 #
-# #312 (the glued-honorific peel site move) sent two of its five
-# DIFFING corpus rows here instead of the CJK-comma rule below, and it
-# is worth recording why, since this rule's prose has nothing to do
-# with CJK. Diffing, not new: four of the five are rows #312 added,
-# while '田中さん, PhD' was already in the corpus and #312 changed only
-# what it parses to. #312 added a fifth row that does NOT appear here,
-# '田中さん, 太郎' -- it agrees with 1.4.0, so it never reaches
-# classify() at all; it is the pinned guard for the case a naive fix
-# breaks (the peel site there is 太郎, so nothing peels).
-# '김, 민준씨' and '田中, 太郎さん' diff as {first, suffix}: the glued
-# honorific peels off the post-comma given name into `suffix`
-# (민준씨->민준/씨, 太郎さん->太郎/さん), and that field pair happens to
-# be a subset of this rule's [first, title, suffix]. No title moved and
-# the peeled token is CJK, not a Latin credential -- this rule doesn't
-# know or care, because compare.py's classify() matches purely on
-# regex-then-field-superset, first rule to satisfy both wins, and this
-# rule sits earlier in file order (name_regex tier, written-order tie-
-# break) than fix(cjk-comma-compound) below. The other two comma rows,
-# 'Dr 김민준씨, V.' and '田中さん, PhD', diff with `last` in the mix
-# (v1 read the whole pre-comma run as `first`; 2.0 splits family into
-# `last`), which is outside this rule's fields, so they correctly fall
-# through to fix(cjk-comma-compound). The fifth row, '威廉·莎士比亚さん',
-# has no comma at all -- its dot is U+00B7 (间隔号), not a comma -- so
-# it skips every comma-keyed rule and the honorific-suffix rule too (no
-# space precedes its glued さん), landing on the fields-only
-# fix(suffix-routing) tier by its plain {first, last, suffix} shape,
-# same as any glued Latin honorific. Verified by the 2026-08-01 run,
-# not assumed: traced through compare.py's classify(), which tries
-# name_regex rules before fields-only ones and, within a tier, file
-# order, returning the first rule whose regex (if any) matches the
-# string and whose fields are a superset of the diff. No rule here
-# inspects the honorific token itself -- the field shape alone decides.
-# The coverage pass that followed added one more row here, '김, 민준씨
-# (Jimmy)': same {first, suffix} diff as '김, 민준씨' -- the extracted
-# nickname matches on both sides and so is not part of the diff -- and
-# it lands on this rule for the same reason. The limit row recorded
-# next to it, '田中さん, V.', appears under no rule at all: it is
-# parity with 1.4.0, so it never reaches classify().
-name_regex = ","
+# LATIN-ANCHORED, and that is what keeps this rule describing what it
+# claims. The regex was a bare comma until #372, which reaches every
+# comma in every script -- so seven CJK glued-honorific names landed
+# here, on a rule whose prose has nothing to do with CJK. They diff as
+# {given, suffix} (민준씨->민준/씨, 太郎さん->太郎/さん), which is a
+# subset of this rule's fields, and classify() takes the first rule
+# whose regex matches and whose fields are a superset. Both this rule
+# and fix(cjk-comma-compound) carry a name_regex, so they share a tier
+# and file order broke the tie -- in favour of the one written first,
+# which was this one.
+#
+# Measured, narrowing it: corpus reach 236 -> 215, and this rule now
+# claims exactly one name, 'Andrews, M.D.', which is the shape named
+# above. All seven moved to fix(cjk-comma-compound), which grew 10 ->
+# 17 and is the rule named for that shape. The gate is unmoved at 108
+# intentional / 0 unexplained -- nothing became unexplained, so no name
+# lost an explanation, it only changed which rule gave it.
+#
+# The cut is at U+0250, the threshold _is_latin_only uses in
+# compare.py, matching the trailing-'Ph. D.' exclusion. Anchored at
+# both ends because a name is Latin or it is not: an unanchored class
+# would match the Latin part of a mixed name and take it back.
+#
+# One row nearby is not here and never was: '田中さん, 太郎' agrees
+# with 1.4.0, so it never reaches classify() at all. It is the pinned
+# guard for the case a naive #312 fix breaks (the peel site is 太郎, so
+# nothing peels).
+#
+# '田中さん, V.' is NOT that case, though this comment said it was
+# until #372 checked. It diffs {family, suffix} and lands on
+# fix(cjk-comma-compound) below, before and after this change. It
+# reads as parity only under Policy(lenient_comma_suffixes=False) --
+# the knob tests/v2/cases.py exercises in
+# ja_honorific_glued_family_comma_strict_knob -- and the corpora run
+# under the default policy, so here it is a real diff.
+name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$"
 fields = ["given", "title", "suffix"]
 
 [[change]]
@@ -169,9 +166,10 @@ issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix sta
 # `suffix`.
 # #312: also claims comma-less glued-honorific names like
 # '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any
-# glued Latin honorific -- see the note on fix(comma-family) above for
-# why its four comma-bearing siblings split between that rule and
-# fix(cjk-comma-compound) instead.
+# glued Latin honorific. Its comma-bearing siblings never reach here:
+# they split by whether the diff moves `family` --
+# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel)
+# when it does not -- both above, both keyed on a comma.
 fields = ["given", "family", "suffix"]
 
 [[change]]
@@ -291,6 +289,38 @@ issue = "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition com
 name_regex = "(?s)(?=.*[（）])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])"
 fields = ["given", "middle", "family", "nickname"]
 
+[[change]]
+issue = "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name"
+# '김, 민준씨' / '田中, 太郎さん': the comma already fixes family/given
+# order on both sides, so NO order flip happens -- `family` is
+# byte-identical. The only change is #312's peel: the honorific glued
+# to the post-comma given name comes off into `suffix` (민준씨 ->
+# 민준/씨, 太郎さん -> 太郎/さん). Diff is exactly {given, suffix} on
+# all seven names this rule claims.
+#
+# Its own rule because neither neighbour describes it, and #372 is
+# about rules explaining what they describe. fix(comma-family) above
+# is the Latin lone-post-comma-piece routing and is now anchored to
+# Latin, so it cannot reach these at all. fix(cjk-comma-compound)
+# below is for the UNION -- comma routing AND the family-first flip
+# together -- which is why its fields include `family`; these names
+# move no family, so claiming them there would be the same field-shape
+# coincidence #372 was filed over, relocated rather than fixed.
+#
+# `fields` is what separates the two, not file order. The union rows
+# ('Dr 김민준씨, V.', '田中さん, PhD') diff with `family` in the mix,
+# which is outside this rule's [given, suffix], so they cannot match
+# here however early it sits and still fall through to the compound
+# rule. Written above it anyway, so the narrower rule is also the
+# earlier one and the tie never has to be reasoned about.
+#
+# The regex is the same comma-plus-classified-codepoint pair the
+# compound rule carries, hand-copied from _SCRIPT_RANGES and pinned by
+# the same sync test -- a comma alone matches every Latin 'Smith, Jr.'
+# in the corpus.
+name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])"
+fields = ["given", "suffix"]
+
 [[change]]
 issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip"
 # '威廉·莎士比亚, PhD': one name, two intended changes at once -- the
@@ -309,11 +339,13 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip
 # #271/#272 substrings it avoids were a constraint of a selector
 # retired in #333.
 #
-# #312 sends only the two glued-honorific comma rows whose diff
-# includes `last` here ('Dr 김민준씨, V.', '田中さん, PhD'); the two
-# whose diff is only {first, suffix} ('김, 민준씨', '田中, 太郎さん')
-# match fix(comma-family) first instead, purely on file order -- see
-# the note there for why.
+# #312 sends only the glued-honorific comma rows whose diff includes
+# `family` here ('Dr 김민준씨, V.', '田中さん, PhD') -- the union this
+# rule is for. The rows whose diff is only {given, suffix} peel
+# without any order flip and belong to fix(cjk-comma-honorific-peel)
+# above, which names that shape and whose fields exclude `family` so
+# the two cannot compete. Until #372 they landed on fix(comma-family)
+# instead, on nothing but file order.
 name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])"
 fields = ["given", "middle", "family", "title", "suffix"]