diff --git a/tests/v2/test_ledger_guards.py b/tests/v2/test_ledger_guards.py index 5c68306..0d651b1 100644 --- a/tests/v2/test_ledger_guards.py +++ b/tests/v2/test_ledger_guards.py @@ -310,9 +310,10 @@ def test_script_ranges_membership_is_decided() -> None: _SPAN_BEARING_RULES: dict[str, frozenset[str]] = { "expected_since_1.4.0.toml": frozenset({ "fix(#271/#272/#298)", # the canonical class - "fix(cjk-delimited-nickname)", # the three compounds, whose + "fix(cjk-delimited-nickname)", # the four compounds, whose "fix(cjk-fullwidth-paren-nickname)", # lookaheads each carry "fix(cjk-comma-compound)", # their own copy + "fix(cjk-comma-honorific-peel)", }), "expected_since_2.0.0.toml": frozenset({ "fix(#271/#272/#298)", # the canonical class @@ -1015,7 +1016,7 @@ def _claim(rule: dict) -> _Claim: "fix(cjk-maiden-marker) maiden marker consumed, compounding with the CJK order flip": _Claim(3, ('family', 'given', 'maiden', 'middle'), "cf5c9d671c14"), "fix(comma-family) lone post-comma piece routes to suffix/title, not first": - _Claim(236, ('given', 'suffix', 'title'), "3416f69d0ce4"), + _Claim(215, ('given', 'suffix', 'title'), "f16a0e79cba3"), "fix(suffix-routing) two-token name with unambiguous trailing suffix stays suffix": _Claim(751, ('family', 'given', 'suffix'), "231640fc7535"), "fix(suffix-delimiter-rendering) no-space delimiter core token kept whole": @@ -1030,6 +1031,8 @@ def _claim(rule: dict) -> _Claim: _Claim(6, ('family', 'given', 'nickname'), "ae1dffa01608"), "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition compounds with the CJK order flip": _Claim(1, ('family', 'given', 'middle', 'nickname'), "cf370e856ae7"), + "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name": + _Claim(20, ('given', 'suffix'), "b2ea8fa59eea"), "fix(cjk-comma-compound) comma routing compounds with the CJK order flip": _Claim(20, ('family', 'given', 'middle', 'suffix', 'title'), "b2ea8fa59eea"), "fix(cjk-honorific-suffix) postnominal honorifics recognized, compounding with the CJK order flip": @@ -1125,6 +1128,83 @@ def test_every_rule_claims_the_recorded_share_of_the_corpus() -> None: f"{sorted(set(_CORPUS_CLAIMS) - {L.name for L in _LEDGERS})}") +#: Which rule classify() actually picks, for names several rules could +#: claim. Keyed by (name, the diff it produces against that baseline). +#: +#: Every other guard here measures a rule ALONE: _CORPUS_CLAIMS records +#: what a regex reaches, and the gate's total counts names. Neither can +#: see which rule wins a contest, and that is the property #372 is +#: about -- seven of these names spent months on fix(comma-family), +#: whose prose describes none of them, with every guard green and the +#: gate reporting 108/0 throughout. +#: +#: The comma family is recorded because it is where that went wrong. +#: 42 corpus names are reachable by two or more rules in the same tier; +#: these ten are the ones whose boundaries this file argues about, and +#: pinning the argument is cheaper than re-deriving it. Note +#: 'Andrews, M.D.' diffs on the SAME {given, suffix} shape as the seven +#: peels -- only the CJK lookahead separates them, so it is the row +#: that fails if that lookahead is ever dropped. +#: +#: The diff shapes are measured against the 1.4.0 wheel, not guessed. +#: Re-measure rather than adjust them if a parser change moves one: +#: a diff shape that shifted is a finding, not a number to update. +_CROSS_RULE_WINNERS: dict[str, dict[tuple[str, tuple[str, ...]], str]] = { + "expected_since_1.4.0.toml": { + ("Andrews, M.D.", ("given", "suffix")): "fix(comma-family)", + ("田中, 太郎さん", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김, 민준씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김, 민준씨 (Jimmy)", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김민준, 씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("김민준, 씨.", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("선생님, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + ("이, J.씨", ("given", "suffix")): "fix(cjk-comma-honorific-peel)", + # the union rows: they move `family`, so the peel rule's fields + # exclude them and they must stay on the compound rule + ("Dr 김민준씨, V.", ("family", "given", "suffix")): + "fix(cjk-comma-compound)", + ("田中さん, PhD", ("family", "given", "suffix")): + "fix(cjk-comma-compound)", + ("田中さん, V.", ("family", "suffix")): "fix(cjk-comma-compound)", + }, +} + + +def test_the_recorded_rule_still_wins_each_contested_name() -> None: + """Who explains what, which nothing else here asks. + + Measured: narrowing fix(cjk-comma-compound)'s script class sends + three of these names to fix(suffix-routing) -- a fields-only + catch-all whose prose is about two-token Latin names -- and the + gate still reports 108 intentional / 0 unexplained. Reach is + per-rule, the total is per-corpus; neither notices a name changing + hands. + + A failure here is not necessarily a regression: it can equally mean + a rule was narrowed correctly and its names found a better home. It + means someone has to look, which is the point. + """ + compare = load_tool("compare") + by_name = {led.name: led for led in _LEDGERS} + checked = 0 + for ledger_name, winners in _CROSS_RULE_WINNERS.items(): + ledger = by_name[ledger_name] + rules = compare._sorted_rules(_rules(ledger)) + never = _exclusions(ledger) + for (name, fields), expected in winners.items(): + got = compare.classify(name, set(fields), rules, never) + assert got is not None and got.startswith(expected), ( + f"{ledger_name}: {name!r} diffing {list(fields)} is now " + f"explained by {got!r}, not {expected!r}. Check the new " + f"rule's prose actually describes this name before " + f"recording it -- a rule claiming a diff it does not " + f"describe is #372, and it stays green everywhere else") + checked += 1 + assert checked, "no contested name was checked, so this pin is vacuous" + assert set(_CROSS_RULE_WINNERS) <= {led.name for led in _LEDGERS}, ( + f"_CROSS_RULE_WINNERS names ledgers that do not exist: " + f"{sorted(set(_CROSS_RULE_WINNERS) - {L.name for L in _LEDGERS})}") + class _Excluded(NamedTuple): """What a [[never]] entry silences, in the two dimensions that can diff --git a/tools/differential/expected_since_1.4.0.toml b/tools/differential/expected_since_1.4.0.toml index 899b06a..80cfed7 100644 --- a/tools/differential/expected_since_1.4.0.toml +++ b/tools/differential/expected_since_1.4.0.toml @@ -121,45 +121,42 @@ issue = "fix(comma-family) lone post-comma piece routes to suffix/title, not fir # (family/`last` is unchanged either way -- "pre-comma is definitionally # family"). # -# #312 (the glued-honorific peel site move) sent two of its five -# DIFFING corpus rows here instead of the CJK-comma rule below, and it -# is worth recording why, since this rule's prose has nothing to do -# with CJK. Diffing, not new: four of the five are rows #312 added, -# while '田中さん, PhD' was already in the corpus and #312 changed only -# what it parses to. #312 added a fifth row that does NOT appear here, -# '田中さん, 太郎' -- it agrees with 1.4.0, so it never reaches -# classify() at all; it is the pinned guard for the case a naive fix -# breaks (the peel site there is 太郎, so nothing peels). -# '김, 민준씨' and '田中, 太郎さん' diff as {first, suffix}: the glued -# honorific peels off the post-comma given name into `suffix` -# (민준씨->민준/씨, 太郎さん->太郎/さん), and that field pair happens to -# be a subset of this rule's [first, title, suffix]. No title moved and -# the peeled token is CJK, not a Latin credential -- this rule doesn't -# know or care, because compare.py's classify() matches purely on -# regex-then-field-superset, first rule to satisfy both wins, and this -# rule sits earlier in file order (name_regex tier, written-order tie- -# break) than fix(cjk-comma-compound) below. The other two comma rows, -# 'Dr 김민준씨, V.' and '田中さん, PhD', diff with `last` in the mix -# (v1 read the whole pre-comma run as `first`; 2.0 splits family into -# `last`), which is outside this rule's fields, so they correctly fall -# through to fix(cjk-comma-compound). The fifth row, '威廉·莎士比亚さん', -# has no comma at all -- its dot is U+00B7 (间隔号), not a comma -- so -# it skips every comma-keyed rule and the honorific-suffix rule too (no -# space precedes its glued さん), landing on the fields-only -# fix(suffix-routing) tier by its plain {first, last, suffix} shape, -# same as any glued Latin honorific. Verified by the 2026-08-01 run, -# not assumed: traced through compare.py's classify(), which tries -# name_regex rules before fields-only ones and, within a tier, file -# order, returning the first rule whose regex (if any) matches the -# string and whose fields are a superset of the diff. No rule here -# inspects the honorific token itself -- the field shape alone decides. -# The coverage pass that followed added one more row here, '김, 민준씨 -# (Jimmy)': same {first, suffix} diff as '김, 민준씨' -- the extracted -# nickname matches on both sides and so is not part of the diff -- and -# it lands on this rule for the same reason. The limit row recorded -# next to it, '田中さん, V.', appears under no rule at all: it is -# parity with 1.4.0, so it never reaches classify(). -name_regex = "," +# LATIN-ANCHORED, and that is what keeps this rule describing what it +# claims. The regex was a bare comma until #372, which reaches every +# comma in every script -- so seven CJK glued-honorific names landed +# here, on a rule whose prose has nothing to do with CJK. They diff as +# {given, suffix} (민준씨->민준/씨, 太郎さん->太郎/さん), which is a +# subset of this rule's fields, and classify() takes the first rule +# whose regex matches and whose fields are a superset. Both this rule +# and fix(cjk-comma-compound) carry a name_regex, so they share a tier +# and file order broke the tie -- in favour of the one written first, +# which was this one. +# +# Measured, narrowing it: corpus reach 236 -> 215, and this rule now +# claims exactly one name, 'Andrews, M.D.', which is the shape named +# above. All seven moved to fix(cjk-comma-compound), which grew 10 -> +# 17 and is the rule named for that shape. The gate is unmoved at 108 +# intentional / 0 unexplained -- nothing became unexplained, so no name +# lost an explanation, it only changed which rule gave it. +# +# The cut is at U+0250, the threshold _is_latin_only uses in +# compare.py, matching the trailing-'Ph. D.' exclusion. Anchored at +# both ends because a name is Latin or it is not: an unanchored class +# would match the Latin part of a mixed name and take it back. +# +# One row nearby is not here and never was: '田中さん, 太郎' agrees +# with 1.4.0, so it never reaches classify() at all. It is the pinned +# guard for the case a naive #312 fix breaks (the peel site is 太郎, so +# nothing peels). +# +# '田中さん, V.' is NOT that case, though this comment said it was +# until #372 checked. It diffs {family, suffix} and lands on +# fix(cjk-comma-compound) below, before and after this change. It +# reads as parity only under Policy(lenient_comma_suffixes=False) -- +# the knob tests/v2/cases.py exercises in +# ja_honorific_glued_family_comma_strict_knob -- and the corpora run +# under the default policy, so here it is a real diff. +name_regex = "^[\\u0000-\\u024f]*,[\\u0000-\\u024f]*$" fields = ["given", "title", "suffix"] [[change]] @@ -169,9 +166,10 @@ issue = "fix(suffix-routing) two-token name with unambiguous trailing suffix sta # `suffix`. # #312: also claims comma-less glued-honorific names like # '威廉·莎士比亚さん' by the same {first, last, suffix} shape as any -# glued Latin honorific -- see the note on fix(comma-family) above for -# why its four comma-bearing siblings split between that rule and -# fix(cjk-comma-compound) instead. +# glued Latin honorific. Its comma-bearing siblings never reach here: +# they split by whether the diff moves `family` -- +# fix(cjk-comma-compound) when it does, fix(cjk-comma-honorific-peel) +# when it does not -- both above, both keyed on a comma. fields = ["given", "family", "suffix"] [[change]] @@ -291,6 +289,38 @@ issue = "fix(cjk-fullwidth-paren-nickname) fullwidth-parenthesis recognition com name_regex = "(?s)(?=.*[（）])(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "nickname"] +[[change]] +issue = "fix(cjk-comma-honorific-peel) glued honorific peels off a post-comma given name" +# '김, 민준씨' / '田中, 太郎さん': the comma already fixes family/given +# order on both sides, so NO order flip happens -- `family` is +# byte-identical. The only change is #312's peel: the honorific glued +# to the post-comma given name comes off into `suffix` (민준씨 -> +# 민준/씨, 太郎さん -> 太郎/さん). Diff is exactly {given, suffix} on +# all seven names this rule claims. +# +# Its own rule because neither neighbour describes it, and #372 is +# about rules explaining what they describe. fix(comma-family) above +# is the Latin lone-post-comma-piece routing and is now anchored to +# Latin, so it cannot reach these at all. fix(cjk-comma-compound) +# below is for the UNION -- comma routing AND the family-first flip +# together -- which is why its fields include `family`; these names +# move no family, so claiming them there would be the same field-shape +# coincidence #372 was filed over, relocated rather than fixed. +# +# `fields` is what separates the two, not file order. The union rows +# ('Dr 김민준씨, V.', '田中さん, PhD') diff with `family` in the mix, +# which is outside this rule's [given, suffix], so they cannot match +# here however early it sits and still fall through to the compound +# rule. Written above it anyway, so the narrower rule is also the +# earlier one and the tie never has to be reasoned about. +# +# The regex is the same comma-plus-classified-codepoint pair the +# compound rule carries, hand-copied from _SCRIPT_RANGES and pinned by +# the same sync test -- a comma alone matches every Latin 'Smith, Jr.' +# in the corpus. +name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" +fields = ["given", "suffix"] + [[change]] issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip" # '威廉·莎士比亚, PhD': one name, two intended changes at once -- the @@ -309,11 +339,13 @@ issue = "fix(cjk-comma-compound) comma routing compounds with the CJK order flip # #271/#272 substrings it avoids were a constraint of a selector # retired in #333. # -# #312 sends only the two glued-honorific comma rows whose diff -# includes `last` here ('Dr 김민준씨, V.', '田中さん, PhD'); the two -# whose diff is only {first, suffix} ('김, 민준씨', '田中, 太郎さん') -# match fix(comma-family) first instead, purely on file order -- see -# the note there for why. +# #312 sends only the glued-honorific comma rows whose diff includes +# `family` here ('Dr 김민준씨, V.', '田中さん, PhD') -- the union this +# rule is for. The rows whose diff is only {given, suffix} peel +# without any order flip and belong to fix(cjk-comma-honorific-peel) +# above, which names that shape and whose fields exclude `family` so +# the two cannot compete. Until #372 they landed on fix(comma-family) +# instead, on nothing but file order. name_regex = "(?s)(?=.*,)(?=.*[\\u3005-\\u3006\\u3040-\\u309F\\u30A0-\\u30FF\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF\\uAC00-\\uD7A3\\uFF65-\\uFF65])" fields = ["given", "middle", "family", "title", "suffix"]