fix(pfs): code→family index keeps every family so indexed and scan anchors agree (refs #688)

code_family_index kept only the first family per code (setdefault),
while anchor_metadata's un-indexed scan unions every qualifying
family a code belongs to — so stack llm index (no code_index) and
stack llm restamp (always indexed) would write different 'families'
values for the same text once a code belongs to two families, which
the next slice (CPT headings intersecting hand families) will create.

Ruling A6: code_family_index now maps each code to the sorted tuple
of every qualifying (>=2-code) family key, and anchor_metadata unions
those tuples across all matched codes. The indexed and un-indexed
paths are now equivalent by construction. Also drops the dead
`fams = ...` computation in anchor_metadata when code_index is given.

Does not touch the running background comments restamp — the five
live hand families (ACP, CCM, PCM, TCM, APCM) don't share any codes,
so its output is unaffected today.
This commit is contained in:
kert
2026-09-09 16:12:13 -04:00
parent 5930a295bd
commit c244cebba1
3 changed files with 46 additions and 19 deletions

View File

@@ -35,7 +35,7 @@ def plan_patches(
rows: Iterable[tuple[str, str, Mapping[str, Any] | None]], rows: Iterable[tuple[str, str, Mapping[str, Any] | None]],
*, *,
families: Mapping[str, Family] | None = None, families: Mapping[str, Family] | None = None,
code_index: Mapping[str, str] | None = None, code_index: Mapping[str, tuple[str, ...]] | None = None,
) -> list[tuple[str, str]]: ) -> list[tuple[str, str]]:
out: list[tuple[str, str]] = [] out: list[tuple[str, str]] = []
for id_, document, md in rows: for id_, document, md in rows:
@@ -60,7 +60,7 @@ def restamp(
batch: int = 2000, batch: int = 2000,
dry_run: bool = False, dry_run: bool = False,
families: Mapping[str, Family] | None = None, families: Mapping[str, Family] | None = None,
code_index: Mapping[str, str] | None = None, code_index: Mapping[str, tuple[str, ...]] | None = None,
log_every: int = 50_000, log_every: int = 50_000,
) -> dict: ) -> dict:
scanned = updated = 0 scanned = updated = 0

View File

@@ -11,9 +11,10 @@ from pfs.elements import parse_descriptor
from pfs.families import FAMILIES, Family, find_codes from pfs.families import FAMILIES, Family, find_codes
def code_family_index(families: Mapping[str, Family]) -> dict[str, str]: def code_family_index(families: Mapping[str, Family]) -> dict[str, tuple[str, ...]]:
"""Code -> family key, built once so ``anchor_metadata`` doesn't scan """Code -> sorted tuple of every qualifying family key, built once so
every family for every chunk (a restamp run calls it ~1.15M times). ``anchor_metadata`` doesn't scan every family for every chunk (a
restamp run calls it ~1.15M times).
Single-code families are excluded (Ruling A5): the derived registry Single-code families are excluded (Ruling A5): the derived registry
is ~17,187 keys, all but a handful stem families with exactly one is ~17,187 keys, all but a handful stem families with exactly one
@@ -21,30 +22,36 @@ def code_family_index(families: Mapping[str, Family]) -> dict[str, str]:
code already in 'codes'. Only families with two or more codes (today: code already in 'codes'. Only families with two or more codes (today:
ACP, CCM, PCM, TCM, APCM) are real groupings worth a separate anchor. ACP, CCM, PCM, TCM, APCM) are real groupings worth a separate anchor.
A code that belongs to two qualifying families keeps the first family Ruling A6: a code that belongs to two qualifying families keeps
reached in *families*' iteration (dict) order — same tie-break a *both* — a single-winner index made the indexed path (``restamp``)
caller would see from scanning in that order.""" disagree with the un-indexed scan in ``anchor_metadata`` (which
out: dict[str, str] = {} unions every match) the moment two families shared a code, which the
next slice (CPT headings intersecting a hand family) will create.
The two paths must write identical 'families' values for the same
text by construction."""
out: dict[str, list[str]] = {}
for key, fam in families.items(): for key, fam in families.items():
if len(fam.codes) < 2: if len(fam.codes) < 2:
continue continue
for code in fam.codes: for code in fam.codes:
out.setdefault(code, key) out.setdefault(code, []).append(key)
return out return {code: tuple(sorted(keys)) for code, keys in out.items()}
def anchor_metadata( def anchor_metadata(
text: str, text: str,
*, *,
families: Mapping[str, Family] | None = None, families: Mapping[str, Family] | None = None,
code_index: Mapping[str, str] | None = None, code_index: Mapping[str, tuple[str, ...]] | None = None,
) -> dict[str, str]: ) -> dict[str, str]:
"""Extract codes, families, and elements from text. """Extract codes, families, and elements from text.
Returns a dict with keys: Returns a dict with keys:
- 'codes': space-joined sorted unique codes (from find_codes) - 'codes': space-joined sorted unique codes (from find_codes)
- 'families': space-joined sorted unique family keys (families with - 'families': space-joined sorted unique family keys — every
only one code are excluded — see ``code_family_index``) qualifying family a code belongs to, unioned across all codes
(families with only one code are excluded — see
``code_family_index``)
- 'elements': space-joined sorted unique element slugs (type=value, no detail) - 'elements': space-joined sorted unique element slugs (type=value, no detail)
Args: Args:
@@ -58,11 +65,11 @@ def anchor_metadata(
Returns: Returns:
dict with 'codes', 'families', and 'elements' keys dict with 'codes', 'families', and 'elements' keys
""" """
fams = FAMILIES if families is None else families
codes = find_codes(text) codes = find_codes(text)
if code_index is not None: if code_index is not None:
keys = sorted({code_index[c] for c in codes if c in code_index}) keys = sorted({k for c in codes if c in code_index for k in code_index[c]})
else: else:
fams = FAMILIES if families is None else families
keys = sorted( keys = sorted(
{ {
k k

View File

@@ -33,13 +33,33 @@ def test_code_index_matches_unindexed_scan():
assert anchor_metadata(text, code_index=index) == anchor_metadata(text) assert anchor_metadata(text, code_index=index) == anchor_metadata(text)
def test_code_index_keeps_first_family_on_collision(): def test_code_index_keeps_every_family_on_collision():
"""Ruling A6: a code shared by two qualifying families must keep
both in the index — a single-winner index made the indexed path
(restamp) disagree with the un-indexed scan the moment two families
shared a code, which the next slice (CPT headings intersecting a
hand family) will create."""
fams = { fams = {
"A": Family("A", "A fam", ("12345", "23456"), ("a fam",)), "A": Family("A", "A fam", ("12345", "23456"), ("a fam",)),
"B": Family("B", "B fam", ("12345", "34567"), ("b fam",)), "B": Family("B", "B fam", ("12345", "34567"), ("b fam",)),
} }
index = code_family_index(fams) index = code_family_index(fams)
assert index["12345"] == "A" assert index["12345"] == ("A", "B")
def test_code_index_matches_unindexed_scan_on_collision():
"""The indexed and un-indexed paths must write identical 'families'
values for the same text once a code belongs to two families."""
fams = {
"A": Family("A", "A fam", ("12345", "23456"), ("a fam",)),
"B": Family("B", "B fam", ("12345", "34567"), ("b fam",)),
}
text = "code 12345"
index = code_family_index(fams)
assert anchor_metadata(text, families=fams, code_index=index) == anchor_metadata(
text, families=fams
)
assert anchor_metadata(text, families=fams)["families"] == "A B"
def test_single_code_family_never_stamped(): def test_single_code_family_never_stamped():
@@ -47,7 +67,7 @@ def test_single_code_family_never_stamped():
derived keys) is not a grouping worth its own anchor — it would just derived keys) is not a grouping worth its own anchor — it would just
echo the code already in 'codes' (Ruling A5).""" echo the code already in 'codes' (Ruling A5)."""
fams = {"SOLO": Family("SOLO", "Solo", ("54321",), ("solo",))} fams = {"SOLO": Family("SOLO", "Solo", ("54321",), ("solo",))}
assert "SOLO" not in code_family_index(fams) assert "54321" not in code_family_index(fams)
assert anchor_metadata("code 54321", families=fams)["families"] == "" assert anchor_metadata("code 54321", families=fams)["families"] == ""
index = code_family_index(fams) index = code_family_index(fams)
assert ( assert (