fix(pfs): code→family index keeps every family so indexed and scan anchors agree (refs #688)
code_family_index kept only the first family per code (setdefault), while anchor_metadata's un-indexed scan unions every qualifying family a code belongs to — so stack llm index (no code_index) and stack llm restamp (always indexed) would write different 'families' values for the same text once a code belongs to two families, which the next slice (CPT headings intersecting hand families) will create. Ruling A6: code_family_index now maps each code to the sorted tuple of every qualifying (>=2-code) family key, and anchor_metadata unions those tuples across all matched codes. The indexed and un-indexed paths are now equivalent by construction. Also drops the dead `fams = ...` computation in anchor_metadata when code_index is given. Does not touch the running background comments restamp — the five live hand families (ACP, CCM, PCM, TCM, APCM) don't share any codes, so its output is unaffected today.
This commit is contained in:
@@ -35,7 +35,7 @@ def plan_patches(
|
|||||||
rows: Iterable[tuple[str, str, Mapping[str, Any] | None]],
|
rows: Iterable[tuple[str, str, Mapping[str, Any] | None]],
|
||||||
*,
|
*,
|
||||||
families: Mapping[str, Family] | None = None,
|
families: Mapping[str, Family] | None = None,
|
||||||
code_index: Mapping[str, str] | None = None,
|
code_index: Mapping[str, tuple[str, ...]] | None = None,
|
||||||
) -> list[tuple[str, str]]:
|
) -> list[tuple[str, str]]:
|
||||||
out: list[tuple[str, str]] = []
|
out: list[tuple[str, str]] = []
|
||||||
for id_, document, md in rows:
|
for id_, document, md in rows:
|
||||||
@@ -60,7 +60,7 @@ def restamp(
|
|||||||
batch: int = 2000,
|
batch: int = 2000,
|
||||||
dry_run: bool = False,
|
dry_run: bool = False,
|
||||||
families: Mapping[str, Family] | None = None,
|
families: Mapping[str, Family] | None = None,
|
||||||
code_index: Mapping[str, str] | None = None,
|
code_index: Mapping[str, tuple[str, ...]] | None = None,
|
||||||
log_every: int = 50_000,
|
log_every: int = 50_000,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
scanned = updated = 0
|
scanned = updated = 0
|
||||||
|
|||||||
@@ -11,9 +11,10 @@ from pfs.elements import parse_descriptor
|
|||||||
from pfs.families import FAMILIES, Family, find_codes
|
from pfs.families import FAMILIES, Family, find_codes
|
||||||
|
|
||||||
|
|
||||||
def code_family_index(families: Mapping[str, Family]) -> dict[str, str]:
|
def code_family_index(families: Mapping[str, Family]) -> dict[str, tuple[str, ...]]:
|
||||||
"""Code -> family key, built once so ``anchor_metadata`` doesn't scan
|
"""Code -> sorted tuple of every qualifying family key, built once so
|
||||||
every family for every chunk (a restamp run calls it ~1.15M times).
|
``anchor_metadata`` doesn't scan every family for every chunk (a
|
||||||
|
restamp run calls it ~1.15M times).
|
||||||
|
|
||||||
Single-code families are excluded (Ruling A5): the derived registry
|
Single-code families are excluded (Ruling A5): the derived registry
|
||||||
is ~17,187 keys, all but a handful stem families with exactly one
|
is ~17,187 keys, all but a handful stem families with exactly one
|
||||||
@@ -21,30 +22,36 @@ def code_family_index(families: Mapping[str, Family]) -> dict[str, str]:
|
|||||||
code already in 'codes'. Only families with two or more codes (today:
|
code already in 'codes'. Only families with two or more codes (today:
|
||||||
ACP, CCM, PCM, TCM, APCM) are real groupings worth a separate anchor.
|
ACP, CCM, PCM, TCM, APCM) are real groupings worth a separate anchor.
|
||||||
|
|
||||||
A code that belongs to two qualifying families keeps the first family
|
Ruling A6: a code that belongs to two qualifying families keeps
|
||||||
reached in *families*' iteration (dict) order — same tie-break a
|
*both* — a single-winner index made the indexed path (``restamp``)
|
||||||
caller would see from scanning in that order."""
|
disagree with the un-indexed scan in ``anchor_metadata`` (which
|
||||||
out: dict[str, str] = {}
|
unions every match) the moment two families shared a code, which the
|
||||||
|
next slice (CPT headings intersecting a hand family) will create.
|
||||||
|
The two paths must write identical 'families' values for the same
|
||||||
|
text by construction."""
|
||||||
|
out: dict[str, list[str]] = {}
|
||||||
for key, fam in families.items():
|
for key, fam in families.items():
|
||||||
if len(fam.codes) < 2:
|
if len(fam.codes) < 2:
|
||||||
continue
|
continue
|
||||||
for code in fam.codes:
|
for code in fam.codes:
|
||||||
out.setdefault(code, key)
|
out.setdefault(code, []).append(key)
|
||||||
return out
|
return {code: tuple(sorted(keys)) for code, keys in out.items()}
|
||||||
|
|
||||||
|
|
||||||
def anchor_metadata(
|
def anchor_metadata(
|
||||||
text: str,
|
text: str,
|
||||||
*,
|
*,
|
||||||
families: Mapping[str, Family] | None = None,
|
families: Mapping[str, Family] | None = None,
|
||||||
code_index: Mapping[str, str] | None = None,
|
code_index: Mapping[str, tuple[str, ...]] | None = None,
|
||||||
) -> dict[str, str]:
|
) -> dict[str, str]:
|
||||||
"""Extract codes, families, and elements from text.
|
"""Extract codes, families, and elements from text.
|
||||||
|
|
||||||
Returns a dict with keys:
|
Returns a dict with keys:
|
||||||
- 'codes': space-joined sorted unique codes (from find_codes)
|
- 'codes': space-joined sorted unique codes (from find_codes)
|
||||||
- 'families': space-joined sorted unique family keys (families with
|
- 'families': space-joined sorted unique family keys — every
|
||||||
only one code are excluded — see ``code_family_index``)
|
qualifying family a code belongs to, unioned across all codes
|
||||||
|
(families with only one code are excluded — see
|
||||||
|
``code_family_index``)
|
||||||
- 'elements': space-joined sorted unique element slugs (type=value, no detail)
|
- 'elements': space-joined sorted unique element slugs (type=value, no detail)
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@@ -58,11 +65,11 @@ def anchor_metadata(
|
|||||||
Returns:
|
Returns:
|
||||||
dict with 'codes', 'families', and 'elements' keys
|
dict with 'codes', 'families', and 'elements' keys
|
||||||
"""
|
"""
|
||||||
fams = FAMILIES if families is None else families
|
|
||||||
codes = find_codes(text)
|
codes = find_codes(text)
|
||||||
if code_index is not None:
|
if code_index is not None:
|
||||||
keys = sorted({code_index[c] for c in codes if c in code_index})
|
keys = sorted({k for c in codes if c in code_index for k in code_index[c]})
|
||||||
else:
|
else:
|
||||||
|
fams = FAMILIES if families is None else families
|
||||||
keys = sorted(
|
keys = sorted(
|
||||||
{
|
{
|
||||||
k
|
k
|
||||||
|
|||||||
@@ -33,13 +33,33 @@ def test_code_index_matches_unindexed_scan():
|
|||||||
assert anchor_metadata(text, code_index=index) == anchor_metadata(text)
|
assert anchor_metadata(text, code_index=index) == anchor_metadata(text)
|
||||||
|
|
||||||
|
|
||||||
def test_code_index_keeps_first_family_on_collision():
|
def test_code_index_keeps_every_family_on_collision():
|
||||||
|
"""Ruling A6: a code shared by two qualifying families must keep
|
||||||
|
both in the index — a single-winner index made the indexed path
|
||||||
|
(restamp) disagree with the un-indexed scan the moment two families
|
||||||
|
shared a code, which the next slice (CPT headings intersecting a
|
||||||
|
hand family) will create."""
|
||||||
fams = {
|
fams = {
|
||||||
"A": Family("A", "A fam", ("12345", "23456"), ("a fam",)),
|
"A": Family("A", "A fam", ("12345", "23456"), ("a fam",)),
|
||||||
"B": Family("B", "B fam", ("12345", "34567"), ("b fam",)),
|
"B": Family("B", "B fam", ("12345", "34567"), ("b fam",)),
|
||||||
}
|
}
|
||||||
index = code_family_index(fams)
|
index = code_family_index(fams)
|
||||||
assert index["12345"] == "A"
|
assert index["12345"] == ("A", "B")
|
||||||
|
|
||||||
|
|
||||||
|
def test_code_index_matches_unindexed_scan_on_collision():
|
||||||
|
"""The indexed and un-indexed paths must write identical 'families'
|
||||||
|
values for the same text once a code belongs to two families."""
|
||||||
|
fams = {
|
||||||
|
"A": Family("A", "A fam", ("12345", "23456"), ("a fam",)),
|
||||||
|
"B": Family("B", "B fam", ("12345", "34567"), ("b fam",)),
|
||||||
|
}
|
||||||
|
text = "code 12345"
|
||||||
|
index = code_family_index(fams)
|
||||||
|
assert anchor_metadata(text, families=fams, code_index=index) == anchor_metadata(
|
||||||
|
text, families=fams
|
||||||
|
)
|
||||||
|
assert anchor_metadata(text, families=fams)["families"] == "A B"
|
||||||
|
|
||||||
|
|
||||||
def test_single_code_family_never_stamped():
|
def test_single_code_family_never_stamped():
|
||||||
@@ -47,7 +67,7 @@ def test_single_code_family_never_stamped():
|
|||||||
derived keys) is not a grouping worth its own anchor — it would just
|
derived keys) is not a grouping worth its own anchor — it would just
|
||||||
echo the code already in 'codes' (Ruling A5)."""
|
echo the code already in 'codes' (Ruling A5)."""
|
||||||
fams = {"SOLO": Family("SOLO", "Solo", ("54321",), ("solo",))}
|
fams = {"SOLO": Family("SOLO", "Solo", ("54321",), ("solo",))}
|
||||||
assert "SOLO" not in code_family_index(fams)
|
assert "54321" not in code_family_index(fams)
|
||||||
assert anchor_metadata("code 54321", families=fams)["families"] == ""
|
assert anchor_metadata("code 54321", families=fams)["families"] == ""
|
||||||
index = code_family_index(fams)
|
index = code_family_index(fams)
|
||||||
assert (
|
assert (
|
||||||
|
|||||||
Reference in New Issue
Block a user