feat(pfs,llm): anchor metadata — families and elements stamped beside codes; GIN indexes (refs #688)

src/llm/chunk.py

@@ -14,7 +14,7 @@ import hashlib
 import re
 from dataclasses import dataclass
 
-from pfs.families import find_codes
+from pfs.anchors import anchor_metadata
 
 
 @dataclass(frozen=True)
@@ -148,7 +148,7 @@ def chunk_doc(
                 # The heading is scanned with the piece: a section titled
                 # "## G0556 — APCM" is about G0556 all the way down, but
                 # only its first chunk repeats the code.
-                "codes": " ".join(find_codes(f"{heading} {piece}")),
+                **anchor_metadata(f"{heading} {piece}"),
             },
         )
         for seq, (heading, piece) in enumerate(pieces)
@@ -190,7 +190,7 @@ def _chunk_paragraphs(doc: Doc, target: int, overlap: int) -> list[Chunk]:
                 "item_key": doc.key,
                 "seq": str(seq),
                 "section": "",
-                "codes": " ".join(find_codes(text)),
+                **anchor_metadata(text),
                 "p_id": str(first.p_id),
                 "p_id_last": str(last.p_id),
                 "page": str(first.page),

src/llm/migrate.py

@@ -56,6 +56,12 @@ _HNSW_DDL = [
     "(string_to_array(coalesce(cmetadata->>'codes',''), ' '))",
     "CREATE INDEX IF NOT EXISTS ix_embedding_collection "
     "ON langchain_pg_embedding (collection_id)",
+    "CREATE INDEX IF NOT EXISTS ix_embedding_families "
+    "ON langchain_pg_embedding USING gin "
+    "(string_to_array(coalesce(cmetadata->>'families',''), ' '))",
+    "CREATE INDEX IF NOT EXISTS ix_embedding_elements "
+    "ON langchain_pg_embedding USING gin "
+    "(string_to_array(coalesce(cmetadata->>'elements',''), ' '))",
 ]
 
 

src/pfs/anchors.py

@@ -0,0 +1,39 @@
+"""Anchor metadata for a piece of text: the codes it names, the families
+those codes belong to, and the element slugs its phrases match. One
+function, used by the indexer at chunk time and by the restamp backfill,
+so both write identical values. Pure."""
+
+from __future__ import annotations
+
+from typing import Mapping
+
+from pfs.elements import parse_descriptor
+from pfs.families import FAMILIES, Family, find_codes
+
+
+def anchor_metadata(
+    text: str, *, families: Mapping[str, Family] | None = None
+) -> dict[str, str]:
+    """Extract codes, families, and elements from text.
+
+    Returns a dict with keys:
+    - 'codes': space-joined sorted unique codes (from find_codes)
+    - 'families': space-joined sorted unique family keys
+    - 'elements': space-joined sorted unique element slugs (type=value, no detail)
+
+    Args:
+        text: The text to analyze
+        families: Optional mapping of family keys to Family objects. Defaults to pfs.families.FAMILIES
+
+    Returns:
+        dict with 'codes', 'families', and 'elements' keys
+    """
+    fams = FAMILIES if families is None else families
+    codes = find_codes(text)
+    keys = sorted({k for k, f in fams.items() for c in codes if c in f.codes})
+    elements = sorted({f"{e.type.value}={e.value}" for e in parse_descriptor(text)})
+    return {
+        "codes": " ".join(codes),
+        "families": " ".join(keys),
+        "elements": " ".join(elements),
+    }

tests/llm/test_chunk.py

@@ -184,3 +184,32 @@ class TestCodesMetadata:
         )
         (chunk,) = chunk_doc(doc)
         assert chunk.metadata["codes"] == "G0556"
+
+
+class TestAnchorMetadata:
+    def test_plain_doc_chunks_have_codes_families_elements_keys(self):
+        text = "Use 99439; consent; per calendar month."
+        chunks = chunk_doc(_doc(text))
+        assert len(chunks) == 1
+        chunk = chunks[0]
+        assert "codes" in chunk.metadata
+        assert "families" in chunk.metadata
+        assert "elements" in chunk.metadata
+        assert chunk.metadata["codes"] == "99439"
+        assert chunk.metadata["families"] == "CCM"
+
+    def test_paragraph_chunks_have_codes_families_elements_keys(self):
+        from llm.chunk import Paragraph
+
+        paras = (Paragraph(1, 100, 1, "G0556 APCM code with consent."),)
+        doc = Doc(
+            key="R", text="G0556 APCM code with consent.", metadata={}, paragraphs=paras
+        )
+        chunks = chunk_doc(doc)
+        assert len(chunks) == 1
+        chunk = chunks[0]
+        assert "codes" in chunk.metadata
+        assert "families" in chunk.metadata
+        assert "elements" in chunk.metadata
+        assert chunk.metadata["codes"] == "G0556"
+        assert chunk.metadata["families"] == "APCM"

tests/llm/test_migrate.py

@@ -33,6 +33,13 @@ class TestDdl:
             "ix_embedding_collection" in executed
             and "langchain_pg_embedding (collection_id)" in executed
         )
+        # family and element indexes follow the same pattern
+        assert (
+            "ix_embedding_families" in executed and "cmetadata->>'families'" in executed
+        )
+        assert (
+            "ix_embedding_elements" in executed and "cmetadata->>'elements'" in executed
+        )
 
     def test_fingerprint_column_and_docket_state(self):
         assert "fingerprint" in migrate.INDEX_STATE_DDL

tests/pfs/test_anchors.py

@@ -0,0 +1,27 @@
+"""pfs.anchors — metadata for chunks: codes, families, and element slugs."""
+
+from pfs.anchors import anchor_metadata
+from pfs.families import Family
+
+
+def test_codes_families_elements():
+    md = anchor_metadata(
+        "Use 99439 in conjunction with 99490; consent; per calendar month"
+    )
+    assert md["codes"] == "99439 99490"
+    assert md["families"] == "CCM"
+    assert (
+        "activity=consent" in md["elements"].split()
+        and "period=calendar-month" in md["elements"].split()
+    )
+    assert "relation=addon-of" in md["elements"].split()  # detail dropped
+
+
+def test_empty_and_unknown():
+    assert anchor_metadata("nothing here") == {
+        "codes": "",
+        "families": "",
+        "elements": "",
+    }
+    fams = {"X": Family("X", "X fam", ("12345",), ("x fam",))}
+    assert anchor_metadata("code 12345", families=fams)["families"] == "X"