diff --git a/notebooks/code_families.py b/notebooks/code_families.py index d5938b6..947c75a 100644 --- a/notebooks/code_families.py +++ b/notebooks/code_families.py @@ -615,6 +615,7 @@ def _(code, mo, not_built, pl, store): from llm.config import load as _load_llm_cfg from llm.config import pg_url as _pg_url + from pfs.anchors import families_array_sql as _families_array_sql _eng = _sa_engine(_pg_url(_load_llm_cfg())) with _eng.begin() as _conn: @@ -623,8 +624,7 @@ def _(code, mo, not_built, pl, store): "SELECT c.name, count(*) AS n " "FROM langchain_pg_embedding e " "JOIN langchain_pg_collection c ON c.uuid = e.collection_id " - "WHERE string_to_array(COALESCE(e.cmetadata->>'families', ''), " - "' ') && ARRAY[:key] " + f"WHERE {_families_array_sql('e')} && ARRAY[:key] " "GROUP BY c.name ORDER BY c.name" ), {"key": _key}, diff --git a/src/bib/tag.py b/src/bib/tag.py index ed355f2..436f58c 100644 --- a/src/bib/tag.py +++ b/src/bib/tag.py @@ -271,16 +271,19 @@ class Tag(BaseModel): Parameters ---------- key : str - The family key exactly as registered (e.g. ``CCM``, - ``APCM``, or a derived ``pfs.code_family`` key) — never - lower-cased, since derived keys aren't necessarily - uppercase-safe round trips. + The family key as registered (e.g. ``CCM``, ``APCM``, or a + derived ``pfs.code_family`` key); normalised through + ``pfs.families.normalize_key`` (stripped, upper-case — every + registry key is stored that way) so one spelling never + becomes two tags. Examples:: Tag.family("CCM") # family:CCM """ - return cls(namespace="family", value=key) + from pfs.families import normalize_key + + return cls(namespace="family", value=normalize_key(key)) @classmethod def fn(cls, qualified_name: str) -> Tag: diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 1b76b02..75d4d6d 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -42,7 +42,13 @@ from pfs.codetables import ( ) from pfs.cpt_load import ingest as cpt_ingest from pfs.extract import extract_code -from pfs.families import FAMILIES, HAND_FAMILIES, derive_families, refresh_from +from pfs.families import ( + FAMILIES, + HAND_FAMILIES, + derive_families, + normalize_key, + refresh_from, +) from pfs.guidance import build as build_guidance from pfs.lineage import lineage, lineage_all from pfs.reaction import series as reaction_series @@ -136,7 +142,7 @@ def _codes_for( if families: refresh_from(con) for family in families: - fam = FAMILIES.get(family.upper()) + fam = FAMILIES.get(normalize_key(family)) if fam is None: # List only the hand families here (M3) — FAMILIES can hold # ~15k derived keys once `refresh_from` has run, and that @@ -475,7 +481,7 @@ def guidance( family's FR paragraphs and the CPT manual cite, with FR provenance.""" if not family: raise typer.BadParameter("pass --family (repeatable)") - keys = [f.upper() for f in family] + keys = [normalize_key(f) for f in family] store = _store() if write: # Ruling A13 (mirrors `stack pfs reaction`): resolve every family @@ -559,7 +565,7 @@ def _reaction_targets( refresh_from(con) targets: list[tuple[str, tuple[str, ...], str | None]] = [] for f in family: - key = f.upper() + key = normalize_key(f) fam = FAMILIES.get(key) if fam is None: raise typer.BadParameter( diff --git a/src/llm/evidence.py b/src/llm/evidence.py index 60b832d..a0748ca 100644 --- a/src/llm/evidence.py +++ b/src/llm/evidence.py @@ -24,6 +24,7 @@ import duckdb from llm.config import LlmConfig from llm.links import as_source +from pfs.anchors import codes_array_sql, families_array_sql from pfs.codetables import is_missing_table_error from pfs.families import FAMILIES, Detection, detect_codes, refresh_from from pfs.valuation import ValuationRow, valuation @@ -296,7 +297,7 @@ _CITED_SQL = ( ") AS rn " "FROM langchain_pg_embedding e " "JOIN unnest(CAST(:codes AS text[])) AS w(code) " - "ON w.code = ANY(string_to_array(COALESCE(e.cmetadata->>'codes', ''), ' ')) " + f"ON w.code = ANY({codes_array_sql('e')}) " "WHERE e.collection_id = (SELECT uuid FROM langchain_pg_collection " "WHERE name = :collection) " ") t " @@ -332,7 +333,7 @@ _FAMILY_CITED_SQL = ( ") AS item_rn " "FROM langchain_pg_embedding e " "JOIN unnest(CAST(:families AS text[])) AS w(family) " - "ON string_to_array(COALESCE(e.cmetadata->>'families', ''), ' ') && ARRAY[w.family] " + f"ON {families_array_sql('e')} && ARRAY[w.family] " "WHERE e.collection_id = (SELECT uuid FROM langchain_pg_collection " "WHERE name = :collection) " ") one_per_item " @@ -358,7 +359,7 @@ _DOCKET_CITED_SQL = ( ") AS rn " "FROM langchain_pg_embedding e " "JOIN unnest(CAST(:codes AS text[])) AS w(code) " - "ON w.code = ANY(string_to_array(COALESCE(e.cmetadata->>'codes', ''), ' ')) " + f"ON w.code = ANY({codes_array_sql('e')}) " "WHERE e.collection_id = (SELECT uuid FROM langchain_pg_collection " "WHERE name = :collection) " ") t " diff --git a/src/pfs/anchors.py b/src/pfs/anchors.py index 83f5d83..ae2b666 100644 --- a/src/pfs/anchors.py +++ b/src/pfs/anchors.py @@ -16,6 +16,32 @@ from pfs.elements import parse_descriptor from pfs.families import FAMILIES, Family, find_codes +#: The chunk-metadata arrays every code/family predicate reads. ONE +#: definition (#705 item 4): the ``&&`` GIN index on pgvector was built on +#: exactly ``string_to_array(coalesce(cmetadata->>'families',''),' ')`` and a +#: predicate that spells it differently silently loses the index. +def codes_array_sql(alias: str = "e") -> str: + """SQL for the chunk's ``codes`` metadata as a text[] (space-joined).""" + return f"string_to_array(COALESCE({alias}.cmetadata->>'codes', ''), ' ')" + + +def families_array_sql(alias: str = "e") -> str: + """SQL for the chunk's ``families`` metadata as a text[] (space-joined).""" + return f"string_to_array(COALESCE({alias}.cmetadata->>'families', ''), ' ')" + + +def code_or_family_predicate( + alias: str = "e", *, codes_param: str = ":codes", family_param: str = ":family_key" +) -> str: + """``(codes && CAST(:codes AS text[])) OR (families && ARRAY[:family_key])`` + — the two-way match the chat's cited-source windows and the reaction + series share.""" + return ( + f"({codes_array_sql(alias)} && CAST({codes_param} AS text[])) " + f"OR ({families_array_sql(alias)} && ARRAY[{family_param}])" + ) + + def code_family_index(families: Mapping[str, Family]) -> dict[str, tuple[str, ...]]: """Code -> sorted tuple of every qualifying family key, built once so ``anchor_metadata`` doesn't scan every family for every chunk (a diff --git a/src/pfs/families.py b/src/pfs/families.py index b5441f0..e3c485d 100644 --- a/src/pfs/families.py +++ b/src/pfs/families.py @@ -28,6 +28,14 @@ _FR_CITE_RE = re.compile(r"\b\d{1,3}\s+FR\s+\d{3,6}\b", re.IGNORECASE) FR_CITE_RE = _FR_CITE_RE +def normalize_key(key: str) -> str: + """The canonical form of a family key: stripped, upper-case. Every + registry key (hand and derived) is stored this way; ``Tag.family``, + the ``stack pfs`` commands and the chat all normalise through here + (#705 item 5) so one spelling never becomes two tags or two families.""" + return (key or "").strip().upper() + + def find_codes(text: str) -> tuple[str, ...]: """Sorted unique codes literally present in *text*, upper-cased. diff --git a/src/pfs/reaction.py b/src/pfs/reaction.py index 3bec47c..9dcc160 100644 --- a/src/pfs/reaction.py +++ b/src/pfs/reaction.py @@ -37,6 +37,7 @@ from typing import Any, Callable, Sequence from sqlalchemy import text +from pfs.anchors import code_or_family_predicate from pfs.descriptors import rule_year_of from pfs.families import code_pattern, codes_in @@ -56,8 +57,7 @@ _DOCKET_COUNTS_SQL = text( "count(DISTINCT item_key) AS n_total " "FROM ( " "SELECT e.cmetadata->>'docket' AS docket, e.cmetadata->>'item_key' AS item_key, " - "(string_to_array(COALESCE(e.cmetadata->>'codes', ''), ' ') && CAST(:codes AS text[])) " - "OR (string_to_array(COALESCE(e.cmetadata->>'families', ''), ' ') && ARRAY[:family_key]) " + f"{code_or_family_predicate('e')} " "AS matched " "FROM langchain_pg_embedding e " "WHERE e.collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = :collection) " @@ -83,8 +83,7 @@ _STANCE_SAMPLE_SQL = text( "WHERE e.collection_id = (SELECT uuid FROM langchain_pg_collection WHERE name = :collection) " "AND e.cmetadata->>'docket' = :docket " "AND ( " - "(string_to_array(COALESCE(e.cmetadata->>'codes', ''), ' ') && CAST(:codes AS text[])) " - "OR (string_to_array(COALESCE(e.cmetadata->>'families', ''), ' ') && ARRAY[:family_key]) " + f"{code_or_family_predicate('e')} " ") " ") one_per_item " "WHERE item_rn = 1 " diff --git a/tests/bib/test_codetags.py b/tests/bib/test_codetags.py index 0ec7307..0557474 100644 --- a/tests/bib/test_codetags.py +++ b/tests/bib/test_codetags.py @@ -28,6 +28,7 @@ class TestTagFactories: def test_family_label(self) -> None: assert Tag.family("CCM").label == "family:CCM" + assert Tag.family(" ccm ").label == "family:CCM" # normalised (#705) class TestItemCodesFromChunks: diff --git a/tests/pfs/test_anchors.py b/tests/pfs/test_anchors.py index b6a7170..af798bb 100644 --- a/tests/pfs/test_anchors.py +++ b/tests/pfs/test_anchors.py @@ -73,3 +73,33 @@ def test_single_code_family_never_stamped(): assert ( anchor_metadata("code 54321", families=fams, code_index=index)["families"] == "" ) + + +class TestSharedPredicateSql: + """#705 item 4: one definition of the metadata-array expressions the + GIN index was built on; every predicate composes from it.""" + + def test_arrays_match_the_index_expression(self): + from pfs.anchors import codes_array_sql, families_array_sql + + assert codes_array_sql("e") == ( + "string_to_array(COALESCE(e.cmetadata->>'codes', ''), ' ')" + ) + assert families_array_sql("x") == ( + "string_to_array(COALESCE(x.cmetadata->>'families', ''), ' ')" + ) + + def test_predicate_composes_both_sides(self): + from pfs.anchors import code_or_family_predicate + + p = code_or_family_predicate("e") + assert "&& CAST(:codes AS text[])" in p and "&& ARRAY[:family_key]" in p + assert p.startswith("(string_to_array(COALESCE(e.cmetadata->>'codes'") + + def test_consumers_use_the_shared_text(self): + import llm.evidence as ev + import pfs.reaction as rx + from pfs.anchors import code_or_family_predicate, codes_array_sql + + assert codes_array_sql("e") in ev._CITED_SQL + assert code_or_family_predicate("e") in str(rx._DOCKET_COUNTS_SQL)