From 3b2175cb801907441fbe32e457c69726192587f6 Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 02:44:21 -0500 Subject: [PATCH] =?UTF-8?q?feat(llm):=20lex=20contents=20and=20sources=20s?= =?UTF-8?q?lices;=20live=20acceptance=20=E2=80=94=20every=20fact=20served?= =?UTF-8?q?=20and=20cited=20(S1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/llm/policy.py | 169 +++++++++++++++++++--- src/llm/policy_slices.py | 191 ++++++++++++++++++++++++- tests/lex/fixtures/contents.json | 171 +++++++++++++++++++++++ tests/lex/fixtures/sources.json | 233 +++++++++++++++++++++++++++++++ tests/llm/policy_fixture.py | 26 ++++ tests/llm/test_policy_api.py | 145 ++++++++++++++++++- tests/llm/test_policy_live.py | 158 +++++++++++++++++++++ tests/llm/test_policy_slices.py | 151 +++++++++++++++++++- 8 files changed, 1217 insertions(+), 27 deletions(-) create mode 100644 tests/lex/fixtures/contents.json create mode 100644 tests/lex/fixtures/sources.json create mode 100644 tests/llm/test_policy_live.py diff --git a/src/llm/policy.py b/src/llm/policy.py index e98d255..3e9b55b 100644 --- a/src/llm/policy.py +++ b/src/llm/policy.py @@ -1,7 +1,9 @@ """The policy API: the stored dossiers served one view at a time (lex S1). ``GET /api/policy/dossier/{section}?family=KEY`` returns one slice built by -``llm.policy_slices``; ``GET /api/policy/fact`` one fact with its home view. +``llm.policy_slices``; ``GET /api/policy/fact`` one fact with its home view; +``GET /api/policy/contents`` the book's contents (every stored family plus the +planned ones). The ``sources`` section also reads the bibliography once. A family's ``Dossier`` is read once per replica mtime and each section is serialized once, so a request is a dict lookup. Slices are protected by a content ETag. Failures are JSON ``{"detail": {...}}`` with a 4xx/503 status, @@ -17,6 +19,7 @@ from __future__ import annotations import hashlib import json import logging +import os import re import threading from collections import OrderedDict @@ -36,6 +39,8 @@ CACHE_MAX = 16 #: family -> (replica mtime, Dossier, {section: serialized bytes}) _CACHE: OrderedDict[str, tuple[int, Any, dict[str, bytes]]] = OrderedDict() _LOCK = threading.Lock() +#: (replica mtime, serialized contents) +_CONTENTS: list[tuple[int, bytes]] = [] class PolicyError(Exception): @@ -48,6 +53,7 @@ class PolicyError(Exception): def clear_cache() -> None: with _LOCK: _CACHE.clear() + _CONTENTS.clear() def config_dep() -> Any: @@ -85,6 +91,22 @@ def _tier(family: str) -> str: return "reviewed" if family in load_curated() else "machine" +def _open(cfg: Any) -> tuple[int, Any]: + """The replica's mtime and a fresh cursor on its cached read-only handle; + the caller closes the cursor. Raises a 503 when it cannot be opened.""" + from llm import evidence + + path = evidence._replica_path(cfg) + if not Path(path).exists(): + raise PolicyError(503, {"message": "The data replica is not available."}) + mtime = evidence._mtime(path) + try: + return mtime, evidence._connect(path).cursor() + except Exception as e: # noqa: BLE001 — duckdb raises several types + log.warning("policy: replica unavailable (%s): %s", path, e) + raise PolicyError(503, {"message": "The data replica is not available."}) + + def _entry(cfg: Any, family: str) -> tuple[Any, dict[str, bytes]]: from llm import evidence from pfs.dossier import read_dossier @@ -92,17 +114,12 @@ def _entry(cfg: Any, family: str) -> tuple[Any, dict[str, bytes]]: path = evidence._replica_path(cfg) if not Path(path).exists(): raise PolicyError(503, {"message": "The data replica is not available."}) - mtime = evidence._mtime(path) with _LOCK: hit = _CACHE.get(family) - if hit is not None and hit[0] == mtime: + if hit is not None and hit[0] == evidence._mtime(path): _CACHE.move_to_end(family) return hit[1], hit[2] - try: - cur = evidence._connect(path).cursor() - except Exception as e: # noqa: BLE001 — duckdb raises several types - log.warning("policy: replica unavailable (%s): %s", path, e) - raise PolicyError(503, {"message": "The data replica is not available."}) + mtime, cur = _open(cfg) try: try: dossier = read_dossier(cur, family) @@ -144,7 +161,62 @@ def _dump(obj: Any) -> bytes: return json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode() -def _build(section: str, dossier: Any, family: str) -> dict[str, Any]: +def bib_info(store: Any, keys: Any) -> dict[str, Any]: + """What the bibliography knows of each key (lex D9): title and url from + ``items``; ``is_rule`` and ``full_text`` when the item has Federal + Register paragraph anchors; ``full_text`` when it has an attachment. + Keys with no item are left out.""" + from llm.policy_slices import SourceInfo + + ks = sorted(set(keys)) + if not ks: + return {} + con = store._con() # noqa: SLF001 — the pattern pfs.cite uses + out: dict[str, Any] = {} + for i in range(0, len(ks), 500): + chunk = ks[i : i + 500] + ph = ",".join("?" * len(chunk)) + rules = { + r[0] + for r in con.execute( + f"SELECT item_key FROM fr_anchor_docs WHERE item_key IN ({ph})", chunk + ) + } + attached = { + r[0] + for r in con.execute( + "SELECT DISTINCT i.key FROM attachments a " + f"JOIN items i ON i.id = a.item_id WHERE i.key IN ({ph})", + chunk, + ) + } + for key, title, url in con.execute( + f"SELECT key, title, url FROM items WHERE key IN ({ph})", chunk + ): + out[key] = SourceInfo( + title or "", url or "", key in rules or key in attached, key in rules + ) + return out + + +def _viewer_base(cfg: Any) -> str: + return f"https://llm.{os.environ.get('DOMAIN') or cfg.domain}/ui/pdf/" + + +def _sources(dossier: Any, cfg: Any, bib: Callable[[], Any] | None) -> dict[str, Any]: + from llm import policy_slices as ps + + store = (bib or bib_dep())() + try: + info = bib_info(store, ps.document_keys(dossier)) + finally: + store.close() + return ps.sources_slice(dossier, info, viewer_base=_viewer_base(cfg)) + + +def _build( + section: str, dossier: Any, family: str, cfg: Any, bib: Callable[[], Any] | None +) -> dict[str, Any]: from llm import policy_slices as ps from pfs.book import load_book @@ -154,17 +226,14 @@ def _build(section: str, dossier: Any, family: str) -> dict[str, Any]: ) if section == "facts": return ps.facts_index(dossier) - builder = { + if section == "sources": + return _sources(dossier, cfg, bib) + return { "timeline": ps.timeline_slice, "valuation": ps.valuation_slice, "payment": ps.payment_slice, "politics": ps.politics_slice, - }.get(section) - if builder is None: # in SECTIONS, but its builder is a later task's - raise PolicyError( - 404, {"message": "section not available yet", "section": section} - ) - return builder(dossier) + }[section](dossier) def slice_bytes( @@ -185,12 +254,66 @@ def slice_bytes( with _LOCK: body = sections.get(section) if body is None: - body = _dump(_build(section, dossier, key)) + body = _dump(_build(section, dossier, key, cfg, bib)) with _LOCK: sections[section] = body return body +def _machine_counts(cur: Any) -> tuple[int, int] | None: + from pfs.codetables import is_missing_table_error + + try: + row = cur.execute( + "SELECT count(*), count(*) FILTER (WHERE n = 1) FROM " + "(SELECT key, count(DISTINCT code) AS n FROM pfs.code_family GROUP BY key)" + ).fetchone() + except Exception as e: # noqa: BLE001 + if is_missing_table_error(e): + return None + raise + return (int(row[0]), int(row[1])) + + +def contents_bytes(cfg: Any) -> bytes: + from llm import policy_slices as ps + from pfs.book import load_book + from pfs.curated import load_curated + from pfs.dossier import dossier_families, read_dossier + + mtime, cur = _open(cfg) + with _LOCK: + if _CONTENTS and _CONTENTS[0][0] == mtime: + cur.close() + return _CONTENTS[0][1] + book = load_book() + try: + try: + entries = [] + for fam in dossier_families(cur): + d = read_dossier(cur, fam) + if d is not None: + entries.append( + ps.contents_entry(d, tier=_tier(fam), book=book.get(fam)) + ) + machine = _machine_counts(cur) + except Exception as e: # noqa: BLE001 + log.warning("policy: contents read failed: %s", e) + raise PolicyError( + 503, {"message": "The data replica is not available."} + ) from e + finally: + cur.close() + body = _dump(ps.contents_slice(entries, book, set(load_curated()), machine)) + with _LOCK: + _CONTENTS[:] = [(mtime, body)] + return body + + +def load_contents(cfg: Any) -> dict[str, Any]: + return json.loads(contents_bytes(cfg)) + + def load_slice( cfg: Any, family: str, @@ -246,6 +369,18 @@ def get_slice( return _respond(body, if_none_match) +@router.get("/contents") +def get_contents( + if_none_match: str | None = Header(None), + cfg: Any = Depends(config_dep), +) -> Response: + try: + body = contents_bytes(cfg) + except PolicyError as e: + return _fail(e) + return _respond(body, if_none_match) + + @router.get("/fact") def get_fact( family: str = Query(...), diff --git a/src/llm/policy_slices.py b/src/llm/policy_slices.py index 6e66072..4497f4a 100644 --- a/src/llm/policy_slices.py +++ b/src/llm/policy_slices.py @@ -11,12 +11,12 @@ from __future__ import annotations import functools import re -from dataclasses import asdict -from typing import Any +from dataclasses import asdict, dataclass +from typing import Any, Mapping from pfs.book import BookEntry from pfs.cite import Cite -from pfs.dossier import Dossier, Fact, stance_accuracy_line +from pfs.dossier import Dossier, Fact, compact, stance_accuracy_line __all__ = [ "ACCURACY_MARK", @@ -24,9 +24,15 @@ __all__ = [ "ROUND_OPTIONAL", "SECTIONS", "SECTION_OF_VIEW", + "SOURCE_GROUPS", + "SOURCE_GROUP_ORDER", + "SourceInfo", "VIEW_OF_KIND", "code_groups", + "contents_entry", + "contents_slice", "document_key", + "document_keys", "fact_detail", "fact_json", "facts_index", @@ -34,6 +40,7 @@ __all__ = [ "last_rule", "payment_slice", "politics_slice", + "sources_slice", "timeline_slice", "valuation_slice", ] @@ -306,3 +313,181 @@ def facts_index(d: Dossier) -> dict[str, Any]: def fact_detail(d: Dossier, fact_id: str) -> dict[str, Any] | None: f = d.facts.get(fact_id) return fact_json(f, d.stance_accuracy) if f else None + + +# ── sources and contents ───────────────────────────────────────────── + +LICENSED = "Books (licensed)" +SOURCE_GROUPS: dict[str, str] = { + "fr_para": "Federal Register rules", + "fr_page": "Federal Register rules", + "cfr": "Regulations (eCFR)", + "statute": "Statute", + "iom": "CMS manuals", + "mln": "CMS manuals", + "rvu_file": "RVU files", + "dataset": "Data files", + "cpt_book": LICENSED, + "study": "Studies", +} +SOURCE_GROUP_ORDER: list[str] = list(dict.fromkeys([*SOURCE_GROUPS.values(), "Other"])) +STATUS_FULL = "full text on file" +STATUS_BOOK = "licensed book: paraphrased, cited by page" +STATUS_LINK = "link only" + + +@dataclass(frozen=True) +class SourceInfo: + """What the bibliography knows of a document (lex D9).""" + + title: str + url: str + full_text: bool + is_rule: bool + + +def _documents(d: Dossier) -> dict[str, dict[str, Any]]: + """Every document the dossier cites (facts, also-cites, governance), keyed + by :func:`document_key`: its cites in first-seen order and the facts that + cite it.""" + docs: dict[str, dict[str, Any]] = {} + + def add(c: Cite, fact_id: str) -> None: + doc = docs.setdefault( + document_key(c), {"first": c, "cites": {}, "facts": set()} + ) + doc["cites"].setdefault( + (c.label, c.url, c.locator), + {"label": c.label, "url": c.url, "locator": c.locator}, + ) + if fact_id: + doc["facts"].add(fact_id) + + for f in d.facts.values(): + for c in (f.cite, *f.also): + add(c, f.id) + for g in d.governance: + add(g.cite, "") + return docs + + +def document_keys(d: Dossier) -> list[str]: + return list(_documents(d)) + + +def sources_slice( + d: Dossier, info: Mapping[str, SourceInfo], *, viewer_base: str +) -> dict[str, Any]: + out = [] + for key, doc in _documents(d).items(): + first: Cite = doc["first"] + group = SOURCE_GROUPS.get(first.kind, "Other") + known = info.get(key) + licensed = group == LICENSED + if licensed: + status = STATUS_BOOK + elif known is not None and known.full_text: + status = STATUS_FULL + else: + status = STATUS_LINK + if licensed: + url = "" + else: + url = (known.url if known else "") or first.url.split("#", 1)[0] + viewer = bool(known and known.full_text and not known.is_rule and not licensed) + out.append( + { + "key": key, + "group": group, + "title": (known.title if known else "") or first.label, + "url": url, + "status": status, + "viewer_url": viewer_base + key if viewer else "", + "cites": list(doc["cites"].values()), + "n_facts": len(doc["facts"]), + } + ) + rank = SOURCE_GROUP_ORDER.index + out.sort(key=lambda x: (rank(x["group"]), x["title"])) + counts = {g: 0 for g in SOURCE_GROUP_ORDER} + for x in out: + counts[x["group"]] += 1 + return _plain( + {**_head(d), "documents": out, "counts": {g: n for g, n in counts.items() if n}} + ) + + +def contents_entry(d: Dossier, *, tier: str, book: BookEntry | None) -> dict[str, Any]: + years = [r.year for r in d.payment] + return _plain( + { + "family": d.family, + "name": d.name, + "tier": tier, + "phase": book.phase if book else None, + "domains": list(book.domains) if book else [], + "codes_text": compact(d.codes), + "status": d.status, + "years_active": d.years_active, + "built_at": d.built_at, + "check_passed": d.check_passed, + "sections": { + "timeline": len(d.timeline), + "valuation": len(d.rounds), + "payment": f"CY{min(years)}–CY{max(years)}" if years else "", + "politics": len(d.stakeholders), + "governance": len(d.governance), + "sources": len(_documents(d)), + }, + "gaps": list(d.gaps), + "n_review": len(d.review), + "last_rule": last_rule(d), + "comment_period": None, + } + ) + + +def contents_slice( + entries: list[dict[str, Any]], + book: dict[str, BookEntry], + curated: set[str], + machine: tuple[int, int] | None, +) -> dict[str, Any]: + built = {e["family"] for e in entries} + + def status(key: str) -> str: + return "curated, not built" if key in curated else "planned" + + planned = [ + { + "family": key, + "name": b.name, + "phase": b.phase, + "domains": list(b.domains), + "codes": b.codes, + "status": status(key), + } + for key, b in book.items() + if key not in built + ] + planned += [ + { + "family": key, + "name": key, + "phase": None, + "domains": [], + "codes": "", + "status": status(key), + } + for key in sorted(curated - set(book) - built) + ] + return _plain( + { + "api": API_VERSION, + "families": sorted(entries, key=lambda e: (e["phase"] or 9, e["name"])), + "planned": planned, + "machine": ( + {"derived": machine[0], "single_code": machine[1]} if machine else None + ), + } + ) diff --git a/tests/lex/fixtures/contents.json b/tests/lex/fixtures/contents.json new file mode 100644 index 0000000..32ea88c --- /dev/null +++ b/tests/lex/fixtures/contents.json @@ -0,0 +1,171 @@ +{ + "api": 1, + "families": [ + { + "family": "HOME-VISITS", + "name": "Home or Residence Services", + "tier": "reviewed", + "phase": 1, + "domains": [ + "home-based care", + "primary care" + ], + "codes_text": "99341 and 99343", + "status": "active", + "years_active": "2007–2027", + "built_at": "2026-11-13T05:23:37", + "check_passed": true, + "sections": { + "timeline": 3, + "valuation": 2, + "payment": "CY2022–CY2027", + "politics": 1, + "governance": 1, + "sources": 12 + }, + "gaps": [ + "Invented gap one.", + "Invented gap two." + ], + "n_review": 3, + "last_rule": { + "year": 2023, + "label": "CY2023 PFS final PETM225D ¶1694", + "url": "https://example.test/fr/PETM225D#p1694" + }, + "comment_period": null + } + ], + "planned": [ + { + "family": "OFFICE-EM", + "name": "Office/outpatient E/M", + "phase": 1, + "domains": [ + "primary care" + ], + "codes": "", + "status": "planned" + }, + { + "family": "G2211-B1-B2", + "name": "G2211 and modifiers B1/B2", + "phase": 1, + "domains": [ + "primary care" + ], + "codes": "G2211, B1, B2", + "status": "planned" + }, + { + "family": "CCM", + "name": "Chronic care management (CCM)", + "phase": 1, + "domains": [ + "care management and coordination", + "primary care" + ], + "codes": "99490, 99491, 99487–99489, 99437, 99439, G2058", + "status": "planned" + }, + { + "family": "PCM", + "name": "Principal care management (PCM)", + "phase": 1, + "domains": [ + "care management and coordination" + ], + "codes": "99424–99427, G2064, G2065", + "status": "planned" + }, + { + "family": "TCM", + "name": "Transitional care management (TCM)", + "phase": 1, + "domains": [ + "care management and coordination", + "primary care" + ], + "codes": "99495–99496", + "status": "planned" + }, + { + "family": "APCM", + "name": "Advanced primary care management (APCM)", + "phase": 1, + "domains": [ + "primary care", + "care management and coordination", + "APM linkage" + ], + "codes": "G0556–G0558", + "status": "planned" + }, + { + "family": "CPO-HH", + "name": "Care plan oversight and home health certification", + "phase": 1, + "domains": [ + "home-based care", + "care management and coordination" + ], + "codes": "G0179–G0182, 99339, 99340, 99374–99380", + "status": "planned" + }, + { + "family": "BHI-COCM", + "name": "Behavioral health integration and CoCM", + "phase": 1, + "domains": [ + "behavioral health integration" + ], + "codes": "99484, 99492–99494", + "status": "planned" + }, + { + "family": "RPM-RTM", + "name": "Remote physiologic and therapeutic monitoring (RPM/RTM)", + "phase": 1, + "domains": [ + "telehealth and virtual care", + "care management and coordination" + ], + "codes": "", + "status": "planned" + }, + { + "family": "ACP", + "name": "Advance care planning (ACP)", + "phase": 1, + "domains": [ + "primary care" + ], + "codes": "99497–99498", + "status": "planned" + }, + { + "family": "PROLONGED", + "name": "Prolonged services", + "phase": 1, + "domains": [ + "primary care" + ], + "codes": "99354–99357, 99417, 99418, G0316–G0318", + "status": "planned" + }, + { + "family": "VIRTUAL", + "name": "Virtual check-ins and communication-technology services", + "phase": 1, + "domains": [ + "telehealth and virtual care" + ], + "codes": "G2010, G2012, G2252", + "status": "planned" + } + ], + "machine": { + "derived": 8662, + "single_code": 7858 + } +} diff --git a/tests/lex/fixtures/sources.json b/tests/lex/fixtures/sources.json new file mode 100644 index 0000000..5d8536c --- /dev/null +++ b/tests/lex/fixtures/sources.json @@ -0,0 +1,233 @@ +{ + "api": 1, + "family": "HOME-VISITS", + "built_at": "2026-11-13T05:23:37", + "documents": [ + { + "key": "MZ24MX5S", + "group": "Federal Register rules", + "title": "CY2007 PFS final MZ24MX5S ¶878", + "url": "https://example.test/fr/MZ24MX5S", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "CY2007 PFS final MZ24MX5S ¶878", + "url": "https://example.test/fr/MZ24MX5S#p878", + "locator": "MZ24MX5S ¶878" + }, + { + "label": "CY2022 PFS final MZ24MX5S ¶877", + "url": "https://example.test/fr/MZ24MX5S#p877", + "locator": "MZ24MX5S ¶877" + } + ], + "n_facts": 2 + }, + { + "key": "PETM225D", + "group": "Federal Register rules", + "title": "Invented final rule", + "url": "https://example.test/fr/PETM225D", + "status": "full text on file", + "viewer_url": "", + "cites": [ + { + "label": "CY2023 PFS final PETM225D ¶1672", + "url": "https://example.test/fr/PETM225D#p1672", + "locator": "PETM225D ¶1672" + }, + { + "label": "CY2023 PFS final PETM225D ¶1676", + "url": "https://example.test/fr/PETM225D#p1676", + "locator": "PETM225D ¶1676" + }, + { + "label": "CY2023 PFS final PETM225D ¶1679", + "url": "https://example.test/fr/PETM225D#p1679", + "locator": "PETM225D ¶1679" + }, + { + "label": "CY2023 PFS final PETM225D ¶1694", + "url": "https://example.test/fr/PETM225D#p1694", + "locator": "PETM225D ¶1694" + }, + { + "label": "CY2021 PFS final PETM225D ¶1600", + "url": "https://example.test/fr/PETM225D#p1600", + "locator": "PETM225D ¶1600" + } + ], + "n_facts": 5 + }, + { + "key": "https://example.test/statute/9999", + "group": "Statute", + "title": "Act section 9999", + "url": "https://example.test/statute/9999", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "Act section 9999", + "url": "https://example.test/statute/9999", + "locator": "Act section 9999" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/rvu/2022", + "group": "RVU files", + "title": "CY2022 RVU file", + "url": "https://example.test/rvu/2022", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "CY2022 RVU file", + "url": "https://example.test/rvu/2022", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/rvu/2023", + "group": "RVU files", + "title": "CY2023 RVU file", + "url": "https://example.test/rvu/2023", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "CY2023 RVU file", + "url": "https://example.test/rvu/2023", + "locator": "" + } + ], + "n_facts": 2 + }, + { + "key": "https://example.test/rvu/2027", + "group": "RVU files", + "title": "CY2027 RVU file", + "url": "https://example.test/rvu/2027", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "CY2027 RVU file", + "url": "https://example.test/rvu/2027", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/impact", + "group": "Data files", + "title": "Impact table", + "url": "https://example.test/impact", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "Impact table", + "url": "https://example.test/impact", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/money", + "group": "Data files", + "title": "Money table", + "url": "https://example.test/money", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "Money table", + "url": "https://example.test/money", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/seat", + "group": "Data files", + "title": "Seat roster", + "url": "https://example.test/seat", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "Seat roster", + "url": "https://example.test/seat", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "BOOK2019", + "group": "Books (licensed)", + "title": "CPT 2019 p. 20", + "url": "", + "status": "licensed book: paraphrased, cited by page", + "viewer_url": "", + "cites": [ + { + "label": "CPT 2019 p. 20", + "url": "", + "locator": "CPT 2019 p. 20" + } + ], + "n_facts": 1 + }, + { + "key": "ART1", + "group": "Studies", + "title": "Invented article", + "url": "https://example.test/study", + "status": "full text on file", + "viewer_url": "https://llm.example.test/ui/pdf/ART1", + "cites": [ + { + "label": "Invented study", + "url": "https://example.test/study", + "locator": "" + } + ], + "n_facts": 1 + }, + { + "key": "https://example.test/comment/1", + "group": "Other", + "title": "Comment 1", + "url": "https://example.test/comment/1", + "status": "link only", + "viewer_url": "", + "cites": [ + { + "label": "Comment 1", + "url": "https://example.test/comment/1", + "locator": "" + } + ], + "n_facts": 2 + } + ], + "counts": { + "Federal Register rules": 2, + "Statute": 1, + "RVU files": 3, + "Data files": 3, + "Books (licensed)": 1, + "Studies": 1, + "Other": 1 + } +} diff --git a/tests/llm/policy_fixture.py b/tests/llm/policy_fixture.py index 989eb77..b111048 100644 --- a/tests/llm/policy_fixture.py +++ b/tests/llm/policy_fixture.py @@ -354,13 +354,30 @@ def fixture_dossier(*, stance_accuracy: str = ACC) -> Dossier: ) +def fixture_info() -> dict: + """The bib facts behind the fixture's sources (invented titles).""" + from llm.policy_slices import SourceInfo + + return { + "PETM225D": SourceInfo( + "Invented final rule", "https://example.test/fr/PETM225D", True, True + ), + "ART1": SourceInfo( + "Invented article", "https://example.test/study", True, False + ), + } + + def dossier_fixture_slices(d: Dossier | None = None) -> dict[str, dict]: """Every per-section slice of the fixture, keyed by fixture file stem.""" from llm.policy_slices import ( + contents_entry, + contents_slice, facts_index, header_slice, payment_slice, politics_slice, + sources_slice, timeline_slice, valuation_slice, ) @@ -374,4 +391,13 @@ def dossier_fixture_slices(d: Dossier | None = None) -> dict[str, dict]: "payment": payment_slice(d), "politics": politics_slice(d), "facts": facts_index(d), + "sources": sources_slice( + d, fixture_info(), viewer_base="https://llm.example.test/ui/pdf/" + ), + "contents": contents_slice( + [contents_entry(d, tier="reviewed", book=load_book()["HOME-VISITS"])], + load_book(), + {"HOME-VISITS"}, + (8662, 7858), + ), } diff --git a/tests/llm/test_policy_api.py b/tests/llm/test_policy_api.py index 568f9c2..6c1dbaa 100644 --- a/tests/llm/test_policy_api.py +++ b/tests/llm/test_policy_api.py @@ -12,7 +12,7 @@ from fastapi.testclient import TestClient from llm import evidence, policy from llm.api import app from llm.config import LlmConfig -from llm.policy_slices import SECTIONS +from llm.policy_slices import SECTIONS, SourceInfo from pfs.codetables import ensure_tables from pfs.dossier import read_dossier, write_dossier from tests.llm.policy_fixture import fixture_dossier @@ -101,17 +101,128 @@ def test_section_ok_etag_and_304(client): assert r5.status_code == 304 -@pytest.mark.parametrize("section", [s for s in SECTIONS if s != "sources"]) +@pytest.mark.parametrize("section", SECTIONS) def test_every_built_section_serves(client, section): r = client.get(URL.format(section), params=HV) assert r.status_code == 200 assert r.json()["family"] == "HOME-VISITS" -def test_sources_section_not_built_yet(client): - r = client.get(URL.format("sources"), params=HV) - assert r.status_code == 404 - assert r.json()["detail"]["section"] == "sources" +def _store(): + from bib.store import Store + + st = Store(":memory:") + con = st._con() + con.executescript( + """ + INSERT INTO items (id, key, item_type, title, url) VALUES + (1, 'PETM225D', 'regulation', 'Final rule PETM', 'https://fr.test/PETM225D'), + (2, 'ART1', 'report', 'The article', 'https://art.test/1'), + (3, 'BARE', 'report', '', ''); + INSERT INTO fr_anchor_docs + (item_key, document_number, html_url, start_page, end_page, fr_volume) + VALUES ('PETM225D', '2022-1', 'https://fr.test', 1, 2, 87); + INSERT INTO attachments (item_id, key, filename) VALUES (2, 'ATT1', 'a.pdf'); + """ + ) + con.commit() + return st + + +def test_bib_info_reads_status(): + st = _store() + info = policy.bib_info(st, ["PETM225D", "ART1", "BARE", "NOPE", "ART1"]) + assert info["PETM225D"] == SourceInfo( + "Final rule PETM", "https://fr.test/PETM225D", True, True + ) + assert info["ART1"] == SourceInfo("The article", "https://art.test/1", True, False) + assert info["BARE"] == SourceInfo("", "", False, False) + assert "NOPE" not in info + assert policy.bib_info(st, []) == {} + + +def test_sources_endpoint_uses_bib_once(client, monkeypatch): + calls = [] + + def factory(): + calls.append(1) + return _store() + + app.dependency_overrides[policy.bib_dep] = lambda: factory + monkeypatch.setenv("DOMAIN", "lex.test") + r1 = client.get(URL.format("sources"), params=HV) + assert r1.status_code == 200 + r2 = client.get(URL.format("sources"), params=HV) + assert r2.content == r1.content and len(calls) == 1 + docs = {x["key"]: x for x in r1.json()["documents"]} + assert docs["PETM225D"]["status"] == "full text on file" + assert docs["ART1"]["viewer_url"] == "https://llm.lex.test/ui/pdf/ART1" + assert docs["MZ24MX5S"]["status"] == "link only" + assert docs["BOOK2019"]["url"] == "" + + +def test_sources_viewer_base_defaults_to_config_domain(replica, monkeypatch): + monkeypatch.delenv("DOMAIN", raising=False) + cfg = dataclasses.replace(CFG, duckdb_replica=str(replica), domain="cfg.test") + body = policy.load_slice(cfg, "HOME-VISITS", "sources", bib=_store) + art = next(x for x in body["documents"] if x["key"] == "ART1") + assert art["viewer_url"] == "https://llm.cfg.test/ui/pdf/ART1" + + +def test_contents_endpoint_and_machine_counts_missing_table(replica, client): + _write(replica, fixture_dossier(), sql="DROP TABLE pfs.code_family") + policy.clear_cache() + r = client.get("/api/policy/contents") + assert r.status_code == 200 + c = r.json() + assert [e["family"] for e in c["families"]] == ["HOME-VISITS"] + assert c["families"][0]["tier"] == "reviewed" + assert c["families"][0]["n_review"] == 3 + assert c["machine"] is None # no pfs.code_family in the replica + assert len(c["planned"]) == 12 + etag = r.headers["etag"] + r2 = client.get("/api/policy/contents", headers={"If-None-Match": etag}) + assert r2.status_code == 304 + + +def test_contents_machine_counts_and_cache(replica, client, monkeypatch): + con = duckdb.connect(str(replica)) + con.execute( + "INSERT INTO pfs.code_family (key, code) VALUES " + "('A','1'),('A','2'),('B','3'),('C','4')" + ) + con.close() + evidence._REPLICA = None + policy.clear_cache() + assert client.get("/api/policy/contents").json()["machine"] == { + "derived": 3, + "single_code": 2, + } + calls = _count_reads(monkeypatch) + client.get("/api/policy/contents") + client.get("/api/policy/contents") + assert calls == [] # served from the cache, no dossier re-read + + +def test_contents_machine_tier_and_replica_errors(tmp_path, replica, client): + other = fixture_dossier() + other = dataclasses.replace(other, family="ZZZ-MACHINE", name="Machine one") + _write(replica, fixture_dossier(), other) + policy.clear_cache() + c = client.get("/api/policy/contents").json() + tiers = {e["family"]: e["tier"] for e in c["families"]} + assert tiers == {"HOME-VISITS": "reviewed", "ZZZ-MACHINE": "machine"} + # a missing replica is a 503, an unopenable one too + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(tmp_path / "gone.duckdb") + ) + assert client.get("/api/policy/contents").status_code == 503 + bad = tmp_path / "bad.duckdb" + bad.write_bytes(b"not a database") + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(bad) + ) + assert client.get("/api/policy/contents").status_code == 503 def test_header_tier_and_book(client): @@ -298,3 +409,25 @@ def test_default_dependencies(monkeypatch): from conf import connect assert policy.bib_dep() is connect.bib + + +def test_contents_read_failure_503(client, monkeypatch): + def boom(con, family): + raise RuntimeError("io") + + monkeypatch.setattr("pfs.dossier.read_dossier", boom) + assert client.get("/api/policy/contents").status_code == 503 + + +def test_machine_counts_reraises_other_errors(): + class Cur: + def execute(self, *a): + raise RuntimeError("io") + + with pytest.raises(RuntimeError): + policy._machine_counts(Cur()) + + +def test_load_contents_matches_http(replica, client): + cfg = dataclasses.replace(CFG, duckdb_replica=str(replica)) + assert policy.load_contents(cfg) == client.get("/api/policy/contents").json() diff --git a/tests/llm/test_policy_live.py b/tests/llm/test_policy_live.py new file mode 100644 index 0000000..52afc61 --- /dev/null +++ b/tests/llm/test_policy_live.py @@ -0,0 +1,158 @@ +"""Live acceptance of the policy API against the real replica and bib (lex S1 +Task 4). Skipped only when the data files are absent; with the data present +every check must pass.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path + +import duckdb +import pytest + +ROOT = Path(__file__).resolve().parents[2] +REPLICA = ROOT / "data" / "replica" / "aco.ro.duckdb" +BIB = ROOT / "data" / "bib.sqlite" + +pytestmark = pytest.mark.skipif( + not (REPLICA.exists() and BIB.exists()), reason="live replica / bib not present" +) + +FAMILY = "HOME-VISITS" +VIEW_SECTIONS = ("header", "timeline", "valuation", "payment", "politics") + + +@pytest.fixture(scope="module") +def cfg(): + from llm import config + + return config.load() + + +@pytest.fixture(scope="module") +def bib(): + from bib.store import Store + + return lambda: Store(str(BIB)) + + +@pytest.fixture(scope="module") +def slices(cfg, bib): + from llm import policy + from llm.policy_slices import SECTIONS + + policy.clear_cache() + return {s: policy.load_slice(cfg, FAMILY, s, bib=bib) for s in SECTIONS} + + +@pytest.fixture(scope="module") +def dossier(cfg): + from llm import evidence + from pfs.dossier import read_dossier + + cur = evidence._connect(evidence._replica_path(cfg)).cursor() + try: + d = read_dossier(cur, FAMILY) + finally: + cur.close() + assert d is not None + return d + + +def test_live_every_fact_served_in_home_view(slices, dossier): + from llm.policy_slices import SECTION_OF_VIEW + + ids = {f["id"] for f in slices["facts"]["facts"]} + assert ids == set(dossier.facts) + for f in slices["facts"]["facts"]: + section = SECTION_OF_VIEW[f["view"]] + assert f["id"] in slices[section]["facts"], (f["id"], f["view"]) + + +def _cites(slices): + for sect in VIEW_SECTIONS: + for f in slices[sect]["facts"].values(): + yield f["id"], f["cite"] + for c in f["also"]: + yield f["id"], c + + +def test_live_citations_resolve(slices, bib): + store = bib() + con = store._con() + n = 0 + for fid, c in _cites(slices): + n += 1 + url = c["url"] + if not url: + assert c["kind"] == "cpt_book" and c["locator"], (fid, c["kind"]) + continue + assert url.startswith("https://") and not re.search(r"\s", url), (fid, url) + m = re.search(r"#p-(\d+)$", url) + if c["kind"] == "fr_para" and m: + row = con.execute( + "SELECT 1 FROM fr_anchors WHERE item_key = ? AND p_id = ?", + (c["item_key"], int(m.group(1))), + ).fetchone() + assert row is not None, (fid, c["item_key"], m.group(1)) + assert n > 0 + + +def test_live_no_cpt_descriptor_text(cfg, slices, dossier, bib): + from llm import policy + + codes = sorted({r["code"] for r in slices["payment"]["rows"]}) + con = duckdb.connect(str(REPLICA), read_only=True) + try: + rows = [] + for table in ("pfs.rvu", "pfs.rvu_proposed"): + rows += con.execute( + f"SELECT DISTINCT hcpcs, description FROM {table} " + "WHERE hcpcs IN (SELECT unnest(?::VARCHAR[])) " + "AND length(description) >= 10", + [codes], + ).fetchall() + finally: + con.close() + assert rows, "no descriptors found to guard against" + contents = policy.load_contents(cfg) + blob = re.sub( + r"\s+", " ", json.dumps([slices, contents], ensure_ascii=False) + ).casefold() + bad = sorted( + {code for code, text in rows if re.sub(r"\s+", " ", text).casefold() in blob} + ) + assert not bad, f"descriptor text served for codes {bad}" + + +def test_live_politics_counts_carry_accuracy(slices): + from llm.policy_slices import ACCURACY_MARK + + pol = slices["politics"] + assert pol["panels"] + for p in pol["panels"]: + assert ACCURACY_MARK in p["accuracy"], p["docket"] + splits = [f for f in pol["facts"].values() if f["kind"] == "stance_split"] + assert splits + for f in splits: + assert ACCURACY_MARK in f["text"] + f.get("accuracy", ""), f["id"] + + +def test_live_review_items_all_served(slices, dossier): + assert len(slices["header"]["review"]) == len(dossier.review) > 0 + + +def test_live_sources_and_contents_shape(slices, cfg): + from llm import policy + + s = slices["sources"] + assert s["documents"] and sum(s["counts"].values()) == len(s["documents"]) + for doc in s["documents"]: + if doc["group"] == "Books (licensed)": + assert doc["url"] == "" and doc["viewer_url"] == "" + if doc["viewer_url"]: + assert doc["status"] == "full text on file" + c = policy.load_contents(cfg) + assert FAMILY in {e["family"] for e in c["families"]} + assert c["machine"] is None or c["machine"]["derived"] > 0 diff --git a/tests/llm/test_policy_slices.py b/tests/llm/test_policy_slices.py index dca4f26..679c66c 100644 --- a/tests/llm/test_policy_slices.py +++ b/tests/llm/test_policy_slices.py @@ -15,8 +15,13 @@ from llm.policy_slices import ( ROUND_OPTIONAL, SECTION_OF_VIEW, SECTIONS, + SOURCE_GROUP_ORDER, + SOURCE_GROUPS, VIEW_OF_KIND, + SourceInfo, code_groups, + contents_entry, + contents_slice, document_key, fact_detail, fact_json, @@ -25,13 +30,19 @@ from llm.policy_slices import ( last_rule, payment_slice, politics_slice, + sources_slice, timeline_slice, valuation_slice, ) from pfs.book import load_book from pfs.cite import Cite from pfs.dossier import FACT_KINDS, stance_accuracy_line -from tests.llm.policy_fixture import ACC, dossier_fixture_slices, fixture_dossier +from tests.llm.policy_fixture import ( + ACC, + dossier_fixture_slices, + fixture_dossier, + fixture_info, +) FIXTURES = Path(__file__).resolve().parents[1] / "lex" / "fixtures" @@ -241,6 +252,8 @@ def test_js_fixtures_match_api(d): "payment", "politics", "facts", + "sources", + "contents", } for name, body in slices.items(): assert json.loads((FIXTURES / f"{name}.json").read_text()) == body @@ -249,3 +262,139 @@ def test_js_fixtures_match_api(d): def test_fixture_has_no_real_descriptor_text(d): blob = json.dumps(dossier_fixture_slices(d)).lower() assert "invented" in blob + + +VB = "https://llm.fhirworx.io/ui/pdf/" + + +def _src(d, info=None): + return sources_slice(d, fixture_info() if info is None else info, viewer_base=VB) + + +def test_source_groups_map(): + assert SOURCE_GROUPS["fr_para"] == SOURCE_GROUPS["fr_page"] + assert SOURCE_GROUPS["fr_para"] == "Federal Register rules" + assert SOURCE_GROUPS["cfr"] == "Regulations (eCFR)" + assert SOURCE_GROUPS["statute"] == "Statute" + assert SOURCE_GROUPS["iom"] == SOURCE_GROUPS["mln"] == "CMS manuals" + assert SOURCE_GROUPS["rvu_file"] == "RVU files" + assert SOURCE_GROUPS["dataset"] == "Data files" + assert SOURCE_GROUPS["cpt_book"] == "Books (licensed)" + assert SOURCE_GROUPS["study"] == "Studies" + assert SOURCE_GROUP_ORDER[0] == "Federal Register rules" + assert SOURCE_GROUP_ORDER[-1] == "Other" + assert set(SOURCE_GROUPS.values()) <= set(SOURCE_GROUP_ORDER) + + +def test_sources_slice_groups_documents(d): + s = _src(d) + fr = next(x for x in s["documents"] if x["key"] == "PETM225D") + assert fr["group"] == "Federal Register rules" + assert fr["status"] == "full text on file" and fr["viewer_url"] == "" + assert fr["title"] == "Invented final rule" + assert fr["url"] == "https://example.test/fr/PETM225D" + # F3, F4, F6, F9 and the unplaced F17 + assert len(fr["cites"]) == 5 and fr["n_facts"] == 5 + assert set(fr["cites"][0]) == {"label", "url", "locator"} + book = next(x for x in s["documents"] if x["group"] == "Books (licensed)") + assert book["status"] == "licensed book: paraphrased, cited by page" + assert book["viewer_url"] == "" and book["url"] == "" + assert book["title"] == "CPT 2019 p. 20" and book["n_facts"] == 1 + art = next(x for x in s["documents"] if x["key"] == "ART1") + assert art["viewer_url"] == VB + "ART1" and art["status"] == "full text on file" + assert art["group"] == "Studies" + assert ( + art["n_facts"] == 1 and len(art["cites"]) == 1 + ) # F16 and governance share one cite + assert list(s["counts"]) == [g for g in SOURCE_GROUP_ORDER if g in s["counts"]] + assert sum(s["counts"].values()) == len(s["documents"]) + keys = [(SOURCE_GROUP_ORDER.index(x["group"]), x["title"]) for x in s["documents"]] + assert keys == sorted(keys) + + +def test_sources_slice_link_only_and_url_documents(d): + s = _src(d, {}) + mz = next(x for x in s["documents"] if x["key"] == "MZ24MX5S") + assert mz["status"] == "link only" and mz["viewer_url"] == "" + assert mz["title"].startswith("CY2007 PFS final") and mz["n_facts"] == 2 + assert mz["url"] == "https://example.test/fr/MZ24MX5S" + # a cite without a bib item is keyed by its url without the fragment + st = next( + x for x in s["documents"] if x["key"] == "https://example.test/statute/9999" + ) + assert st["group"] == "Statute" and st["status"] == "link only" + assert st["url"] == "https://example.test/statute/9999" + # an unknown kind falls into Other + odd = replace(d.facts["F13"], cite=Cite("weird", "Odd thing", "", locator="x")) + s2 = _src(replace(d, facts={"F13": odd}), {}) + assert ( + next(x for x in s2["documents"] if x["key"] == "Odd thing")["group"] == "Other" + ) + # a non-rule, non-attached bib item is link only + s3 = _src(d, {"ART1": SourceInfo("A", "u", False, False)}) + art = next(x for x in s3["documents"] if x["key"] == "ART1") + assert art["status"] == "link only" and art["viewer_url"] == "" + + +def test_sources_licensed_book_never_linked(d): + info = {"BOOK2019": SourceInfo("Book", "https://x.test/b", True, False)} + book = next(x for x in _src(d, info)["documents"] if x["key"] == "BOOK2019") + assert book["url"] == "" and book["viewer_url"] == "" + assert book["status"].startswith("licensed book") + + +def test_contents_entry_and_slice(d): + book = load_book() + e = contents_entry(d, tier="reviewed", book=book["HOME-VISITS"]) + assert e["family"] == "HOME-VISITS" and e["name"] == d.name + assert e["tier"] == "reviewed" and e["phase"] == book["HOME-VISITS"].phase + assert e["codes_text"] == "99341 and 99343" + assert e["n_review"] == 3 and e["gaps"] == d.gaps + assert e["check_passed"] is True and e["comment_period"] is None + assert e["sections"] == { + "timeline": 3, + "valuation": 2, + "payment": "CY2022–CY2027", + "politics": 1, + "governance": 1, + "sources": len(_src(d)["documents"]), + } + assert e["last_rule"]["label"] == d.facts["F9"].cite.label + bare = contents_entry(replace(d, payment=[]), tier="machine", book=None) + assert bare["phase"] is None and bare["domains"] == [] + assert bare["sections"]["payment"] == "" + c = contents_slice([e], book, {"HOME-VISITS"}, (8662, 7858)) + assert [x["family"] for x in c["families"]] == ["HOME-VISITS"] + assert [p["family"] for p in c["planned"]][:2] == ["OFFICE-EM", "G2211-B1-B2"] + assert len(c["planned"]) == len(book) - 1 == 12 + assert c["machine"] == {"derived": 8662, "single_code": 7858} + assert c["planned"][0]["status"] == "planned" + assert set(c["planned"][0]) == { + "family", + "name", + "phase", + "domains", + "codes", + "status", + } + + +def test_contents_slice_order_curated_extras_and_no_machine(d): + book = load_book() + a = contents_entry(d, tier="reviewed", book=None) + b = {**a, "family": "ZZ", "name": "Alpha", "phase": 2} + c = {**a, "family": "YY", "name": "Beta", "phase": None} + out = contents_slice([c, b, a], book, {"HOME-VISITS", "CURATED-ONLY"}, None) + assert [x["family"] for x in out["families"]] == ["ZZ", "YY", "HOME-VISITS"] + assert out["machine"] is None + last = out["planned"][-1] + assert last["family"] == "CURATED-ONLY" + assert last["status"] == "curated, not built" and last["phase"] is None + assert all(p["family"] != "HOME-VISITS" for p in out["planned"]) + book2 = {**book} + only = contents_slice([], book2, {"OFFICE-EM"}, None) + assert ( + next(p for p in only["planned"] if p["family"] == "OFFICE-EM")["status"] + == "curated, not built" + ) + assert len(only["planned"]) == len(book)