From 9a266b0fd0f6d39a7cfc1942ee29cf265a99c075 Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 02:36:24 -0500 Subject: [PATCH] =?UTF-8?q?feat(llm):=20/api/policy=20=E2=80=94=20the=20do?= =?UTF-8?q?ssier=20sliced=20and=20cached=20per=20replica,=20ETag/304=20(le?= =?UTF-8?q?x=20S1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/llm/api.py | 4 + src/llm/policy.py | 260 ++++++++++++++++++++++++++++++ tests/llm/test_policy_api.py | 300 +++++++++++++++++++++++++++++++++++ 3 files changed, 564 insertions(+) create mode 100644 src/llm/policy.py create mode 100644 tests/llm/test_policy_api.py diff --git a/src/llm/api.py b/src/llm/api.py index a8967f2..82cffb4 100644 --- a/src/llm/api.py +++ b/src/llm/api.py @@ -49,6 +49,10 @@ app = FastAPI( lifespan=_lifespan, ) +from llm import policy as _policy # noqa: E402 — the lex policy API, /api/policy + +app.include_router(_policy.router) + _ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$") _MODES = {"auto", "timeline", "recent"} #: /search, /similar collection + kind query values — mirrors diff --git a/src/llm/policy.py b/src/llm/policy.py new file mode 100644 index 0000000..e98d255 --- /dev/null +++ b/src/llm/policy.py @@ -0,0 +1,260 @@ +"""The policy API: the stored dossiers served one view at a time (lex S1). + +``GET /api/policy/dossier/{section}?family=KEY`` returns one slice built by +``llm.policy_slices``; ``GET /api/policy/fact`` one fact with its home view. +A family's ``Dossier`` is read once per replica mtime and each section is +serialized once, so a request is a dict lookup. Slices are protected by a +content ETag. Failures are JSON ``{"detail": {...}}`` with a 4xx/503 status, +never a 500: the page renders the message. + +``load_slice``/``load_fact`` are the in-process seam the HTTP routes, the +tests and the golden check share. Module scope imports fastapi and stdlib +only (as ``llm.api`` does); the heavy modules load on first use. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import re +import threading +from collections import OrderedDict +from pathlib import Path +from typing import Any, Callable + +from fastapi import APIRouter, Depends, Header, Query, Response +from fastapi.responses import JSONResponse + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/policy") + +KEY_RE = re.compile(r"^[A-Z0-9][A-Z0-9@/_.-]{0,199}$") +CACHE_MAX = 16 + +#: family -> (replica mtime, Dossier, {section: serialized bytes}) +_CACHE: OrderedDict[str, tuple[int, Any, dict[str, bytes]]] = OrderedDict() +_LOCK = threading.Lock() + + +class PolicyError(Exception): + def __init__(self, status: int, detail: dict[str, Any]) -> None: + super().__init__(detail.get("message", "")) + self.status = status + self.detail = detail + + +def clear_cache() -> None: + with _LOCK: + _CACHE.clear() + + +def config_dep() -> Any: + from llm import config + + return config.load() + + +def bib_dep() -> Callable[[], Any]: + from conf import connect + + return connect.bib + + +def _status_of(family: str) -> str: + """Why a family has no dossier: curated (hand-written, not built yet), + planned (in the book), derived (a ``pfs.code_family`` key; opening the + replica refreshes the registry from it) or unknown.""" + from pfs.book import load_book + from pfs.curated import load_curated + from pfs.families import FAMILIES + + if family in load_curated(): + return "curated" + if family in load_book(): + return "planned" + if family in FAMILIES: + return "derived" + return "unknown" + + +def _tier(family: str) -> str: + from pfs.curated import load_curated + + return "reviewed" if family in load_curated() else "machine" + + +def _entry(cfg: Any, family: str) -> tuple[Any, dict[str, bytes]]: + from llm import evidence + from pfs.dossier import read_dossier + + path = evidence._replica_path(cfg) + if not Path(path).exists(): + raise PolicyError(503, {"message": "The data replica is not available."}) + mtime = evidence._mtime(path) + with _LOCK: + hit = _CACHE.get(family) + if hit is not None and hit[0] == mtime: + _CACHE.move_to_end(family) + return hit[1], hit[2] + try: + cur = evidence._connect(path).cursor() + except Exception as e: # noqa: BLE001 — duckdb raises several types + log.warning("policy: replica unavailable (%s): %s", path, e) + raise PolicyError(503, {"message": "The data replica is not available."}) + try: + try: + dossier = read_dossier(cur, family) + except Exception as e: # noqa: BLE001 + log.warning("policy: dossier read failed (%s): %s", family, e) + raise PolicyError( + 503, {"message": "The data replica is not available."} + ) from e + if dossier is None: + raise PolicyError( + 404, + { + "message": f"No dossier is built for {family} yet.", + "family": family, + "status": _status_of(family), + }, + ) + finally: + cur.close() + sections: dict[str, bytes] = {} + with _LOCK: + _CACHE[family] = (mtime, dossier, sections) + _CACHE.move_to_end(family) + while len(_CACHE) > CACHE_MAX: + _CACHE.popitem(last=False) + return dossier, sections + + +def _norm(family: str) -> str: + from pfs.families import normalize_key + + key = normalize_key(family) + if not KEY_RE.match(key): + raise PolicyError(400, {"message": "invalid family key", "family": family}) + return key + + +def _dump(obj: Any) -> bytes: + return json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode() + + +def _build(section: str, dossier: Any, family: str) -> dict[str, Any]: + from llm import policy_slices as ps + from pfs.book import load_book + + if section == "header": + return ps.header_slice( + dossier, tier=_tier(family), book=load_book().get(family) + ) + if section == "facts": + return ps.facts_index(dossier) + builder = { + "timeline": ps.timeline_slice, + "valuation": ps.valuation_slice, + "payment": ps.payment_slice, + "politics": ps.politics_slice, + }.get(section) + if builder is None: # in SECTIONS, but its builder is a later task's + raise PolicyError( + 404, {"message": "section not available yet", "section": section} + ) + return builder(dossier) + + +def slice_bytes( + cfg: Any, + family: str, + section: str, + *, + bib: Callable[[], Any] | None = None, +) -> bytes: + from llm.policy_slices import SECTIONS + + key = _norm(family) + if section not in SECTIONS: + raise PolicyError( + 404, {"message": "unknown section", "sections": list(SECTIONS)} + ) + dossier, sections = _entry(cfg, key) + with _LOCK: + body = sections.get(section) + if body is None: + body = _dump(_build(section, dossier, key)) + with _LOCK: + sections[section] = body + return body + + +def load_slice( + cfg: Any, + family: str, + section: str, + *, + bib: Callable[[], Any] | None = None, +) -> dict[str, Any]: + return json.loads(slice_bytes(cfg, family, section, bib=bib)) + + +def load_fact(cfg: Any, family: str, fact_id: str) -> dict[str, Any]: + from llm.policy_slices import fact_detail + + key = _norm(family) + dossier, _ = _entry(cfg, key) + out = fact_detail(dossier, fact_id) + if out is None: + raise PolicyError( + 404, {"message": "unknown fact", "family": key, "id": fact_id} + ) + return out + + +def _fail(e: PolicyError) -> JSONResponse: + return JSONResponse({"detail": e.detail}, status_code=e.status) + + +def _etag(body: bytes) -> str: + return '"%s"' % hashlib.sha1(body).hexdigest()[:20] + + +def _respond(body: bytes, if_none_match: str | None) -> Response: + etag = _etag(body) + headers = {"ETag": etag, "Cache-Control": "no-cache"} + sent = {t.strip().removeprefix("W/") for t in (if_none_match or "").split(",")} + if etag in sent or "*" in sent: + return Response(status_code=304, headers=headers) + return Response(body, media_type="application/json", headers=headers) + + +@router.get("/dossier/{section}") +def get_slice( + section: str, + family: str = Query(...), + if_none_match: str | None = Header(None), + cfg: Any = Depends(config_dep), + bib: Callable[[], Any] = Depends(bib_dep), +) -> Response: + try: + body = slice_bytes(cfg, family, section, bib=bib) + except PolicyError as e: + return _fail(e) + return _respond(body, if_none_match) + + +@router.get("/fact") +def get_fact( + family: str = Query(...), + id: str = Query(..., pattern=r"^F\d{1,6}$"), + if_none_match: str | None = Header(None), + cfg: Any = Depends(config_dep), +) -> Response: + try: + body = _dump(load_fact(cfg, family, id)) + except PolicyError as e: + return _fail(e) + return _respond(body, if_none_match) diff --git a/tests/llm/test_policy_api.py b/tests/llm/test_policy_api.py new file mode 100644 index 0000000..568f9c2 --- /dev/null +++ b/tests/llm/test_policy_api.py @@ -0,0 +1,300 @@ +"""The policy API (lex S1 Task 3): router, per-mtime cache, ETag/304, errors.""" + +from __future__ import annotations + +import dataclasses +import os + +import duckdb +import pytest +from fastapi.testclient import TestClient + +from llm import evidence, policy +from llm.api import app +from llm.config import LlmConfig +from llm.policy_slices import SECTIONS +from pfs.codetables import ensure_tables +from pfs.dossier import read_dossier, write_dossier +from tests.llm.policy_fixture import fixture_dossier + +CFG = LlmConfig( + ollama_hosts=("http://h1:11434",), + embed_model="e", + instruct_model="c", + embed_dim=768, + build_ann_index=False, + pg_host="x", + pg_port=5432, + pg_db="llm", + pg_user="llm", + duckdb_replica="/nonexistent/aco.ro.duckdb", +) +URL = "/api/policy/dossier/{}" +HV = {"family": "HOME-VISITS"} + + +def _write(path, *dossiers, sql=None): + # a republish replaces the file; drop the cached read-only handle first + if evidence._REPLICA is not None: + evidence._REPLICA[2].close() + evidence._REPLICA = None + con = duckdb.connect(str(path)) + ensure_tables(con) + for d in dossiers: + write_dossier(con, d) + if sql: + con.execute(sql) + con.close() + + +@pytest.fixture +def replica(tmp_path, monkeypatch): + # hermetic: opening the replica must not rewrite the global registry + monkeypatch.setattr(evidence, "refresh_from", lambda con: 0) + path = tmp_path / "aco.ro.duckdb" + _write(path, fixture_dossier()) + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(path) + ) + evidence._REPLICA = None + policy.clear_cache() + yield path + app.dependency_overrides.clear() + evidence._REPLICA = None + policy.clear_cache() + + +@pytest.fixture +def client(replica): + return TestClient(app) + + +def _count_reads(monkeypatch): + calls = [] + + def counting(con, family): + calls.append(family) + return read_dossier(con, family) + + monkeypatch.setattr("pfs.dossier.read_dossier", counting) + return calls + + +def test_section_ok_etag_and_304(client): + r = client.get(URL.format("header"), params=HV) + assert r.status_code == 200 + assert r.json()["family"] == "HOME-VISITS" + assert r.headers["cache-control"] == "no-cache" + assert r.headers["content-type"] == "application/json" + etag = r.headers["etag"] + assert etag.startswith('"') and len(etag) == 22 + r2 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": etag}) + assert r2.status_code == 304 and r2.content == b"" + assert r2.headers["etag"] == etag + r3 = client.get( + URL.format("header"), params=HV, headers={"If-None-Match": "W/" + etag} + ) + assert r3.status_code == 304 + r4 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": '"x"'}) + assert r4.status_code == 200 + r5 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": "*"}) + assert r5.status_code == 304 + + +@pytest.mark.parametrize("section", [s for s in SECTIONS if s != "sources"]) +def test_every_built_section_serves(client, section): + r = client.get(URL.format(section), params=HV) + assert r.status_code == 200 + assert r.json()["family"] == "HOME-VISITS" + + +def test_sources_section_not_built_yet(client): + r = client.get(URL.format("sources"), params=HV) + assert r.status_code == 404 + assert r.json()["detail"]["section"] == "sources" + + +def test_header_tier_and_book(client): + h = client.get(URL.format("header"), params=HV).json() + assert h["tier"] == "reviewed" + assert h["phase"] is not None + + +def test_machine_tier_for_uncurated_family(replica, client, monkeypatch): + _write(replica, dataclasses.replace(fixture_dossier(), family="ZZZ-MACH")) + r = client.get(URL.format("header"), params={"family": "zzz-mach"}) + assert r.status_code == 200 + h = r.json() + assert h["tier"] == "machine" and h["phase"] is None and h["domains"] == [] + + +def test_lowercase_family_normalized(client): + r = client.get(URL.format("header"), params={"family": "home-visits"}) + assert r.status_code == 200 + assert r.json()["family"] == "HOME-VISITS" + + +def test_bad_family_400(client): + for bad in ("a b", "../x", ""): + r = client.get(URL.format("header"), params={"family": bad}) + assert r.status_code == 400, bad + assert r.json()["detail"]["message"] == "invalid family key" + assert r.json()["detail"]["family"] == bad + assert client.get(URL.format("header")).status_code == 422 + + +def test_unknown_section_404(client): + r = client.get(URL.format("nope"), params=HV) + assert r.status_code == 404 + assert r.json()["detail"]["sections"] == list(SECTIONS) + + +def test_no_dossier_404_with_status(client, monkeypatch): + r = client.get(URL.format("header"), params={"family": "CCM"}) + assert r.status_code == 404 + d = r.json()["detail"] + assert d["status"] == "planned" and d["family"] == "CCM" + assert d["message"] == "No dossier is built for CCM yet." + + from pfs.curated import load_curated + + real = load_curated() + monkeypatch.setattr( + "pfs.curated.load_curated", lambda *a, **k: {**real, "ZZZ-CUR": 1} + ) + r = client.get(URL.format("header"), params={"family": "ZZZ-CUR"}) + assert r.status_code == 404 and r.json()["detail"]["status"] == "curated" + + r = client.get(URL.format("header"), params={"family": "ZZZ-NONE"}) + assert r.status_code == 404 and r.json()["detail"]["status"] == "unknown" + + +def test_derived_status(replica, client, monkeypatch): + from pfs.families import FAMILIES + + monkeypatch.setitem(FAMILIES, "ZZZ-REG", object()) + r = client.get(URL.format("header"), params={"family": "ZZZ-REG"}) + assert r.json()["detail"]["status"] == "derived" + + +def test_slash_family_key_is_valid(client): + r = client.get(URL.format("header"), params={"family": "A/D-TX-PROGRAM-HOUR"}) + assert r.status_code == 404 + assert r.json()["detail"]["family"] == "A/D-TX-PROGRAM-HOUR" + + +def test_replica_missing_503(tmp_path, client): + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(tmp_path / "gone.duckdb") + ) + r = client.get(URL.format("header"), params=HV) + assert r.status_code == 503 + assert r.json()["detail"]["message"] == "The data replica is not available." + + +def test_replica_unopenable_503(tmp_path, client): + bad = tmp_path / "bad.duckdb" + bad.write_bytes(b"this is not a duckdb file" * 100) + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(bad) + ) + r = client.get(URL.format("header"), params=HV) + assert r.status_code == 503 + + +def test_dossier_read_failure_503(client, monkeypatch): + def boom(con, family): + raise RuntimeError("io") + + monkeypatch.setattr("pfs.dossier.read_dossier", boom) + r = client.get(URL.format("header"), params=HV) + assert r.status_code == 503 + + +def test_table_missing_404(tmp_path, client): + path = tmp_path / "empty.duckdb" + con = duckdb.connect(str(path)) + con.execute("CREATE TABLE t (a INTEGER)") + con.close() + app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace( + CFG, duckdb_replica=str(path) + ) + r = client.get(URL.format("header"), params=HV) + assert r.status_code == 404 + assert r.json()["detail"]["family"] == "HOME-VISITS" + assert r.json()["detail"]["status"] == "curated" + + +def test_cache_reloads_after_replica_republish(replica, client): + r1 = client.get(URL.format("header"), params=HV) + assert r1.json()["built_at"] != "2026-11-14T00:00:00" + t = os.stat(replica).st_mtime + _write( + replica, dataclasses.replace(fixture_dossier(), built_at="2026-11-14T00:00:00") + ) + os.utime(replica, (t + 5, t + 5)) + r2 = client.get(URL.format("header"), params=HV) + assert r2.json()["built_at"] == "2026-11-14T00:00:00" + assert r2.headers["etag"] != r1.headers["etag"] + + +def test_cache_reads_dossier_once_per_mtime(client, monkeypatch): + calls = _count_reads(monkeypatch) + for s in ("header", "timeline", "payment", "header"): + assert client.get(URL.format(s), params=HV).status_code == 200 + assert calls == ["HOME-VISITS"] + + +def test_cache_bounded(replica, client, monkeypatch): + _write(replica, dataclasses.replace(fixture_dossier(), family="ZZZ-TWO")) + monkeypatch.setattr(policy, "CACHE_MAX", 1) + calls = _count_reads(monkeypatch) + client.get(URL.format("header"), params=HV) + client.get(URL.format("header"), params={"family": "ZZZ-TWO"}) + assert list(policy._CACHE) == ["ZZZ-TWO"] + client.get(URL.format("header"), params=HV) + assert calls == ["HOME-VISITS", "ZZZ-TWO", "HOME-VISITS"] + + +def test_fact_endpoint(client): + r = client.get("/api/policy/fact", params={**HV, "id": "F4"}) + assert r.status_code == 200 + assert r.json()["view"] == "valuation" + etag = r.headers["etag"] + r2 = client.get( + "/api/policy/fact", params={**HV, "id": "F4"}, headers={"If-None-Match": etag} + ) + assert r2.status_code == 304 + r = client.get("/api/policy/fact", params={**HV, "id": "F999"}) + assert r.status_code == 404 and r.json()["detail"]["message"] == "unknown fact" + assert client.get("/api/policy/fact", params={**HV, "id": "x"}).status_code == 422 + r = client.get("/api/policy/fact", params={"family": "CCM", "id": "F1"}) + assert r.status_code == 404 and r.json()["detail"]["status"] == "planned" + + +def test_load_slice_matches_http(replica, client): + cfg = dataclasses.replace(CFG, duckdb_replica=str(replica)) + for s in ("header", "politics", "facts"): + assert ( + policy.load_slice(cfg, "home-visits", s) + == client.get(URL.format(s), params=HV).json() + ) + with pytest.raises(policy.PolicyError) as ei: + policy.load_slice(cfg, "a b", "header") + assert ei.value.status == 400 + assert policy.load_fact(cfg, "HOME-VISITS", "F4")["view"] == "valuation" + + +def test_router_mounted_on_app(): + # this FastAPI nests included routers under app.routes; openapi flattens + paths = set(app.openapi()["paths"]) + assert "/api/policy/dossier/{section}" in paths + assert "/api/policy/fact" in paths + + +def test_default_dependencies(monkeypatch): + monkeypatch.setattr("llm.config.load", lambda: "cfg") + assert policy.config_dep() == "cfg" + from conf import connect + + assert policy.bib_dep() is connect.bib