feat(llm): /api/policy — the dossier sliced and cached per replica, ETag/304 (lex S1)
Some checks failed
CI / lint (push) Successful in 1m24s
CI / notebooks-smoke (push) Successful in 2m8s
Deploy / changes (push) Successful in 21s
CI / test (push) Failing after 5m27s
Infra CI / notebooks (push) Successful in 1m38s
Infra CI / zotero (push) Successful in 38s
Infra CI / docs (push) Successful in 48s
Infra CI / api (push) Successful in 3m4s
Infra CI / bcda (push) Successful in 4m6s
Infra CI / mc (push) Successful in 35s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 2m9s
Deploy / api (push) Successful in 1m40s
Deploy / bcda (push) Successful in 2m6s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 2m0s
Deploy / report (push) Successful in 22s

This commit is contained in:
kert
2026-11-13 02:36:24 -05:00
parent 3a18f63841
commit 9a266b0fd0
3 changed files with 564 additions and 0 deletions

View File

@@ -49,6 +49,10 @@ app = FastAPI(
lifespan=_lifespan,
)
from llm import policy as _policy # noqa: E402 — the lex policy API, /api/policy
app.include_router(_policy.router)
_ISO_DATE = re.compile(r"^\d{4}-\d{2}-\d{2}$")
_MODES = {"auto", "timeline", "recent"}
#: /search, /similar collection + kind query values — mirrors

260
src/llm/policy.py Normal file
View File

@@ -0,0 +1,260 @@
"""The policy API: the stored dossiers served one view at a time (lex S1).
``GET /api/policy/dossier/{section}?family=KEY`` returns one slice built by
``llm.policy_slices``; ``GET /api/policy/fact`` one fact with its home view.
A family's ``Dossier`` is read once per replica mtime and each section is
serialized once, so a request is a dict lookup. Slices are protected by a
content ETag. Failures are JSON ``{"detail": {...}}`` with a 4xx/503 status,
never a 500: the page renders the message.
``load_slice``/``load_fact`` are the in-process seam the HTTP routes, the
tests and the golden check share. Module scope imports fastapi and stdlib
only (as ``llm.api`` does); the heavy modules load on first use.
"""
from __future__ import annotations
import hashlib
import json
import logging
import re
import threading
from collections import OrderedDict
from pathlib import Path
from typing import Any, Callable
from fastapi import APIRouter, Depends, Header, Query, Response
from fastapi.responses import JSONResponse
log = logging.getLogger(__name__)
router = APIRouter(prefix="/api/policy")
KEY_RE = re.compile(r"^[A-Z0-9][A-Z0-9@/_.-]{0,199}$")
CACHE_MAX = 16
#: family -> (replica mtime, Dossier, {section: serialized bytes})
_CACHE: OrderedDict[str, tuple[int, Any, dict[str, bytes]]] = OrderedDict()
_LOCK = threading.Lock()
class PolicyError(Exception):
def __init__(self, status: int, detail: dict[str, Any]) -> None:
super().__init__(detail.get("message", ""))
self.status = status
self.detail = detail
def clear_cache() -> None:
with _LOCK:
_CACHE.clear()
def config_dep() -> Any:
from llm import config
return config.load()
def bib_dep() -> Callable[[], Any]:
from conf import connect
return connect.bib
def _status_of(family: str) -> str:
"""Why a family has no dossier: curated (hand-written, not built yet),
planned (in the book), derived (a ``pfs.code_family`` key; opening the
replica refreshes the registry from it) or unknown."""
from pfs.book import load_book
from pfs.curated import load_curated
from pfs.families import FAMILIES
if family in load_curated():
return "curated"
if family in load_book():
return "planned"
if family in FAMILIES:
return "derived"
return "unknown"
def _tier(family: str) -> str:
from pfs.curated import load_curated
return "reviewed" if family in load_curated() else "machine"
def _entry(cfg: Any, family: str) -> tuple[Any, dict[str, bytes]]:
from llm import evidence
from pfs.dossier import read_dossier
path = evidence._replica_path(cfg)
if not Path(path).exists():
raise PolicyError(503, {"message": "The data replica is not available."})
mtime = evidence._mtime(path)
with _LOCK:
hit = _CACHE.get(family)
if hit is not None and hit[0] == mtime:
_CACHE.move_to_end(family)
return hit[1], hit[2]
try:
cur = evidence._connect(path).cursor()
except Exception as e: # noqa: BLE001 — duckdb raises several types
log.warning("policy: replica unavailable (%s): %s", path, e)
raise PolicyError(503, {"message": "The data replica is not available."})
try:
try:
dossier = read_dossier(cur, family)
except Exception as e: # noqa: BLE001
log.warning("policy: dossier read failed (%s): %s", family, e)
raise PolicyError(
503, {"message": "The data replica is not available."}
) from e
if dossier is None:
raise PolicyError(
404,
{
"message": f"No dossier is built for {family} yet.",
"family": family,
"status": _status_of(family),
},
)
finally:
cur.close()
sections: dict[str, bytes] = {}
with _LOCK:
_CACHE[family] = (mtime, dossier, sections)
_CACHE.move_to_end(family)
while len(_CACHE) > CACHE_MAX:
_CACHE.popitem(last=False)
return dossier, sections
def _norm(family: str) -> str:
from pfs.families import normalize_key
key = normalize_key(family)
if not KEY_RE.match(key):
raise PolicyError(400, {"message": "invalid family key", "family": family})
return key
def _dump(obj: Any) -> bytes:
return json.dumps(obj, ensure_ascii=False, separators=(",", ":")).encode()
def _build(section: str, dossier: Any, family: str) -> dict[str, Any]:
from llm import policy_slices as ps
from pfs.book import load_book
if section == "header":
return ps.header_slice(
dossier, tier=_tier(family), book=load_book().get(family)
)
if section == "facts":
return ps.facts_index(dossier)
builder = {
"timeline": ps.timeline_slice,
"valuation": ps.valuation_slice,
"payment": ps.payment_slice,
"politics": ps.politics_slice,
}.get(section)
if builder is None: # in SECTIONS, but its builder is a later task's
raise PolicyError(
404, {"message": "section not available yet", "section": section}
)
return builder(dossier)
def slice_bytes(
cfg: Any,
family: str,
section: str,
*,
bib: Callable[[], Any] | None = None,
) -> bytes:
from llm.policy_slices import SECTIONS
key = _norm(family)
if section not in SECTIONS:
raise PolicyError(
404, {"message": "unknown section", "sections": list(SECTIONS)}
)
dossier, sections = _entry(cfg, key)
with _LOCK:
body = sections.get(section)
if body is None:
body = _dump(_build(section, dossier, key))
with _LOCK:
sections[section] = body
return body
def load_slice(
cfg: Any,
family: str,
section: str,
*,
bib: Callable[[], Any] | None = None,
) -> dict[str, Any]:
return json.loads(slice_bytes(cfg, family, section, bib=bib))
def load_fact(cfg: Any, family: str, fact_id: str) -> dict[str, Any]:
from llm.policy_slices import fact_detail
key = _norm(family)
dossier, _ = _entry(cfg, key)
out = fact_detail(dossier, fact_id)
if out is None:
raise PolicyError(
404, {"message": "unknown fact", "family": key, "id": fact_id}
)
return out
def _fail(e: PolicyError) -> JSONResponse:
return JSONResponse({"detail": e.detail}, status_code=e.status)
def _etag(body: bytes) -> str:
return '"%s"' % hashlib.sha1(body).hexdigest()[:20]
def _respond(body: bytes, if_none_match: str | None) -> Response:
etag = _etag(body)
headers = {"ETag": etag, "Cache-Control": "no-cache"}
sent = {t.strip().removeprefix("W/") for t in (if_none_match or "").split(",")}
if etag in sent or "*" in sent:
return Response(status_code=304, headers=headers)
return Response(body, media_type="application/json", headers=headers)
@router.get("/dossier/{section}")
def get_slice(
section: str,
family: str = Query(...),
if_none_match: str | None = Header(None),
cfg: Any = Depends(config_dep),
bib: Callable[[], Any] = Depends(bib_dep),
) -> Response:
try:
body = slice_bytes(cfg, family, section, bib=bib)
except PolicyError as e:
return _fail(e)
return _respond(body, if_none_match)
@router.get("/fact")
def get_fact(
family: str = Query(...),
id: str = Query(..., pattern=r"^F\d{1,6}$"),
if_none_match: str | None = Header(None),
cfg: Any = Depends(config_dep),
) -> Response:
try:
body = _dump(load_fact(cfg, family, id))
except PolicyError as e:
return _fail(e)
return _respond(body, if_none_match)

View File

@@ -0,0 +1,300 @@
"""The policy API (lex S1 Task 3): router, per-mtime cache, ETag/304, errors."""
from __future__ import annotations
import dataclasses
import os
import duckdb
import pytest
from fastapi.testclient import TestClient
from llm import evidence, policy
from llm.api import app
from llm.config import LlmConfig
from llm.policy_slices import SECTIONS
from pfs.codetables import ensure_tables
from pfs.dossier import read_dossier, write_dossier
from tests.llm.policy_fixture import fixture_dossier
CFG = LlmConfig(
ollama_hosts=("http://h1:11434",),
embed_model="e",
instruct_model="c",
embed_dim=768,
build_ann_index=False,
pg_host="x",
pg_port=5432,
pg_db="llm",
pg_user="llm",
duckdb_replica="/nonexistent/aco.ro.duckdb",
)
URL = "/api/policy/dossier/{}"
HV = {"family": "HOME-VISITS"}
def _write(path, *dossiers, sql=None):
# a republish replaces the file; drop the cached read-only handle first
if evidence._REPLICA is not None:
evidence._REPLICA[2].close()
evidence._REPLICA = None
con = duckdb.connect(str(path))
ensure_tables(con)
for d in dossiers:
write_dossier(con, d)
if sql:
con.execute(sql)
con.close()
@pytest.fixture
def replica(tmp_path, monkeypatch):
# hermetic: opening the replica must not rewrite the global registry
monkeypatch.setattr(evidence, "refresh_from", lambda con: 0)
path = tmp_path / "aco.ro.duckdb"
_write(path, fixture_dossier())
app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace(
CFG, duckdb_replica=str(path)
)
evidence._REPLICA = None
policy.clear_cache()
yield path
app.dependency_overrides.clear()
evidence._REPLICA = None
policy.clear_cache()
@pytest.fixture
def client(replica):
return TestClient(app)
def _count_reads(monkeypatch):
calls = []
def counting(con, family):
calls.append(family)
return read_dossier(con, family)
monkeypatch.setattr("pfs.dossier.read_dossier", counting)
return calls
def test_section_ok_etag_and_304(client):
r = client.get(URL.format("header"), params=HV)
assert r.status_code == 200
assert r.json()["family"] == "HOME-VISITS"
assert r.headers["cache-control"] == "no-cache"
assert r.headers["content-type"] == "application/json"
etag = r.headers["etag"]
assert etag.startswith('"') and len(etag) == 22
r2 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": etag})
assert r2.status_code == 304 and r2.content == b""
assert r2.headers["etag"] == etag
r3 = client.get(
URL.format("header"), params=HV, headers={"If-None-Match": "W/" + etag}
)
assert r3.status_code == 304
r4 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": '"x"'})
assert r4.status_code == 200
r5 = client.get(URL.format("header"), params=HV, headers={"If-None-Match": "*"})
assert r5.status_code == 304
@pytest.mark.parametrize("section", [s for s in SECTIONS if s != "sources"])
def test_every_built_section_serves(client, section):
r = client.get(URL.format(section), params=HV)
assert r.status_code == 200
assert r.json()["family"] == "HOME-VISITS"
def test_sources_section_not_built_yet(client):
r = client.get(URL.format("sources"), params=HV)
assert r.status_code == 404
assert r.json()["detail"]["section"] == "sources"
def test_header_tier_and_book(client):
h = client.get(URL.format("header"), params=HV).json()
assert h["tier"] == "reviewed"
assert h["phase"] is not None
def test_machine_tier_for_uncurated_family(replica, client, monkeypatch):
_write(replica, dataclasses.replace(fixture_dossier(), family="ZZZ-MACH"))
r = client.get(URL.format("header"), params={"family": "zzz-mach"})
assert r.status_code == 200
h = r.json()
assert h["tier"] == "machine" and h["phase"] is None and h["domains"] == []
def test_lowercase_family_normalized(client):
r = client.get(URL.format("header"), params={"family": "home-visits"})
assert r.status_code == 200
assert r.json()["family"] == "HOME-VISITS"
def test_bad_family_400(client):
for bad in ("a b", "../x", ""):
r = client.get(URL.format("header"), params={"family": bad})
assert r.status_code == 400, bad
assert r.json()["detail"]["message"] == "invalid family key"
assert r.json()["detail"]["family"] == bad
assert client.get(URL.format("header")).status_code == 422
def test_unknown_section_404(client):
r = client.get(URL.format("nope"), params=HV)
assert r.status_code == 404
assert r.json()["detail"]["sections"] == list(SECTIONS)
def test_no_dossier_404_with_status(client, monkeypatch):
r = client.get(URL.format("header"), params={"family": "CCM"})
assert r.status_code == 404
d = r.json()["detail"]
assert d["status"] == "planned" and d["family"] == "CCM"
assert d["message"] == "No dossier is built for CCM yet."
from pfs.curated import load_curated
real = load_curated()
monkeypatch.setattr(
"pfs.curated.load_curated", lambda *a, **k: {**real, "ZZZ-CUR": 1}
)
r = client.get(URL.format("header"), params={"family": "ZZZ-CUR"})
assert r.status_code == 404 and r.json()["detail"]["status"] == "curated"
r = client.get(URL.format("header"), params={"family": "ZZZ-NONE"})
assert r.status_code == 404 and r.json()["detail"]["status"] == "unknown"
def test_derived_status(replica, client, monkeypatch):
from pfs.families import FAMILIES
monkeypatch.setitem(FAMILIES, "ZZZ-REG", object())
r = client.get(URL.format("header"), params={"family": "ZZZ-REG"})
assert r.json()["detail"]["status"] == "derived"
def test_slash_family_key_is_valid(client):
r = client.get(URL.format("header"), params={"family": "A/D-TX-PROGRAM-HOUR"})
assert r.status_code == 404
assert r.json()["detail"]["family"] == "A/D-TX-PROGRAM-HOUR"
def test_replica_missing_503(tmp_path, client):
app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace(
CFG, duckdb_replica=str(tmp_path / "gone.duckdb")
)
r = client.get(URL.format("header"), params=HV)
assert r.status_code == 503
assert r.json()["detail"]["message"] == "The data replica is not available."
def test_replica_unopenable_503(tmp_path, client):
bad = tmp_path / "bad.duckdb"
bad.write_bytes(b"this is not a duckdb file" * 100)
app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace(
CFG, duckdb_replica=str(bad)
)
r = client.get(URL.format("header"), params=HV)
assert r.status_code == 503
def test_dossier_read_failure_503(client, monkeypatch):
def boom(con, family):
raise RuntimeError("io")
monkeypatch.setattr("pfs.dossier.read_dossier", boom)
r = client.get(URL.format("header"), params=HV)
assert r.status_code == 503
def test_table_missing_404(tmp_path, client):
path = tmp_path / "empty.duckdb"
con = duckdb.connect(str(path))
con.execute("CREATE TABLE t (a INTEGER)")
con.close()
app.dependency_overrides[policy.config_dep] = lambda: dataclasses.replace(
CFG, duckdb_replica=str(path)
)
r = client.get(URL.format("header"), params=HV)
assert r.status_code == 404
assert r.json()["detail"]["family"] == "HOME-VISITS"
assert r.json()["detail"]["status"] == "curated"
def test_cache_reloads_after_replica_republish(replica, client):
r1 = client.get(URL.format("header"), params=HV)
assert r1.json()["built_at"] != "2026-11-14T00:00:00"
t = os.stat(replica).st_mtime
_write(
replica, dataclasses.replace(fixture_dossier(), built_at="2026-11-14T00:00:00")
)
os.utime(replica, (t + 5, t + 5))
r2 = client.get(URL.format("header"), params=HV)
assert r2.json()["built_at"] == "2026-11-14T00:00:00"
assert r2.headers["etag"] != r1.headers["etag"]
def test_cache_reads_dossier_once_per_mtime(client, monkeypatch):
calls = _count_reads(monkeypatch)
for s in ("header", "timeline", "payment", "header"):
assert client.get(URL.format(s), params=HV).status_code == 200
assert calls == ["HOME-VISITS"]
def test_cache_bounded(replica, client, monkeypatch):
_write(replica, dataclasses.replace(fixture_dossier(), family="ZZZ-TWO"))
monkeypatch.setattr(policy, "CACHE_MAX", 1)
calls = _count_reads(monkeypatch)
client.get(URL.format("header"), params=HV)
client.get(URL.format("header"), params={"family": "ZZZ-TWO"})
assert list(policy._CACHE) == ["ZZZ-TWO"]
client.get(URL.format("header"), params=HV)
assert calls == ["HOME-VISITS", "ZZZ-TWO", "HOME-VISITS"]
def test_fact_endpoint(client):
r = client.get("/api/policy/fact", params={**HV, "id": "F4"})
assert r.status_code == 200
assert r.json()["view"] == "valuation"
etag = r.headers["etag"]
r2 = client.get(
"/api/policy/fact", params={**HV, "id": "F4"}, headers={"If-None-Match": etag}
)
assert r2.status_code == 304
r = client.get("/api/policy/fact", params={**HV, "id": "F999"})
assert r.status_code == 404 and r.json()["detail"]["message"] == "unknown fact"
assert client.get("/api/policy/fact", params={**HV, "id": "x"}).status_code == 422
r = client.get("/api/policy/fact", params={"family": "CCM", "id": "F1"})
assert r.status_code == 404 and r.json()["detail"]["status"] == "planned"
def test_load_slice_matches_http(replica, client):
cfg = dataclasses.replace(CFG, duckdb_replica=str(replica))
for s in ("header", "politics", "facts"):
assert (
policy.load_slice(cfg, "home-visits", s)
== client.get(URL.format(s), params=HV).json()
)
with pytest.raises(policy.PolicyError) as ei:
policy.load_slice(cfg, "a b", "header")
assert ei.value.status == 400
assert policy.load_fact(cfg, "HOME-VISITS", "F4")["view"] == "valuation"
def test_router_mounted_on_app():
# this FastAPI nests included routers under app.routes; openapi flattens
paths = set(app.openapi()["paths"])
assert "/api/policy/dossier/{section}" in paths
assert "/api/policy/fact" in paths
def test_default_dependencies(monkeypatch):
monkeypatch.setattr("llm.config.load", lambda: "cfg")
assert policy.config_dep() == "cfg"
from conf import connect
assert policy.bib_dep() is connect.bib