fix(pfs): --write refuses model units with no ledger entry in rounds, origin, guideline-facts, cms-summaries and dossier (M9); a dossier whose check failed is never written (M8); governance excerpts kept under --no-llm (M7); the dry run times the family's own events and stance-eval stamps its event kinds

This commit is contained in:
kert
2026-11-13 03:35:00 -05:00
parent d70dbd2540
commit c8f95ba634
10 changed files with 219 additions and 15 deletions

View File

@@ -21,7 +21,7 @@ from __future__ import annotations
import json
import logging
from collections import Counter
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
@@ -126,6 +126,7 @@ from pfs.stance import (
docket_events,
docket_rule_year,
eval_record,
event_kinds_of,
event_subjects,
fixed_text,
golden_dockets,
@@ -1712,8 +1713,11 @@ def rounds(
typer.echo(
f"rounds: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
)
if write:
if not tables_only: # --tables-only asks for no model rows
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
with _batch() as con:
ensure_tables(con)
n = write_rounds(con, fam.key, rows)
@@ -1805,8 +1809,10 @@ def origin(
typer.echo(
f"origins: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
)
if write:
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
with _batch() as con:
ensure_tables(con)
n = write_origin(con, fam.key, rows)
@@ -1866,8 +1872,10 @@ def guideline_facts_cmd(
f"{counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
f"failed_pending {counts.get('failed_pending', 0)}, "
f"cited by section (no page) {counts.get('unpaged', 0)}"
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
)
if write:
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
with _batch() as bcon:
ensure_tables(bcon)
n = write_guideline_facts(bcon, fam.key, rows)
@@ -1919,8 +1927,10 @@ def cms_summaries(
f"summaries: {len(rows)}; done {counts.get('done', 0)}, "
f"skipped {counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
f"failed_pending {counts.get('failed_pending', 0)}"
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
)
if write:
_refuse_pending(counts, model)
with _batch() as con:
ensure_tables(con)
n = write_cms_summaries(con, fam.key, rows)
@@ -2212,6 +2222,8 @@ def _method_eval(
rec = eval_record(primary, sc)
rec.pop("docket")
rec["generation"] = gen
# M2: which event kinds the set holds positions on, stamped beside it
rec["event_kinds"] = event_kinds_of(primary)
if golden.exists():
tuning = json.loads(golden.read_text(encoding="utf-8"))
if tuning.get("family") == family:
@@ -2406,6 +2418,18 @@ def stance_eval(
)
def _refuse_pending(counts: Mapping[str, int], model: str | None) -> None:
"""M9: refuse a --write while model units have no ledger entry (a
prompt-version bump, a --limit cut): their rows would silently be
missing from the table."""
n = counts.get("pending", 0)
if n:
raise typer.BadParameter(
f"{n} units have no ledger entry for {model or 'the model'}; rerun "
"without --no-llm (or with a larger --limit) before --write"
)
def _model_of(ask: Any, large: bool) -> str:
"""The pinned model's name: the asker's, or (no asker) the configured one."""
if ask is not None:
@@ -2438,6 +2462,12 @@ def dossier_cmd(
"--dry-run",
help="Count facts and timeline events; no model calls, no writes.",
),
template_ok: bool = typer.Option(
False,
"--template-ok",
help="Let --write store template copy for event lines the prose "
"ledger has no entry for (otherwise refused).",
),
write: bool = typer.Option(False, "--write"),
) -> None:
"""Assemble a family's dossier (pfs.dossier): every stored fact with its
@@ -2462,7 +2492,7 @@ def dossier_cmd(
try:
if dry_run:
facts = collect_facts(con, store, fam)
events = dossier_timeline(facts)
events = dossier_timeline(facts, fam.codes)
drafted = [e for e in events if not template_only(e.kind)]
typer.echo(
f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; "
@@ -2471,15 +2501,19 @@ def dossier_cmd(
"comments, link and CPT edition lines are template copy)"
)
continue
ask = None if no_llm else _asker()
pending: list[str] = []
d = build_dossier(
con,
store,
fam,
ask=None if no_llm else _asker(),
excerpts=[] if no_llm else _excerpts(store, fam.name),
ask=ask,
# M7: embeddings, not generation: kept under --no-llm
excerpts=_excerpts(store, fam.name),
log=RunLog(ledger_path("prose", fam.key)),
replay_model=_model_name() if no_llm else None,
limit=limit,
pending=pending,
)
finally:
con.close()
@@ -2502,6 +2536,22 @@ def dossier_cmd(
)
if d.extractor != "template" and d.fallbacks:
typer.echo("template fallback: " + "; ".join(d.fallbacks))
model = getattr(ask, "model", None) or _model_name()
if pending:
typer.echo(
f"{len(pending)} event lines have no prose ledger entry for "
f"{model} (template copy): " + "; ".join(pending)
)
if write and pending and not template_ok:
# M9: never a silent fallback after the facts or the prompt changed
raise typer.BadParameter(
f"{len(pending)} event lines have no prose ledger entry for {model}; "
"rerun without --no-llm, or pass --template-ok"
)
if write and not d.check_passed:
# M8: a dossier whose prose failed the check is never stored
typer.echo(f"{fam.key}: check FAILED; not written")
raise typer.Exit(1)
built.append(d)
if write and built:
with _batch() as bcon:

View File

@@ -184,10 +184,15 @@ def summaries(
)
if counts is not None:
counts.update(result)
pending = 0
for u in units:
entry = log.get(u, hashes[u], model_name, SUMMARY_PROMPT_VERSION)
pending += entry is None
if entry is not None and entry.output is not None:
recorded[u] = entry.output["stance"]
# M9: pairs with no ledger answer, written as "unclear" by regex
if counts is not None:
counts["pending"] = pending
extractor = f"model:{model_name}@{SUMMARY_PROMPT_VERSION}"
return [
CmsSummaryRow(

View File

@@ -803,8 +803,11 @@ def guideline_facts(
run.update(counts_run)
extractor = f"model:{model}@{GUIDELINE_PROMPT_VERSION}"
scope: dict[int, list[list[GuidelineFactRow]]] = {}
# M9: units with no ledger answer, so a write would miss their rows
run["pending"] = 0
for j in jobs:
entry = log.get(j["unit"], j["hash"], model, GUIDELINE_PROMPT_VERSION)
run["pending"] += entry is None
if entry is None or entry.output is None:
continue
mine = [

View File

@@ -1185,10 +1185,14 @@ def family_origin(
limit=limit,
retry_failed=retry_failed,
)
pending = 0
for j in jobs:
entry = log.get(j["unit"], j["hash"], model, ORIGIN_PROMPT_VERSION)
pending += entry is None
if entry is not None and entry.output is not None:
model_rows += _rows(j, entry.output, family, model)
# M9: units with no ledger answer, so a write would miss their rows
counts = {**counts, "pending": pending}
found = _drop_misstated(model_rows, events) + lineage_origins(store, family, events)
rows = _consolidate(found) + guidance_origins(con, family.key, store)
rows.sort(

View File

@@ -1332,13 +1332,18 @@ def family_rounds(
limit=limit,
retry_failed=retry_failed,
)
pending = 0
for p in naming:
entry = log.get(
_unit(p), hashes[_unit(p)], model_name, ROUND_PROMPT_VERSION
)
pending += entry is None
if entry is not None and entry.output is not None:
model += rows_from_output(
entry.output, p, codes, family=family.key, model=model_name
)
# M9: units the ledger has no answer for (a prompt-version bump, a
# --limit cut): their model rows would be missing from a write
counts = {**counts, "pending": pending}
merged = apply_paragraph_rules(merge_rounds(table, regex, model), paras)
return [r for r in merged if has_content(r)], counts

View File

@@ -2678,7 +2678,9 @@ class TestStanceMethodRecord:
prim = tmp_path / "p.json"
prim.write_text(json.dumps({
"family": family, "dockets": ["H1", "H2"], "in_sample": False,
"role": "primary", "generation": generation, "items": [{}, {}, {}],
"role": "primary", "generation": generation,
"items": [{"cells": {"F:2026:G2211": {"label": "support"},
"F:2026:other": {"label": "none"}}}, {}, {}],
})) # fmt: skip
foot = tmp_path / "f.json"
foot.write_text(json.dumps({
@@ -2707,7 +2709,7 @@ class TestStanceMethodRecord:
assert got == {
"cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 3,
"n_positions": 27, "family": "HOME-VISITS", "dockets": ["H1", "H2"],
"in_sample": False, "generation": "stance-v5",
"in_sample": False, "generation": "stance-v5", "event_kinds": ["G2211"],
"in_sample_record": {
"cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 2,
"n_positions": 27, "docket": "T1",
@@ -3751,15 +3753,28 @@ class TestDossier:
assert "check: passed; prose template" in res.output
assert "timeline years: 2008" in res.output
assert "stakeholders (CY2023, HOME-VISITS:2023:valuation" in res.output
assert env.published == [] and env.excerpts == []
# M7: governance excerpts are embeddings, not generation: kept
assert env.published == [] and env.excerpts == ["Home or Residence Services"]
from pfs.dossier import dossier_families
assert dossier_families(env.con) == ()
def test_write_stores_the_dossier_and_publishes(self, env):
# M9: the prose ledger has no entry for the event lines: refused ...
res = runner.invoke(
app, ["pfs", "dossier", "--family", "home-visits", "--no-llm", "--write"]
)
assert res.exit_code != 0
assert "event lines have no prose ledger entry for fake-14b" in res.output
assert env.published == []
# ... unless template copy for them is asked for
res = runner.invoke(
app,
[
"pfs", "dossier", "--family", "home-visits", "--no-llm", "--write",
"--template-ok",
],
) # fmt: skip
assert res.exit_code == 0, res.output
from pfs.dossier import read_dossier
@@ -3768,6 +3783,27 @@ class TestDossier:
assert "wrote dossier HOME-VISITS" in res.output
assert env.published == [True]
def test_a_failed_check_is_never_written(self, env, monkeypatch):
"""M8: a dossier whose check failed is printed, never stored."""
real = pfs_cli.build_dossier
def failing(*a, **k):
return dataclasses.replace(real(*a, **k), check_passed=False)
monkeypatch.setattr(pfs_cli, "build_dossier", failing)
res = runner.invoke(
app,
[
"pfs", "dossier", "--family", "home-visits", "--no-llm", "--write",
"--template-ok",
],
) # fmt: skip
assert res.exit_code != 0 and "check: FAILED" in res.output
assert "not written" in res.output and env.published == []
from pfs.dossier import dossier_families
assert dossier_families(env.con) == ()
def test_all_builds_every_curated_family(self, env):
res = runner.invoke(app, ["pfs", "dossier", "--all", "--no-llm"])
assert res.exit_code == 0, res.output
@@ -3830,3 +3866,71 @@ def test_dossier_excerpts_use_the_llm_pool(monkeypatch):
)
assert pfs_cli._excerpts("STORE", "Home or Residence Services") == []
assert calls == [("CFG", "POOL", "STORE", "Home or Residence Services")]
class TestPendingLedger:
"""M9: a --write whose model units have no ledger entry (a prompt-version
bump, a --limit cut) is refused, never a silent drop of model rows."""
PENDING = {"done": 0, "skipped": 3, "failed": 0, "failed_pending": 0, "pending": 2}
def test_rounds(self, con, monkeypatch):
TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING))
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
res = runner.invoke(
app, ["pfs", "rounds", "--family", "HOME-VISITS", "--no-llm", "--write"]
)
assert res.exit_code != 0
assert "2 units have no ledger entry for qwen-x" in res.output
# --tables-only says no model rows are wanted: not refused
TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING))
res = runner.invoke(
app, ["pfs", "rounds", "--family", "HOME-VISITS", "--tables-only"]
)
assert res.exit_code == 0 and "pending 2" in res.output
def test_origin(self, con, monkeypatch):
TestOrigin._patch(TestOrigin(), monkeypatch, [])
monkeypatch.setattr(
pfs_cli, "family_origin", lambda *a, **k: ([], dict(self.PENDING))
)
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
res = runner.invoke(
app, ["pfs", "origin", "--family", "HOME-VISITS", "--no-llm", "--write"]
)
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output
def test_guideline_facts(self, con, monkeypatch):
def fake(con_, store, fam, **kw):
kw["counts"].update(self.PENDING, unpaged=0)
return []
monkeypatch.setattr(pfs_cli, "guideline_facts", fake)
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
res = runner.invoke(
app,
[
"pfs",
"guideline-facts",
"--family",
"HOME-VISITS",
"--no-llm",
"--write",
],
)
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output
def test_cms_summaries(self, con, monkeypatch):
seen: dict = {}
TestCmsSummaries._patch(
TestCmsSummaries(), monkeypatch, seen, counts=dict(self.PENDING)
)
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
res = runner.invoke(
app,
["pfs", "cms-summaries", "--family", "HOME-VISITS", "--no-llm", "--write"],
)
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output

View File

@@ -323,14 +323,21 @@ def test_summaries_ledger_resume_and_replay(tmp_path):
# --no-llm replay: same rows, zero calls
replay = summaries(s, FAM, log=log, replay_model="m")
assert replay == first and len(calls) == n
# unrecorded under a different model: unclear / regex
other = summaries(s, FAM, log=log, replay_model="other")
# unrecorded under a different model: unclear / regex, and (M9) every
# pair counted as pending
pending: dict = {}
other = summaries(s, FAM, log=log, replay_model="other", counts=pending)
assert {r.stance for r in other} == {"unclear"}
assert {r.extractor for r in other} == {"regex"}
# limit caps new calls; the rest stay regex
assert pending == {"pending": n}
# limit caps new calls; the rest stay regex and pending
fresh = RunLog(tmp_path / "l2.jsonl")
capped = summaries(s, FAM, classify=classify, model="m", log=fresh, limit=1)
cut: dict = {}
capped = summaries(
s, FAM, classify=classify, model="m", log=fresh, limit=1, counts=cut
)
assert sum(r.extractor != "regex" for r in capped) == 1
assert cut["pending"] == n - 1
finally:
s.close()

View File

@@ -509,8 +509,18 @@ def test_scope_points_through_ledger(con, books, tmp_path):
calls.clear()
gf.guideline_facts(con, object(), FAM, ask=ask, log=log)
assert calls == [] # resume: no new calls
rows2 = gf.guideline_facts(con, object(), FAM, log=log, replay_model="qwen-test")
assert _by(rows2, fact="scope_point") == pts
replayed: dict = {}
rows2 = gf.guideline_facts(
con, object(), FAM, log=log, replay_model="qwen-test", counts=replayed
)
assert _by(rows2, fact="scope_point") == pts and replayed["pending"] == 0
# M9: a replay for a model the ledger never ran counts every unit pending
none: dict = {}
gf.guideline_facts(con, object(), FAM, log=log, replay_model="other", counts=none)
assert none["pending"] == 2 and not _by(
gf.guideline_facts(con, object(), FAM, log=log, replay_model="other"),
fact="scope_point",
)
WORDS = (

View File

@@ -914,7 +914,15 @@ def test_family_origin_fr_rows_cite_the_paragraph(tmp_path):
]
assert fr[0].locator == "87 FR 100, ¶2" and fr[0].kind == "statute"
assert [r.cite_kind for r in rows if r.extractor == "guidance"] == ["cfr"]
assert counts == {"done": 2, "skipped": 0, "failed": 0, "failed_pending": 0}
assert counts == {
"done": 2, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
} # fmt: skip
# M9: a replay for a model the ledger never ran has every unit pending
_rows, replay = family_origin(
c, s, FAM, ask=None, log=RunLog(tmp_path / "l.jsonl"), books=[],
replay_model="another",
) # fmt: skip
assert replay["pending"] == 2
def _chain_store(tmp_path, paras, *, volume="87"):
@@ -1115,6 +1123,7 @@ def test_family_origin_resumes_and_replays_without_a_model(tmp_path):
"skipped": 0,
"failed": 0,
"failed_pending": 0,
"pending": 0,
}
# no model and nothing to replay: only the deterministic guidance rows
none, _ = family_origin(_con(), s, FAM, ask=None, log=None, books=books)

View File

@@ -1055,8 +1055,15 @@ def test_family_rounds_end_to_end_and_resume(tmp_path):
s, key = _store(tmp_path, with_text=False)
ask = FakeAsk(_good_reply())
log = RunLog(tmp_path / "ledger.jsonl")
# M9: a replay over an empty ledger counts the unit as pending
_rows0, c0 = family_rounds(
None, s, FAM, ask=None, log=RunLog(tmp_path / "empty.jsonl"), replay_model="m"
)
assert c0["pending"] == 1
rows, counts = family_rounds(None, s, FAM, ask=ask, log=log)
assert counts == {"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0}
assert counts == {
"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
} # fmt: skip
by = {r.code: r for r in rows}
r = by["99341"]
assert (r.proposed_work_rvu, r.final_work_rvu, r.ruc_work_rvu) == (1.0, 1.0, 1.0)