fix(pfs): --write refuses model units with no ledger entry in rounds, origin, guideline-facts, cms-summaries and dossier (M9); a dossier whose check failed is never written (M8); governance excerpts kept under --no-llm (M7); the dry run times the family's own events and stance-eval stamps its event kinds
This commit is contained in:
@@ -21,7 +21,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
from collections import Counter
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -126,6 +126,7 @@ from pfs.stance import (
|
||||
docket_events,
|
||||
docket_rule_year,
|
||||
eval_record,
|
||||
event_kinds_of,
|
||||
event_subjects,
|
||||
fixed_text,
|
||||
golden_dockets,
|
||||
@@ -1712,8 +1713,11 @@ def rounds(
|
||||
typer.echo(
|
||||
f"rounds: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
|
||||
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
|
||||
)
|
||||
if write:
|
||||
if not tables_only: # --tables-only asks for no model rows
|
||||
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_rounds(con, fam.key, rows)
|
||||
@@ -1805,8 +1809,10 @@ def origin(
|
||||
typer.echo(
|
||||
f"origins: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, "
|
||||
f"failed {counts['failed']}, failed_pending {counts['failed_pending']}"
|
||||
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
|
||||
)
|
||||
if write:
|
||||
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_origin(con, fam.key, rows)
|
||||
@@ -1866,8 +1872,10 @@ def guideline_facts_cmd(
|
||||
f"{counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
|
||||
f"failed_pending {counts.get('failed_pending', 0)}, "
|
||||
f"cited by section (no page) {counts.get('unpaged', 0)}"
|
||||
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
|
||||
)
|
||||
if write:
|
||||
_refuse_pending(counts, getattr(ask, "model", None) or _model_name())
|
||||
with _batch() as bcon:
|
||||
ensure_tables(bcon)
|
||||
n = write_guideline_facts(bcon, fam.key, rows)
|
||||
@@ -1919,8 +1927,10 @@ def cms_summaries(
|
||||
f"summaries: {len(rows)}; done {counts.get('done', 0)}, "
|
||||
f"skipped {counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, "
|
||||
f"failed_pending {counts.get('failed_pending', 0)}"
|
||||
+ (f", pending {counts['pending']}" if counts.get("pending") else "")
|
||||
)
|
||||
if write:
|
||||
_refuse_pending(counts, model)
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
n = write_cms_summaries(con, fam.key, rows)
|
||||
@@ -2212,6 +2222,8 @@ def _method_eval(
|
||||
rec = eval_record(primary, sc)
|
||||
rec.pop("docket")
|
||||
rec["generation"] = gen
|
||||
# M2: which event kinds the set holds positions on, stamped beside it
|
||||
rec["event_kinds"] = event_kinds_of(primary)
|
||||
if golden.exists():
|
||||
tuning = json.loads(golden.read_text(encoding="utf-8"))
|
||||
if tuning.get("family") == family:
|
||||
@@ -2406,6 +2418,18 @@ def stance_eval(
|
||||
)
|
||||
|
||||
|
||||
def _refuse_pending(counts: Mapping[str, int], model: str | None) -> None:
|
||||
"""M9: refuse a --write while model units have no ledger entry (a
|
||||
prompt-version bump, a --limit cut): their rows would silently be
|
||||
missing from the table."""
|
||||
n = counts.get("pending", 0)
|
||||
if n:
|
||||
raise typer.BadParameter(
|
||||
f"{n} units have no ledger entry for {model or 'the model'}; rerun "
|
||||
"without --no-llm (or with a larger --limit) before --write"
|
||||
)
|
||||
|
||||
|
||||
def _model_of(ask: Any, large: bool) -> str:
|
||||
"""The pinned model's name: the asker's, or (no asker) the configured one."""
|
||||
if ask is not None:
|
||||
@@ -2438,6 +2462,12 @@ def dossier_cmd(
|
||||
"--dry-run",
|
||||
help="Count facts and timeline events; no model calls, no writes.",
|
||||
),
|
||||
template_ok: bool = typer.Option(
|
||||
False,
|
||||
"--template-ok",
|
||||
help="Let --write store template copy for event lines the prose "
|
||||
"ledger has no entry for (otherwise refused).",
|
||||
),
|
||||
write: bool = typer.Option(False, "--write"),
|
||||
) -> None:
|
||||
"""Assemble a family's dossier (pfs.dossier): every stored fact with its
|
||||
@@ -2462,7 +2492,7 @@ def dossier_cmd(
|
||||
try:
|
||||
if dry_run:
|
||||
facts = collect_facts(con, store, fam)
|
||||
events = dossier_timeline(facts)
|
||||
events = dossier_timeline(facts, fam.codes)
|
||||
drafted = [e for e in events if not template_only(e.kind)]
|
||||
typer.echo(
|
||||
f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; "
|
||||
@@ -2471,15 +2501,19 @@ def dossier_cmd(
|
||||
"comments, link and CPT edition lines are template copy)"
|
||||
)
|
||||
continue
|
||||
ask = None if no_llm else _asker()
|
||||
pending: list[str] = []
|
||||
d = build_dossier(
|
||||
con,
|
||||
store,
|
||||
fam,
|
||||
ask=None if no_llm else _asker(),
|
||||
excerpts=[] if no_llm else _excerpts(store, fam.name),
|
||||
ask=ask,
|
||||
# M7: embeddings, not generation: kept under --no-llm
|
||||
excerpts=_excerpts(store, fam.name),
|
||||
log=RunLog(ledger_path("prose", fam.key)),
|
||||
replay_model=_model_name() if no_llm else None,
|
||||
limit=limit,
|
||||
pending=pending,
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
@@ -2502,6 +2536,22 @@ def dossier_cmd(
|
||||
)
|
||||
if d.extractor != "template" and d.fallbacks:
|
||||
typer.echo("template fallback: " + "; ".join(d.fallbacks))
|
||||
model = getattr(ask, "model", None) or _model_name()
|
||||
if pending:
|
||||
typer.echo(
|
||||
f"{len(pending)} event lines have no prose ledger entry for "
|
||||
f"{model} (template copy): " + "; ".join(pending)
|
||||
)
|
||||
if write and pending and not template_ok:
|
||||
# M9: never a silent fallback after the facts or the prompt changed
|
||||
raise typer.BadParameter(
|
||||
f"{len(pending)} event lines have no prose ledger entry for {model}; "
|
||||
"rerun without --no-llm, or pass --template-ok"
|
||||
)
|
||||
if write and not d.check_passed:
|
||||
# M8: a dossier whose prose failed the check is never stored
|
||||
typer.echo(f"{fam.key}: check FAILED; not written")
|
||||
raise typer.Exit(1)
|
||||
built.append(d)
|
||||
if write and built:
|
||||
with _batch() as bcon:
|
||||
|
||||
@@ -184,10 +184,15 @@ def summaries(
|
||||
)
|
||||
if counts is not None:
|
||||
counts.update(result)
|
||||
pending = 0
|
||||
for u in units:
|
||||
entry = log.get(u, hashes[u], model_name, SUMMARY_PROMPT_VERSION)
|
||||
pending += entry is None
|
||||
if entry is not None and entry.output is not None:
|
||||
recorded[u] = entry.output["stance"]
|
||||
# M9: pairs with no ledger answer, written as "unclear" by regex
|
||||
if counts is not None:
|
||||
counts["pending"] = pending
|
||||
extractor = f"model:{model_name}@{SUMMARY_PROMPT_VERSION}"
|
||||
return [
|
||||
CmsSummaryRow(
|
||||
|
||||
@@ -803,8 +803,11 @@ def guideline_facts(
|
||||
run.update(counts_run)
|
||||
extractor = f"model:{model}@{GUIDELINE_PROMPT_VERSION}"
|
||||
scope: dict[int, list[list[GuidelineFactRow]]] = {}
|
||||
# M9: units with no ledger answer, so a write would miss their rows
|
||||
run["pending"] = 0
|
||||
for j in jobs:
|
||||
entry = log.get(j["unit"], j["hash"], model, GUIDELINE_PROMPT_VERSION)
|
||||
run["pending"] += entry is None
|
||||
if entry is None or entry.output is None:
|
||||
continue
|
||||
mine = [
|
||||
|
||||
@@ -1185,10 +1185,14 @@ def family_origin(
|
||||
limit=limit,
|
||||
retry_failed=retry_failed,
|
||||
)
|
||||
pending = 0
|
||||
for j in jobs:
|
||||
entry = log.get(j["unit"], j["hash"], model, ORIGIN_PROMPT_VERSION)
|
||||
pending += entry is None
|
||||
if entry is not None and entry.output is not None:
|
||||
model_rows += _rows(j, entry.output, family, model)
|
||||
# M9: units with no ledger answer, so a write would miss their rows
|
||||
counts = {**counts, "pending": pending}
|
||||
found = _drop_misstated(model_rows, events) + lineage_origins(store, family, events)
|
||||
rows = _consolidate(found) + guidance_origins(con, family.key, store)
|
||||
rows.sort(
|
||||
|
||||
@@ -1332,13 +1332,18 @@ def family_rounds(
|
||||
limit=limit,
|
||||
retry_failed=retry_failed,
|
||||
)
|
||||
pending = 0
|
||||
for p in naming:
|
||||
entry = log.get(
|
||||
_unit(p), hashes[_unit(p)], model_name, ROUND_PROMPT_VERSION
|
||||
)
|
||||
pending += entry is None
|
||||
if entry is not None and entry.output is not None:
|
||||
model += rows_from_output(
|
||||
entry.output, p, codes, family=family.key, model=model_name
|
||||
)
|
||||
# M9: units the ledger has no answer for (a prompt-version bump, a
|
||||
# --limit cut): their model rows would be missing from a write
|
||||
counts = {**counts, "pending": pending}
|
||||
merged = apply_paragraph_rules(merge_rounds(table, regex, model), paras)
|
||||
return [r for r in merged if has_content(r)], counts
|
||||
|
||||
@@ -2678,7 +2678,9 @@ class TestStanceMethodRecord:
|
||||
prim = tmp_path / "p.json"
|
||||
prim.write_text(json.dumps({
|
||||
"family": family, "dockets": ["H1", "H2"], "in_sample": False,
|
||||
"role": "primary", "generation": generation, "items": [{}, {}, {}],
|
||||
"role": "primary", "generation": generation,
|
||||
"items": [{"cells": {"F:2026:G2211": {"label": "support"},
|
||||
"F:2026:other": {"label": "none"}}}, {}, {}],
|
||||
})) # fmt: skip
|
||||
foot = tmp_path / "f.json"
|
||||
foot.write_text(json.dumps({
|
||||
@@ -2707,7 +2709,7 @@ class TestStanceMethodRecord:
|
||||
assert got == {
|
||||
"cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 3,
|
||||
"n_positions": 27, "family": "HOME-VISITS", "dockets": ["H1", "H2"],
|
||||
"in_sample": False, "generation": "stance-v5",
|
||||
"in_sample": False, "generation": "stance-v5", "event_kinds": ["G2211"],
|
||||
"in_sample_record": {
|
||||
"cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 2,
|
||||
"n_positions": 27, "docket": "T1",
|
||||
@@ -3751,15 +3753,28 @@ class TestDossier:
|
||||
assert "check: passed; prose template" in res.output
|
||||
assert "timeline years: 2008" in res.output
|
||||
assert "stakeholders (CY2023, HOME-VISITS:2023:valuation" in res.output
|
||||
assert env.published == [] and env.excerpts == []
|
||||
# M7: governance excerpts are embeddings, not generation: kept
|
||||
assert env.published == [] and env.excerpts == ["Home or Residence Services"]
|
||||
from pfs.dossier import dossier_families
|
||||
|
||||
assert dossier_families(env.con) == ()
|
||||
|
||||
def test_write_stores_the_dossier_and_publishes(self, env):
|
||||
# M9: the prose ledger has no entry for the event lines: refused ...
|
||||
res = runner.invoke(
|
||||
app, ["pfs", "dossier", "--family", "home-visits", "--no-llm", "--write"]
|
||||
)
|
||||
assert res.exit_code != 0
|
||||
assert "event lines have no prose ledger entry for fake-14b" in res.output
|
||||
assert env.published == []
|
||||
# ... unless template copy for them is asked for
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"pfs", "dossier", "--family", "home-visits", "--no-llm", "--write",
|
||||
"--template-ok",
|
||||
],
|
||||
) # fmt: skip
|
||||
assert res.exit_code == 0, res.output
|
||||
from pfs.dossier import read_dossier
|
||||
|
||||
@@ -3768,6 +3783,27 @@ class TestDossier:
|
||||
assert "wrote dossier HOME-VISITS" in res.output
|
||||
assert env.published == [True]
|
||||
|
||||
def test_a_failed_check_is_never_written(self, env, monkeypatch):
|
||||
"""M8: a dossier whose check failed is printed, never stored."""
|
||||
real = pfs_cli.build_dossier
|
||||
|
||||
def failing(*a, **k):
|
||||
return dataclasses.replace(real(*a, **k), check_passed=False)
|
||||
|
||||
monkeypatch.setattr(pfs_cli, "build_dossier", failing)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"pfs", "dossier", "--family", "home-visits", "--no-llm", "--write",
|
||||
"--template-ok",
|
||||
],
|
||||
) # fmt: skip
|
||||
assert res.exit_code != 0 and "check: FAILED" in res.output
|
||||
assert "not written" in res.output and env.published == []
|
||||
from pfs.dossier import dossier_families
|
||||
|
||||
assert dossier_families(env.con) == ()
|
||||
|
||||
def test_all_builds_every_curated_family(self, env):
|
||||
res = runner.invoke(app, ["pfs", "dossier", "--all", "--no-llm"])
|
||||
assert res.exit_code == 0, res.output
|
||||
@@ -3830,3 +3866,71 @@ def test_dossier_excerpts_use_the_llm_pool(monkeypatch):
|
||||
)
|
||||
assert pfs_cli._excerpts("STORE", "Home or Residence Services") == []
|
||||
assert calls == [("CFG", "POOL", "STORE", "Home or Residence Services")]
|
||||
|
||||
|
||||
class TestPendingLedger:
|
||||
"""M9: a --write whose model units have no ledger entry (a prompt-version
|
||||
bump, a --limit cut) is refused, never a silent drop of model rows."""
|
||||
|
||||
PENDING = {"done": 0, "skipped": 3, "failed": 0, "failed_pending": 0, "pending": 2}
|
||||
|
||||
def test_rounds(self, con, monkeypatch):
|
||||
TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING))
|
||||
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
|
||||
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
|
||||
res = runner.invoke(
|
||||
app, ["pfs", "rounds", "--family", "HOME-VISITS", "--no-llm", "--write"]
|
||||
)
|
||||
assert res.exit_code != 0
|
||||
assert "2 units have no ledger entry for qwen-x" in res.output
|
||||
# --tables-only says no model rows are wanted: not refused
|
||||
TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING))
|
||||
res = runner.invoke(
|
||||
app, ["pfs", "rounds", "--family", "HOME-VISITS", "--tables-only"]
|
||||
)
|
||||
assert res.exit_code == 0 and "pending 2" in res.output
|
||||
|
||||
def test_origin(self, con, monkeypatch):
|
||||
TestOrigin._patch(TestOrigin(), monkeypatch, [])
|
||||
monkeypatch.setattr(
|
||||
pfs_cli, "family_origin", lambda *a, **k: ([], dict(self.PENDING))
|
||||
)
|
||||
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
|
||||
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
|
||||
res = runner.invoke(
|
||||
app, ["pfs", "origin", "--family", "HOME-VISITS", "--no-llm", "--write"]
|
||||
)
|
||||
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output
|
||||
|
||||
def test_guideline_facts(self, con, monkeypatch):
|
||||
def fake(con_, store, fam, **kw):
|
||||
kw["counts"].update(self.PENDING, unpaged=0)
|
||||
return []
|
||||
|
||||
monkeypatch.setattr(pfs_cli, "guideline_facts", fake)
|
||||
monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x")
|
||||
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"pfs",
|
||||
"guideline-facts",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--no-llm",
|
||||
"--write",
|
||||
],
|
||||
)
|
||||
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output
|
||||
|
||||
def test_cms_summaries(self, con, monkeypatch):
|
||||
seen: dict = {}
|
||||
TestCmsSummaries._patch(
|
||||
TestCmsSummaries(), monkeypatch, seen, counts=dict(self.PENDING)
|
||||
)
|
||||
monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
["pfs", "cms-summaries", "--family", "HOME-VISITS", "--no-llm", "--write"],
|
||||
)
|
||||
assert res.exit_code != 0 and "2 units have no ledger entry" in res.output
|
||||
|
||||
@@ -323,14 +323,21 @@ def test_summaries_ledger_resume_and_replay(tmp_path):
|
||||
# --no-llm replay: same rows, zero calls
|
||||
replay = summaries(s, FAM, log=log, replay_model="m")
|
||||
assert replay == first and len(calls) == n
|
||||
# unrecorded under a different model: unclear / regex
|
||||
other = summaries(s, FAM, log=log, replay_model="other")
|
||||
# unrecorded under a different model: unclear / regex, and (M9) every
|
||||
# pair counted as pending
|
||||
pending: dict = {}
|
||||
other = summaries(s, FAM, log=log, replay_model="other", counts=pending)
|
||||
assert {r.stance for r in other} == {"unclear"}
|
||||
assert {r.extractor for r in other} == {"regex"}
|
||||
# limit caps new calls; the rest stay regex
|
||||
assert pending == {"pending": n}
|
||||
# limit caps new calls; the rest stay regex and pending
|
||||
fresh = RunLog(tmp_path / "l2.jsonl")
|
||||
capped = summaries(s, FAM, classify=classify, model="m", log=fresh, limit=1)
|
||||
cut: dict = {}
|
||||
capped = summaries(
|
||||
s, FAM, classify=classify, model="m", log=fresh, limit=1, counts=cut
|
||||
)
|
||||
assert sum(r.extractor != "regex" for r in capped) == 1
|
||||
assert cut["pending"] == n - 1
|
||||
finally:
|
||||
s.close()
|
||||
|
||||
|
||||
@@ -509,8 +509,18 @@ def test_scope_points_through_ledger(con, books, tmp_path):
|
||||
calls.clear()
|
||||
gf.guideline_facts(con, object(), FAM, ask=ask, log=log)
|
||||
assert calls == [] # resume: no new calls
|
||||
rows2 = gf.guideline_facts(con, object(), FAM, log=log, replay_model="qwen-test")
|
||||
assert _by(rows2, fact="scope_point") == pts
|
||||
replayed: dict = {}
|
||||
rows2 = gf.guideline_facts(
|
||||
con, object(), FAM, log=log, replay_model="qwen-test", counts=replayed
|
||||
)
|
||||
assert _by(rows2, fact="scope_point") == pts and replayed["pending"] == 0
|
||||
# M9: a replay for a model the ledger never ran counts every unit pending
|
||||
none: dict = {}
|
||||
gf.guideline_facts(con, object(), FAM, log=log, replay_model="other", counts=none)
|
||||
assert none["pending"] == 2 and not _by(
|
||||
gf.guideline_facts(con, object(), FAM, log=log, replay_model="other"),
|
||||
fact="scope_point",
|
||||
)
|
||||
|
||||
|
||||
WORDS = (
|
||||
|
||||
@@ -914,7 +914,15 @@ def test_family_origin_fr_rows_cite_the_paragraph(tmp_path):
|
||||
]
|
||||
assert fr[0].locator == "87 FR 100, ¶2" and fr[0].kind == "statute"
|
||||
assert [r.cite_kind for r in rows if r.extractor == "guidance"] == ["cfr"]
|
||||
assert counts == {"done": 2, "skipped": 0, "failed": 0, "failed_pending": 0}
|
||||
assert counts == {
|
||||
"done": 2, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
|
||||
} # fmt: skip
|
||||
# M9: a replay for a model the ledger never ran has every unit pending
|
||||
_rows, replay = family_origin(
|
||||
c, s, FAM, ask=None, log=RunLog(tmp_path / "l.jsonl"), books=[],
|
||||
replay_model="another",
|
||||
) # fmt: skip
|
||||
assert replay["pending"] == 2
|
||||
|
||||
|
||||
def _chain_store(tmp_path, paras, *, volume="87"):
|
||||
@@ -1115,6 +1123,7 @@ def test_family_origin_resumes_and_replays_without_a_model(tmp_path):
|
||||
"skipped": 0,
|
||||
"failed": 0,
|
||||
"failed_pending": 0,
|
||||
"pending": 0,
|
||||
}
|
||||
# no model and nothing to replay: only the deterministic guidance rows
|
||||
none, _ = family_origin(_con(), s, FAM, ask=None, log=None, books=books)
|
||||
|
||||
@@ -1055,8 +1055,15 @@ def test_family_rounds_end_to_end_and_resume(tmp_path):
|
||||
s, key = _store(tmp_path, with_text=False)
|
||||
ask = FakeAsk(_good_reply())
|
||||
log = RunLog(tmp_path / "ledger.jsonl")
|
||||
# M9: a replay over an empty ledger counts the unit as pending
|
||||
_rows0, c0 = family_rounds(
|
||||
None, s, FAM, ask=None, log=RunLog(tmp_path / "empty.jsonl"), replay_model="m"
|
||||
)
|
||||
assert c0["pending"] == 1
|
||||
rows, counts = family_rounds(None, s, FAM, ask=ask, log=log)
|
||||
assert counts == {"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0}
|
||||
assert counts == {
|
||||
"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0,
|
||||
} # fmt: skip
|
||||
by = {r.code: r for r in rows}
|
||||
r = by["99341"]
|
||||
assert (r.proposed_work_rvu, r.final_work_rvu, r.ruc_work_rvu) == (1.0, 1.0, 1.0)
|
||||
|
||||
Reference in New Issue
Block a user