diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 1ab1481..f18cccc 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -21,7 +21,7 @@ from __future__ import annotations import json import logging from collections import Counter -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from dataclasses import replace from pathlib import Path from types import SimpleNamespace @@ -126,6 +126,7 @@ from pfs.stance import ( docket_events, docket_rule_year, eval_record, + event_kinds_of, event_subjects, fixed_text, golden_dockets, @@ -1712,8 +1713,11 @@ def rounds( typer.echo( f"rounds: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, " f"failed {counts['failed']}, failed_pending {counts['failed_pending']}" + + (f", pending {counts['pending']}" if counts.get("pending") else "") ) if write: + if not tables_only: # --tables-only asks for no model rows + _refuse_pending(counts, getattr(ask, "model", None) or _model_name()) with _batch() as con: ensure_tables(con) n = write_rounds(con, fam.key, rows) @@ -1805,8 +1809,10 @@ def origin( typer.echo( f"origins: {len(rows)}; done {counts['done']}, skipped {counts['skipped']}, " f"failed {counts['failed']}, failed_pending {counts['failed_pending']}" + + (f", pending {counts['pending']}" if counts.get("pending") else "") ) if write: + _refuse_pending(counts, getattr(ask, "model", None) or _model_name()) with _batch() as con: ensure_tables(con) n = write_origin(con, fam.key, rows) @@ -1866,8 +1872,10 @@ def guideline_facts_cmd( f"{counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, " f"failed_pending {counts.get('failed_pending', 0)}, " f"cited by section (no page) {counts.get('unpaged', 0)}" + + (f", pending {counts['pending']}" if counts.get("pending") else "") ) if write: + _refuse_pending(counts, getattr(ask, "model", None) or _model_name()) with _batch() as bcon: ensure_tables(bcon) n = write_guideline_facts(bcon, fam.key, rows) @@ -1919,8 +1927,10 @@ def cms_summaries( f"summaries: {len(rows)}; done {counts.get('done', 0)}, " f"skipped {counts.get('skipped', 0)}, failed {counts.get('failed', 0)}, " f"failed_pending {counts.get('failed_pending', 0)}" + + (f", pending {counts['pending']}" if counts.get("pending") else "") ) if write: + _refuse_pending(counts, model) with _batch() as con: ensure_tables(con) n = write_cms_summaries(con, fam.key, rows) @@ -2212,6 +2222,8 @@ def _method_eval( rec = eval_record(primary, sc) rec.pop("docket") rec["generation"] = gen + # M2: which event kinds the set holds positions on, stamped beside it + rec["event_kinds"] = event_kinds_of(primary) if golden.exists(): tuning = json.loads(golden.read_text(encoding="utf-8")) if tuning.get("family") == family: @@ -2406,6 +2418,18 @@ def stance_eval( ) +def _refuse_pending(counts: Mapping[str, int], model: str | None) -> None: + """M9: refuse a --write while model units have no ledger entry (a + prompt-version bump, a --limit cut): their rows would silently be + missing from the table.""" + n = counts.get("pending", 0) + if n: + raise typer.BadParameter( + f"{n} units have no ledger entry for {model or 'the model'}; rerun " + "without --no-llm (or with a larger --limit) before --write" + ) + + def _model_of(ask: Any, large: bool) -> str: """The pinned model's name: the asker's, or (no asker) the configured one.""" if ask is not None: @@ -2438,6 +2462,12 @@ def dossier_cmd( "--dry-run", help="Count facts and timeline events; no model calls, no writes.", ), + template_ok: bool = typer.Option( + False, + "--template-ok", + help="Let --write store template copy for event lines the prose " + "ledger has no entry for (otherwise refused).", + ), write: bool = typer.Option(False, "--write"), ) -> None: """Assemble a family's dossier (pfs.dossier): every stored fact with its @@ -2462,7 +2492,7 @@ def dossier_cmd( try: if dry_run: facts = collect_facts(con, store, fam) - events = dossier_timeline(facts) + events = dossier_timeline(facts, fam.codes) drafted = [e for e in events if not template_only(e.kind)] typer.echo( f"{fam.key}: {len(facts)} facts, {len(events)} timeline events; " @@ -2471,15 +2501,19 @@ def dossier_cmd( "comments, link and CPT edition lines are template copy)" ) continue + ask = None if no_llm else _asker() + pending: list[str] = [] d = build_dossier( con, store, fam, - ask=None if no_llm else _asker(), - excerpts=[] if no_llm else _excerpts(store, fam.name), + ask=ask, + # M7: embeddings, not generation: kept under --no-llm + excerpts=_excerpts(store, fam.name), log=RunLog(ledger_path("prose", fam.key)), replay_model=_model_name() if no_llm else None, limit=limit, + pending=pending, ) finally: con.close() @@ -2502,6 +2536,22 @@ def dossier_cmd( ) if d.extractor != "template" and d.fallbacks: typer.echo("template fallback: " + "; ".join(d.fallbacks)) + model = getattr(ask, "model", None) or _model_name() + if pending: + typer.echo( + f"{len(pending)} event lines have no prose ledger entry for " + f"{model} (template copy): " + "; ".join(pending) + ) + if write and pending and not template_ok: + # M9: never a silent fallback after the facts or the prompt changed + raise typer.BadParameter( + f"{len(pending)} event lines have no prose ledger entry for {model}; " + "rerun without --no-llm, or pass --template-ok" + ) + if write and not d.check_passed: + # M8: a dossier whose prose failed the check is never stored + typer.echo(f"{fam.key}: check FAILED; not written") + raise typer.Exit(1) built.append(d) if write and built: with _batch() as bcon: diff --git a/src/pfs/cms_summaries.py b/src/pfs/cms_summaries.py index 2b510ea..47c8225 100644 --- a/src/pfs/cms_summaries.py +++ b/src/pfs/cms_summaries.py @@ -184,10 +184,15 @@ def summaries( ) if counts is not None: counts.update(result) + pending = 0 for u in units: entry = log.get(u, hashes[u], model_name, SUMMARY_PROMPT_VERSION) + pending += entry is None if entry is not None and entry.output is not None: recorded[u] = entry.output["stance"] + # M9: pairs with no ledger answer, written as "unclear" by regex + if counts is not None: + counts["pending"] = pending extractor = f"model:{model_name}@{SUMMARY_PROMPT_VERSION}" return [ CmsSummaryRow( diff --git a/src/pfs/guideline_facts.py b/src/pfs/guideline_facts.py index 7b67a5a..1539d53 100644 --- a/src/pfs/guideline_facts.py +++ b/src/pfs/guideline_facts.py @@ -803,8 +803,11 @@ def guideline_facts( run.update(counts_run) extractor = f"model:{model}@{GUIDELINE_PROMPT_VERSION}" scope: dict[int, list[list[GuidelineFactRow]]] = {} + # M9: units with no ledger answer, so a write would miss their rows + run["pending"] = 0 for j in jobs: entry = log.get(j["unit"], j["hash"], model, GUIDELINE_PROMPT_VERSION) + run["pending"] += entry is None if entry is None or entry.output is None: continue mine = [ diff --git a/src/pfs/origin.py b/src/pfs/origin.py index 13f3798..8a4c2d6 100644 --- a/src/pfs/origin.py +++ b/src/pfs/origin.py @@ -1185,10 +1185,14 @@ def family_origin( limit=limit, retry_failed=retry_failed, ) + pending = 0 for j in jobs: entry = log.get(j["unit"], j["hash"], model, ORIGIN_PROMPT_VERSION) + pending += entry is None if entry is not None and entry.output is not None: model_rows += _rows(j, entry.output, family, model) + # M9: units with no ledger answer, so a write would miss their rows + counts = {**counts, "pending": pending} found = _drop_misstated(model_rows, events) + lineage_origins(store, family, events) rows = _consolidate(found) + guidance_origins(con, family.key, store) rows.sort( diff --git a/src/pfs/rounds.py b/src/pfs/rounds.py index 65c970d..5906cdc 100644 --- a/src/pfs/rounds.py +++ b/src/pfs/rounds.py @@ -1332,13 +1332,18 @@ def family_rounds( limit=limit, retry_failed=retry_failed, ) + pending = 0 for p in naming: entry = log.get( _unit(p), hashes[_unit(p)], model_name, ROUND_PROMPT_VERSION ) + pending += entry is None if entry is not None and entry.output is not None: model += rows_from_output( entry.output, p, codes, family=family.key, model=model_name ) + # M9: units the ledger has no answer for (a prompt-version bump, a + # --limit cut): their model rows would be missing from a write + counts = {**counts, "pending": pending} merged = apply_paragraph_rules(merge_rounds(table, regex, model), paras) return [r for r in merged if has_content(r)], counts diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index 2abc35a..c781c04 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -2678,7 +2678,9 @@ class TestStanceMethodRecord: prim = tmp_path / "p.json" prim.write_text(json.dumps({ "family": family, "dockets": ["H1", "H2"], "in_sample": False, - "role": "primary", "generation": generation, "items": [{}, {}, {}], + "role": "primary", "generation": generation, + "items": [{"cells": {"F:2026:G2211": {"label": "support"}, + "F:2026:other": {"label": "none"}}}, {}, {}], })) # fmt: skip foot = tmp_path / "f.json" foot.write_text(json.dumps({ @@ -2707,7 +2709,7 @@ class TestStanceMethodRecord: assert got == { "cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 3, "n_positions": 27, "family": "HOME-VISITS", "dockets": ["H1", "H2"], - "in_sample": False, "generation": "stance-v5", + "in_sample": False, "generation": "stance-v5", "event_kinds": ["G2211"], "in_sample_record": { "cell": 0.83, "recall": 0.556, "direction": 0.667, "n_items": 2, "n_positions": 27, "docket": "T1", @@ -3751,15 +3753,28 @@ class TestDossier: assert "check: passed; prose template" in res.output assert "timeline years: 2008" in res.output assert "stakeholders (CY2023, HOME-VISITS:2023:valuation" in res.output - assert env.published == [] and env.excerpts == [] + # M7: governance excerpts are embeddings, not generation: kept + assert env.published == [] and env.excerpts == ["Home or Residence Services"] from pfs.dossier import dossier_families assert dossier_families(env.con) == () def test_write_stores_the_dossier_and_publishes(self, env): + # M9: the prose ledger has no entry for the event lines: refused ... res = runner.invoke( app, ["pfs", "dossier", "--family", "home-visits", "--no-llm", "--write"] ) + assert res.exit_code != 0 + assert "event lines have no prose ledger entry for fake-14b" in res.output + assert env.published == [] + # ... unless template copy for them is asked for + res = runner.invoke( + app, + [ + "pfs", "dossier", "--family", "home-visits", "--no-llm", "--write", + "--template-ok", + ], + ) # fmt: skip assert res.exit_code == 0, res.output from pfs.dossier import read_dossier @@ -3768,6 +3783,27 @@ class TestDossier: assert "wrote dossier HOME-VISITS" in res.output assert env.published == [True] + def test_a_failed_check_is_never_written(self, env, monkeypatch): + """M8: a dossier whose check failed is printed, never stored.""" + real = pfs_cli.build_dossier + + def failing(*a, **k): + return dataclasses.replace(real(*a, **k), check_passed=False) + + monkeypatch.setattr(pfs_cli, "build_dossier", failing) + res = runner.invoke( + app, + [ + "pfs", "dossier", "--family", "home-visits", "--no-llm", "--write", + "--template-ok", + ], + ) # fmt: skip + assert res.exit_code != 0 and "check: FAILED" in res.output + assert "not written" in res.output and env.published == [] + from pfs.dossier import dossier_families + + assert dossier_families(env.con) == () + def test_all_builds_every_curated_family(self, env): res = runner.invoke(app, ["pfs", "dossier", "--all", "--no-llm"]) assert res.exit_code == 0, res.output @@ -3830,3 +3866,71 @@ def test_dossier_excerpts_use_the_llm_pool(monkeypatch): ) assert pfs_cli._excerpts("STORE", "Home or Residence Services") == [] assert calls == [("CFG", "POOL", "STORE", "Home or Residence Services")] + + +class TestPendingLedger: + """M9: a --write whose model units have no ledger entry (a prompt-version + bump, a --limit cut) is refused, never a silent drop of model rows.""" + + PENDING = {"done": 0, "skipped": 3, "failed": 0, "failed_pending": 0, "pending": 2} + + def test_rounds(self, con, monkeypatch): + TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING)) + monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x") + monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0) + res = runner.invoke( + app, ["pfs", "rounds", "--family", "HOME-VISITS", "--no-llm", "--write"] + ) + assert res.exit_code != 0 + assert "2 units have no ledger entry for qwen-x" in res.output + # --tables-only says no model rows are wanted: not refused + TestRounds._patch(TestRounds(), monkeypatch, [], counts=dict(self.PENDING)) + res = runner.invoke( + app, ["pfs", "rounds", "--family", "HOME-VISITS", "--tables-only"] + ) + assert res.exit_code == 0 and "pending 2" in res.output + + def test_origin(self, con, monkeypatch): + TestOrigin._patch(TestOrigin(), monkeypatch, []) + monkeypatch.setattr( + pfs_cli, "family_origin", lambda *a, **k: ([], dict(self.PENDING)) + ) + monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x") + monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0) + res = runner.invoke( + app, ["pfs", "origin", "--family", "HOME-VISITS", "--no-llm", "--write"] + ) + assert res.exit_code != 0 and "2 units have no ledger entry" in res.output + + def test_guideline_facts(self, con, monkeypatch): + def fake(con_, store, fam, **kw): + kw["counts"].update(self.PENDING, unpaged=0) + return [] + + monkeypatch.setattr(pfs_cli, "guideline_facts", fake) + monkeypatch.setattr(pfs_cli, "_model_name", lambda large=False: "qwen-x") + monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0) + res = runner.invoke( + app, + [ + "pfs", + "guideline-facts", + "--family", + "HOME-VISITS", + "--no-llm", + "--write", + ], + ) + assert res.exit_code != 0 and "2 units have no ledger entry" in res.output + + def test_cms_summaries(self, con, monkeypatch): + seen: dict = {} + TestCmsSummaries._patch( + TestCmsSummaries(), monkeypatch, seen, counts=dict(self.PENDING) + ) + monkeypatch.setattr(pfs_cli, "_batch", lambda: 1 / 0) + res = runner.invoke( + app, + ["pfs", "cms-summaries", "--family", "HOME-VISITS", "--no-llm", "--write"], + ) + assert res.exit_code != 0 and "2 units have no ledger entry" in res.output diff --git a/tests/pfs/test_cms_summaries.py b/tests/pfs/test_cms_summaries.py index 1fe0b68..82aac7e 100644 --- a/tests/pfs/test_cms_summaries.py +++ b/tests/pfs/test_cms_summaries.py @@ -323,14 +323,21 @@ def test_summaries_ledger_resume_and_replay(tmp_path): # --no-llm replay: same rows, zero calls replay = summaries(s, FAM, log=log, replay_model="m") assert replay == first and len(calls) == n - # unrecorded under a different model: unclear / regex - other = summaries(s, FAM, log=log, replay_model="other") + # unrecorded under a different model: unclear / regex, and (M9) every + # pair counted as pending + pending: dict = {} + other = summaries(s, FAM, log=log, replay_model="other", counts=pending) assert {r.stance for r in other} == {"unclear"} assert {r.extractor for r in other} == {"regex"} - # limit caps new calls; the rest stay regex + assert pending == {"pending": n} + # limit caps new calls; the rest stay regex and pending fresh = RunLog(tmp_path / "l2.jsonl") - capped = summaries(s, FAM, classify=classify, model="m", log=fresh, limit=1) + cut: dict = {} + capped = summaries( + s, FAM, classify=classify, model="m", log=fresh, limit=1, counts=cut + ) assert sum(r.extractor != "regex" for r in capped) == 1 + assert cut["pending"] == n - 1 finally: s.close() diff --git a/tests/pfs/test_guideline_facts.py b/tests/pfs/test_guideline_facts.py index fd68b3e..3c25b69 100644 --- a/tests/pfs/test_guideline_facts.py +++ b/tests/pfs/test_guideline_facts.py @@ -509,8 +509,18 @@ def test_scope_points_through_ledger(con, books, tmp_path): calls.clear() gf.guideline_facts(con, object(), FAM, ask=ask, log=log) assert calls == [] # resume: no new calls - rows2 = gf.guideline_facts(con, object(), FAM, log=log, replay_model="qwen-test") - assert _by(rows2, fact="scope_point") == pts + replayed: dict = {} + rows2 = gf.guideline_facts( + con, object(), FAM, log=log, replay_model="qwen-test", counts=replayed + ) + assert _by(rows2, fact="scope_point") == pts and replayed["pending"] == 0 + # M9: a replay for a model the ledger never ran counts every unit pending + none: dict = {} + gf.guideline_facts(con, object(), FAM, log=log, replay_model="other", counts=none) + assert none["pending"] == 2 and not _by( + gf.guideline_facts(con, object(), FAM, log=log, replay_model="other"), + fact="scope_point", + ) WORDS = ( diff --git a/tests/pfs/test_origin.py b/tests/pfs/test_origin.py index 2daed4d..dfeeb37 100644 --- a/tests/pfs/test_origin.py +++ b/tests/pfs/test_origin.py @@ -914,7 +914,15 @@ def test_family_origin_fr_rows_cite_the_paragraph(tmp_path): ] assert fr[0].locator == "87 FR 100, ΒΆ2" and fr[0].kind == "statute" assert [r.cite_kind for r in rows if r.extractor == "guidance"] == ["cfr"] - assert counts == {"done": 2, "skipped": 0, "failed": 0, "failed_pending": 0} + assert counts == { + "done": 2, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0, + } # fmt: skip + # M9: a replay for a model the ledger never ran has every unit pending + _rows, replay = family_origin( + c, s, FAM, ask=None, log=RunLog(tmp_path / "l.jsonl"), books=[], + replay_model="another", + ) # fmt: skip + assert replay["pending"] == 2 def _chain_store(tmp_path, paras, *, volume="87"): @@ -1115,6 +1123,7 @@ def test_family_origin_resumes_and_replays_without_a_model(tmp_path): "skipped": 0, "failed": 0, "failed_pending": 0, + "pending": 0, } # no model and nothing to replay: only the deterministic guidance rows none, _ = family_origin(_con(), s, FAM, ask=None, log=None, books=books) diff --git a/tests/pfs/test_rounds.py b/tests/pfs/test_rounds.py index d7e182c..c79a0e5 100644 --- a/tests/pfs/test_rounds.py +++ b/tests/pfs/test_rounds.py @@ -1055,8 +1055,15 @@ def test_family_rounds_end_to_end_and_resume(tmp_path): s, key = _store(tmp_path, with_text=False) ask = FakeAsk(_good_reply()) log = RunLog(tmp_path / "ledger.jsonl") + # M9: a replay over an empty ledger counts the unit as pending + _rows0, c0 = family_rounds( + None, s, FAM, ask=None, log=RunLog(tmp_path / "empty.jsonl"), replay_model="m" + ) + assert c0["pending"] == 1 rows, counts = family_rounds(None, s, FAM, ask=ask, log=log) - assert counts == {"done": 1, "skipped": 0, "failed": 0, "failed_pending": 0} + assert counts == { + "done": 1, "skipped": 0, "failed": 0, "failed_pending": 0, "pending": 0, + } # fmt: skip by = {r.code: r for r in rows} r = by["99341"] assert (r.proposed_work_rvu, r.final_work_rvu, r.ruc_work_rvu) == (1.0, 1.0, 1.0)