From 66341094848eb5a5c6d3650d711ec91bd90d0c67 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:06:56 -0400 Subject: [PATCH 01/30] docs: LEAD evidence chassis design and part 1 plan --- .../2026-10-31-lead-evidence-chassis-part1.md | 733 ++++++++++++++++++ ...2026-10-31-lead-evidence-chassis-design.md | 484 ++++++++++++ 2 files changed, 1217 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-31-lead-evidence-chassis-part1.md create mode 100644 docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md diff --git a/docs/superpowers/plans/2026-10-31-lead-evidence-chassis-part1.md b/docs/superpowers/plans/2026-10-31-lead-evidence-chassis-part1.md new file mode 100644 index 0000000..28fe438 --- /dev/null +++ b/docs/superpowers/plans/2026-10-31-lead-evidence-chassis-part1.md @@ -0,0 +1,733 @@ +# LEAD evidence chassis, Part 1 — implementation plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A five-strand PRISMA review of LEAD whose counts, decisions, and claims land in an append-only evidence ledger a later modeling loop can read. + +**Architecture:** Zotero stays the source of truth for documents, criteria, and screening decisions. New `prisma` pieces (search, dedup, calibration, audit, the flow diagram) write tags and notes there. `lead.targets` names every scored LEAD component. `lead.ledger` mirrors those decisions into a DuckDB file whose schema is the `lead_evidence` schema, append-only and keyed so a re-export changes nothing. Parts 2–4 of the spec (the math kernel, the modeling loop, the agent) are out of scope. + +**Tech Stack:** Python, Typer (`stack `), the existing Zotero sqlite schema (`zot.schema.create_db`), DuckDB, pytest. No new service, no live HTTP in unit tests. + +**Spec:** `docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md` + +## Global Constraints + +- Public data and public research only. No PHI, CCLF, or client data, and no dependency on the deleted LEAD repository. +- Lives in `stack`. Tests in `tests//`. Commands are `stack `, idempotent, and touchless. +- Licensed reference content (AMA CPT, ICD codebooks, UMLS, SNOMED, VSAC) is not published or redistributed. +- Ledger rows are append-only. Every row has `created_at`, `actor`, and `content_hash`. A second insert of the same `content_hash` inserts nothing. +- An API or model failure never writes a screening decision. `stack prisma screen` and the screen stage of `stack prisma run` exit non-zero when the error count is above zero. `prisma.screen.run` still returns the stats dict. +- The Sci-Hub fallback stays available for `lead-history`, `lead-limits`, `lead-scan`, `lead-precursors`, and `lead-external`. A project named in `SCIHUB_OFF` skips it even when a proxy client exists. +- Searches re-run nightly. A re-run inserts a new `search` row. It never overwrites an older one. +- Calibration before a production screen: 200 title/abstract labels and 50 full-text labels per strand, sensitivity ≥ 0.95 on include-or-uncertain. +- Production screening is two passes with different prompt texts, the same model. Disagreements go to a person. A deterministic 10% of agreed excludes is the audit sample. `uncertain` always advances. +- Strand 4 excludes a cross-sectional study with no time order, and says so in the criteria. +- Suspecting is a status transition by code uplift or by care. Both phrases are in the `lead-limits` and `lead-precursors` criteria. +- The ledger schema name is `lead_evidence`. Per bundle target it sits in the catalog `databricks.yml` already selects: `aco_dev`, `aco_staging`, or `aco`. +- OSF registration is an outward step and is not a task in this plan. + +## Review Focus + +These are the inputs most likely to be mishandled. Each one has a test on the task that owns it. + +- A provider exception during screening must leave the item untagged and the CLI must exit 1. Task 3. +- Two searches with the same query string on different days must produce two ledger rows. Task 7. +- A near-duplicate title must stay in `borderline` and out of an auto-merged group. Task 6. +- Exporting the same Zotero item and extraction version twice must insert one claim. Task 13. +- A flow diagram must print the count read from tags, including a source that contributed zero records. Task 11. + +## File structure + +- `src/lead/targets.py` — the target registry. Nothing else. +- `src/lead/ledger/schema.py` — DDL for DuckDB and for the Delta schema. +- `src/lead/ledger/store.py` — open, append, status. +- `src/lead/ledger/export.py` — Zotero tags and extraction notes to ledger rows. +- `src/lead/ledger/claims.py` — one extraction to claims, GRADE start, pool gate, precursor catalog. +- `src/cli/lead.py` — `stack lead targets` and `stack lead ledger`. +- `src/prisma/dedup.py` — match keys and clusters. +- `src/prisma/search/base.py` — hit, log, and searcher types. +- `src/prisma/search/pubmed.py` and sibling modules — fixture parsers, one database each. +- `src/prisma/search/protocols.py` — the query strings each strand runs. +- `src/prisma/search/due.py` — nightly and triggered due reasons. +- `src/prisma/calibrate.py` — sensitivity and the size gate. +- `src/prisma/audit.py` — disagreements, agreed excludes, the 10% sample. +- `src/prisma/harvest.py` — grey-literature HTML parsers. +- `src/prisma/congress.py` — Congress.gov payload parsers and a client that takes a transport. +- `src/bls/ppi.py`, `src/bls/jolts.py`, `src/bls/qcew.py` — BLS series parsers beside OEWS and CES. +- `src/econ/fred.py`, `src/econ/irs_soi.py` — FRED and IRS are not BLS products. +- Modify `src/prisma/project.py`, `src/prisma/fetch.py`, `src/prisma/flow.py`, `src/cli/prisma.py`, `src/cli/__init__.py`. + +--- + +### Task 1: Target registry + +**Files:** +- Create: `src/lead/targets.py` +- Create: `src/cli/lead.py` +- Modify: `src/cli/__init__.py` +- Test: `tests/lead/test_targets.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `Target(id: str, family: str, name: str)`, `TARGETS: tuple[Target, ...]`, `by_id(target_id: str) -> Target` (KeyError on a miss), `families() -> dict[str, tuple[Target, ...]]` + +- [ ] **Step 1: Write the failing test** + +```python +from lead.targets import TARGETS, by_id + +def test_registry_matches_the_spec(): + assert {t.id for t in TARGETS} == EXPECTED_IDS + assert by_id("HN.3").name.startswith("Kim") + assert by_id("Q.ACR").family == "quality" +``` + +`EXPECTED_IDS` is exactly: + +`Q.ACR`, `Q.UAMCC`, `Q.DAH`, `Q.TFU`, `Q.CAHPS`, `Q.CBP`, `Q.GSD`, `HN.1`, `HN.2`, `HN.3`, `HN.4`, `HN.5`, `HN.6`, `HN.7`, `RA.PROSP`, `RA.CONC`, `RA.ESRD`, `RA.NORM`, `RA.CI`, `RA.CAP`, `RA.SAHS`, `BM.HISTORICAL`, `BM.VOLUNTARY_GUARDRAIL`, `BM.REGIONAL`, `BM.SPENDING_DESIGNATION`, `BM.ADJ_CAP`, `BM.REGIONAL_EFFICIENCY`, `BM.PRIOR_SAVINGS`, `BM.PY`, `BM.DISCOUNT`, `BM.QUALITY_WITHHOLD`, `BM.QUALITY_EARN_BACK`, `BM.RETENTION`, `BM.GROSS`, `BM.SHARED`, `BM.SEQUESTRATION`. + +Names, copied from the spec for the non-benchmark ids: `HN.1` mobility-impairment diagnosis, `HN.2` hospital bed or transfer-equipment DME claim, `HN.3` Kim claims-based frailty index, `HN.4` A&D risk score ≥ 3.0, `HN.5` ESRD risk score ≥ 0.35, `HN.6` A&D 2.0–3.0 or ESRD 0.24–0.35 plus ≥ 2 unplanned admissions, `HN.7` ≥ 45 covered SNF days. `RA.PROSP` non-High Needs A&D, `RA.CONC` High Needs concurrent, `RA.CAP` 4% High Needs growth cap, `RA.SAHS` anomalous-billing and skin-substitute mitigation. The `BM.*` names are the `src/lead` functions they point at: `historical_benchmark`, `voluntary_guardrail`, `regional_ffs_expenditure`, `spending_designation`, `benchmark_adjustment_cap`, `regional_efficiency_adjustment`, `prior_savings_adjustment`, `performance_year_benchmark_pbpm`, `apply_discount`, `quality_withhold`, `quality_earn_back`, `retention_incentive_adjustment`, `gross_savings_losses`, `shared_savings_losses`, `sequestration_amount`. + +Also assert `by_id("nope")` raises `KeyError`, and `CliRunner().invoke(app, ["lead", "targets"])` exits 0 and its stdout contains `HN.3` and `BM.SEQUESTRATION`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/lead/test_targets.py -q` +Expected: FAIL, `lead.targets` missing. + +- [ ] **Step 3: Implement the registry and `stack lead targets`** + +`TARGETS` is a module-level tuple in the order above. `stack lead targets` prints `id\tfamily\tname`, one target per line. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/lead/test_targets.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/lead/targets.py src/cli/lead.py src/cli/__init__.py tests/lead/test_targets.py +git commit -m "feat(lead): add the LEAD target registry" +``` + +### Task 2: Ledger schema, append, and status + +**Files:** +- Create: `src/lead/ledger/__init__.py` +- Create: `src/lead/ledger/schema.py` +- Create: `src/lead/ledger/store.py` +- Modify: `src/cli/lead.py` +- Test: `tests/lead/test_ledger_store.py` + +**Interfaces:** +- Consumes: `lead.targets.TARGETS`, `by_id` +- Produces: `schema_sql(dialect: str) -> str` (`"duckdb"` or `"delta"`), `open_ledger(path: str) -> duckdb.DuckDBPyConnection`, `append(con, table: str, row: Mapping[str, object]) -> bool`, `status(con) -> list[TargetStatus]`. `TargetStatus` has `target_id`, `claims_by_tier` (`rule`, `aggregate`, `literature`, `synthetic`), `gaps` (1 when the target has no claim), `last_run`, `open_contradictions`. + +Tables, all in schema `lead_evidence`: `source`, `search`, `decision`, `extraction`, `claim`, `feature`, `run`, `finding`, `exogenous`. Every table has `created_at`, `actor`, `content_hash`. `claim` also has `kind` (`precursor`, `limit`, `lineage`, `context`, `method`), `subject`, `target_id`, `direction`, `effect_measure`, `effect_value`, `effect_ci_low`, `effect_ci_high`, `lead_time`, `population`, `setting`, `grain` (`beneficiary`, `provider`, `ACO`, `area`), `design`, `risk_of_bias`, `certainty`, `extraction_ids`, `tiers`. `search` also has `database`, `interface`, `searched_on`, `query`, `filters`, `hit_count`, `export_sha256`. `decision` also has `zotero_key`, `stage`, `decision`, `reason`, `model`, `prompt_hash`, `criteria_hash`, `human_audited`. `finding` also has `claim_id`, `relation` (`confirms`, `contradicts`, `inconclusive`), `grain`. `run` also has `mlflow_run_id`, `where_ran` (`local` or `databricks`), `finished_on`. `source` also has `zotero_key`, `doi`, `database`. + +- [ ] **Step 1: Write the failing test** + +```python +def test_append_is_idempotent_and_status_covers_every_target(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + row = claim_row(target_id="Q.ACR", tiers="literature", content_hash="abc") + assert append(con, "claim", row) is True + assert append(con, "claim", row) is False + assert con.execute("select count(*) from lead_evidence.claim").fetchone()[0] == 1 + found = {s.target_id: s for s in status(con)} + assert set(found) == {t.id for t in TARGETS} + assert found["Q.ACR"].claims_by_tier["literature"] == 1 + assert found["HN.1"].gaps == 1 +``` + +Also assert `append` of a claim whose `target_id` is `nope` raises `KeyError`, `schema_sql("delta")` contains `lead_evidence` and `aco` is not hardcoded, and `schema_sql("nope")` raises `ValueError`. A contradicting `finding` for a claim bumps `open_contradictions` on that claim's target. `stack lead ledger status --db ` prints `Q.ACR` and exits 0. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/lead/test_ledger_store.py -q` +Expected: FAIL, ledger modules missing. + +- [ ] **Step 3: Implement schema, append, and status** + +`append` computes nothing: the caller supplies `content_hash`. The store rejects a row whose hash is already in that table and returns `False`. `created_at` is set by the store when the row omits it. DuckDB gets the schema applied in `open_ledger`. The Delta string is the same columns with `CREATE TABLE IF NOT EXISTS` and no catalog name, so the bundle target's catalog wraps it. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/lead/test_ledger_store.py tests/lead/test_targets.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/lead/ledger src/cli/lead.py tests/lead/test_ledger_store.py +git commit -m "feat(lead): append-only evidence ledger in DuckDB" +``` + +### Task 3: A screening error is not a decision + +**Files:** +- Modify: `src/cli/prisma.py` (`screen`, and the screen stage inside `run`) +- Test: `tests/cli/test_prisma_screen_errors.py` + +**Interfaces:** +- Consumes: `prisma.screen.run` stats key `errors` +- Produces: CLI exit code 1 when that count is above zero, after the counts are printed. `prisma.screen.run` does not raise. + +- [ ] **Step 1: Write the failing test** + +Patch `prisma.screen.run` to return `{"screened": 0, "include": 0, "exclude": 0, "uncertain": 0, "errors": 2}`. `CliRunner.invoke` on `screen` exits 1 and the output contains `errors`. The same return from the screen stage of `run` exits 1. A return with `errors: 0` exits 0. The existing `tests/prisma/test_screen_exercise.py::TestRun::test_handles_error` still expects `run()` to return `errors == 1` and write no decision. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/cli/test_prisma_screen_errors.py tests/prisma/test_screen_exercise.py::TestRun::test_handles_error -q` +Expected: the new test FAILs because the CLI exits 0. + +- [ ] **Step 3: Implement the exit** + +After printing stats, `raise typer.Exit(1)` when `errors > 0`. Do that in `screen` and after the screen stage of `run`. Do not change `prisma.screen.run`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/cli/test_prisma_screen_errors.py tests/prisma/test_screen_exercise.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/cli/prisma.py tests/cli/test_prisma_screen_errors.py +git commit -m "fix(prisma): fail the screen command when the model errors" +``` + +### Task 4: Sci-Hub switch, on for the LEAD projects + +**Files:** +- Modify: `src/prisma/fetch.py` (`fetch_one`, `run`) +- Test: `tests/prisma/test_fetch_fallback_switch.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `SCIHUB_OFF: frozenset[str]` (empty), `fallback_allowed(project: str) -> bool`, `fetch_one(..., allow_fallback: bool = True)`, `run(..., allow_fallback: bool = True)` + +- [ ] **Step 1: Write the failing test** + +```python +def test_lead_projects_keep_the_fallback(): + for name in ("lead-history", "lead-limits", "lead-scan", "lead-precursors", "lead-external"): + assert fallback_allowed(name) is True + +def test_named_project_skips_the_mirror(monkeypatch): + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + assert fallback_allowed("skin-subs") is False +``` + +Patch `fetch_fallback` to return a URL and patch `download` to return a path. `fetch_one(..., client_proxied=client, allow_fallback=False)` returns `None` and `fetch_fallback` was not called. The same call with `allow_fallback=True` returns `("fallback", path)`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_fetch_fallback_switch.py -q` +Expected: FAIL, `fallback_allowed` missing. + +- [ ] **Step 3: Implement the switch** + +`fallback_allowed` is `project not in SCIHUB_OFF`. `fetch_one` runs the existing fallback block only when `allow_fallback` is true and `client_proxied` is not None. `run` threads the flag through. The fetch CLI passes `allow_fallback=fallback_allowed(name)`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_fetch_fallback_switch.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/fetch.py src/cli/prisma.py tests/prisma/test_fetch_fallback_switch.py +git commit -m "feat(prisma): per-project Sci-Hub switch, on for LEAD" +``` + +### Task 5: Five LEAD starters + +**Files:** +- Modify: `src/prisma/project.py` (`STARTERS`) +- Test: `tests/prisma/test_lead_starters.py` + +**Interfaces:** +- Consumes: `prisma.tools.parse_extraction_fields`, `prisma.project.init` +- Produces: starter names `lead-history`, `lead-limits`, `lead-scan`, `lead-precursors`, `lead-external`. Each extraction template uses `- **field** — description` lines. Closed lists use ` / ` so `parse_extraction_fields` builds enums. + +- [ ] **Step 1: Write the failing test** + +`init` on a fresh db from `zot.schema.create_db` succeeds for each of the five names. `parse_extraction_fields` on each template contains these field names: + +- history: `target_id`, `origin`, `assumptions`, `supporting_evidence`, `critiques`, `changes_to_lead`, `status_in_lead`, `page_refs`. `status_in_lead` is an enum `adopted / modified / dropped`. +- limits: `target_id`, `limit_type`, `statement`, `formula`, `empirical_value`, `computable_expression`, `page_refs`. `limit_type` is `accuracy / movability / fairness / incentive`. +- scan: `item_type`, `item_date`, `summary`, `target_ids`, `aacods`, `chassis_meaning`. `item_type` is `policy / method / data / model / industry`. +- precursors: `design`, `data_source`, `population`, `n`, `setting`, `country`, `years`, `factor_definition`, `outcome_target_id`, `proxy_closeness`, `lead_time`, `grain`, `effect_measure`, `effect_value`, `effect_ci`, `adjusted_for`, `c_statistic`, `calibration`, `risk_of_bias`, `observable_public`, `observable_claims`. `grain` is `beneficiary / provider / ACO / area`. +- external: `series_or_event`, `dates`, `geography`, `source`, `mechanism`, `magnitude`. + +The `lead-precursors` criteria text contains `cross-sectional` and `temporal order`. The `lead-limits` and `lead-precursors` criteria each contain `code uplift`, `care`, and `status`. `init(db, "nope")` still raises `ValueError`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_lead_starters.py -q` +Expected: FAIL, names absent from `STARTERS`. + +- [ ] **Step 3: Implement the five starters** + +Follow the existing triple `(criteria, extraction, reasons)` in `STARTERS`. Criteria state the strand's question from the spec in a few sentences, including the two pins above. Reasons include `cross-sectional-no-time-order` for precursors. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_lead_starters.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/project.py tests/prisma/test_lead_starters.py +git commit -m "feat(prisma): add the five LEAD review starters" +``` + +### Task 6: Dedup + +**Files:** +- Create: `src/prisma/dedup.py` +- Modify: `src/cli/prisma.py` +- Test: `tests/prisma/test_dedup.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `Record(key: str, doi: str, pmid: str, title: str, year: int | None, first_author: str)`, `DedupResult(groups: tuple[tuple[Record, ...], ...], borderline: tuple[tuple[Record, Record], ...])`, `cluster(records: Sequence[Record]) -> DedupResult`, `normalize(title: str) -> str`. The `dedup` command writes tag `dedup:duplicate` on every record in a group except the first, and writes nothing on a borderline pair. + +- [ ] **Step 1: Write the failing test** + +Exact DOI match, ignoring case and a trailing period, is one group. Same PMID is one group. Same normalized title (`"Risk Adjustment!"` → `risk adjustment`), same year, same first author is one group. Titles `"High needs frailty index"` and `"High-need frailty index"`, same year, different author, are a borderline pair and two groups of one. A pair below that similarity is neither merged nor borderline. `normalize` lowercases, drops punctuation, and collapses whitespace. On a Zotero db, two items that share a DOI end with exactly one `dedup:duplicate` tag between them. Two borderline items have no `dedup:duplicate` tag. `stack prisma dedup` is registered (`--help` exits 0). + +Similarity rule: `difflib.SequenceMatcher` ratio ≥ 0.85, same year, and not an exact key match → borderline, not merged. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_dedup.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement `cluster`** + +Match order is DOI, then PMID, then normalized title + year + first author. Only the leftover pairs are eligible for the borderline test. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_dedup.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/dedup.py src/cli/prisma.py tests/prisma/test_dedup.py +git commit -m "feat(prisma): dedup on DOI, PMID, then title and year" +``` + +### Task 7: Search log, PubMed parser, seed recall + +**Files:** +- Create: `src/prisma/search/__init__.py` +- Create: `src/prisma/search/base.py` +- Create: `src/prisma/search/pubmed.py` +- Create: `src/prisma/search/protocols.py` +- Modify: `src/cli/prisma.py` +- Test: `tests/prisma/test_search_pubmed.py` + +**Interfaces:** +- Consumes: `lead.ledger.store.append` +- Produces: `SearchHit(title, year, doi, pmid, first_author, database)`, `SearchLog(database, interface, searched_on, query, filters, hit_count, export_sha256)`, `SearchResult(hits, log)`, `parse_pubmed_xml(xml: str, *, query: str, searched_on: str) -> SearchResult`, `seed_recall(retrieved: set[str], seeds: set[str]) -> float`, `PROTOCOLS: dict[str, tuple[str, ...]]` keyed by the five project names. `record(con, result: SearchResult, *, actor: str) -> bool` appends a `search` row. `import_hits(zdb: Db, project: str, result: SearchResult) -> int` creates one Zotero item per hit, tagged `project:` and `found-in:`, and returns how many new items it inserted. A second import of the same DOI inserts nothing. + +- [ ] **Step 1: Write the failing test** + +A two-article PubmedArticleSet fixture parses to two hits, `database == "pubmed"`, and `hit_count == 2`. `export_sha256` is the sha256 of the xml bytes. `seed_recall({"a", "b"}, {"a", "c"}) == 0.5`. `seed_recall(set(), set())` raises `ValueError`. Recording the same xml on `2026-10-31` and `2026-11-01` appends two `search` rows. `import_hits` of a one-hit result onto a fresh Zotero db inserts one item tagged `found-in:pubmed`. Importing that hit again returns 0. `PROTOCOLS["lead-precursors"]` is a non-empty tuple of query strings. Each hit's database is the tag stem `found-in:pubmed` would use: the test asserts `hit.database == "pubmed"`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_search_pubmed.py -q` +Expected: FAIL, package missing. + +- [ ] **Step 3: Implement the parser and the recall function** + +Parse with the stdlib `xml.etree.ElementTree`. Read article title, year, DOI, PMID, and the first author last name. `record` hashes the caller-supplied row fields and calls `append`. Protocols hold the strand's PubMed query strings as data. They are the starting strings, not a claim of live recall. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_search_pubmed.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/search src/cli/prisma.py tests/prisma/test_search_pubmed.py +git commit -m "feat(prisma): PubMed search log and seed recall" +``` + +### Task 8: The other searchers + +**Files:** +- Create: `src/prisma/search/europepmc.py`, `openalex.py`, `crossref.py`, `semanticscholar.py`, `arxiv.py` +- Test: `tests/prisma/test_search_others.py` + +**Interfaces:** +- Consumes: `SearchResult`, `SearchHit`, `SearchLog` from Task 7 +- Produces: `parse_(payload: str, *, query: str, searched_on: str) -> SearchResult` for `europepmc` (XML), `openalex` (JSON), `crossref` (JSON), `semanticscholar` (JSON), `arxiv` (Atom XML). Each sets `database` to that source name. + +- [ ] **Step 1: Write the failing test** + +One parametrized test. Each parser is given a one-record fixture and returns one hit with that record's title, the right `database`, and `hit_count == 1`. An empty payload returns zero hits and does not raise. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_search_others.py -q` +Expected: FAIL, modules missing. + +- [ ] **Step 3: Implement the five parsers** + +Read title, year, DOI, and first author from each format's real field names. Missing fields become empty strings and `year=None`. No HTTP. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_search_others.py tests/prisma/test_search_pubmed.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/search tests/prisma/test_search_others.py +git commit -m "feat(prisma): parsers for Europe PMC, OpenAlex, Crossref, Semantic Scholar, arXiv" +``` + +### Task 9: Calibration gate + +**Files:** +- Create: `src/prisma/calibrate.py` +- Modify: `src/cli/prisma.py` +- Test: `tests/prisma/test_calibrate.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `sensitivity(gold: Sequence[str], pred: Sequence[str]) -> float`, `sized(n: int, stage: str) -> bool`, `cleared(sens: float, n_ta: int, n_ft: int) -> bool`. Positive class is `include` and `uncertain`. `stage` is `title_abstract` (200) or `full_text` (50). + +- [ ] **Step 1: Write the failing test** + +Gold `include, exclude, uncertain` and pred `uncertain, exclude, exclude` has sensitivity 0.5 (one of two positives caught). `sized(200, "title_abstract")` is true, `sized(199, "title_abstract")` is false, `sized(50, "full_text")` is true, `sized(49, "full_text")` is false. `cleared(0.95, 200, 50)` is true. `cleared(0.949, 200, 50)` is false. `cleared(1.0, 199, 50)` is false. Mismatched gold and pred lengths raise `ValueError`. `stack prisma calibrate --help` exits 0. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_calibrate.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the three functions** + +Sensitivity is true positives over gold positives. Zero gold positives raises `ValueError`. `cleared` requires sensitivity ≥ 0.95 and both size checks. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_calibrate.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/calibrate.py src/cli/prisma.py tests/prisma/test_calibrate.py +git commit -m "feat(prisma): screening calibration gate" +``` + +### Task 10: Two passes and the audit sample + +**Files:** +- Create: `src/prisma/audit.py` +- Create: `src/prisma/prompts.py` +- Modify: `src/cli/prisma.py` +- Test: `tests/prisma/test_audit.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `PROMPT_A: str`, `PROMPT_B: str`, `prompt_hash(text: str) -> str`, `disagreements(a: Mapping[str, str], b: Mapping[str, str]) -> tuple[str, ...]`, `agreed_excludes(a, b) -> tuple[str, ...]`, `advances(decision: str) -> bool`, `audit_sample(ids: Sequence[str], *, rate: float = 0.10, seed: int) -> tuple[str, ...]` + +- [ ] **Step 1: Write the failing test** + +`PROMPT_A != PROMPT_B` and their hashes differ. Passes `{i1: include, i2: exclude}` and `{i1: exclude, i2: exclude}` disagree on `i1` only. Agreed excludes are `i2` only. `advances("uncertain")` and `advances("include")` are true. `advances("exclude")` is false. `audit_sample` of 10 ids at rate 0.10 with seed 1 returns exactly one id, and the same seed returns the same id. Rate 0 returns an empty tuple. An id present in only one pass raises `KeyError` from `disagreements`. `stack prisma audit --help` exits 0. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_audit.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the functions** + +`prompt_hash` is sha256 of the utf-8 text. The sample sorts the ids, then uses `random.Random(seed).sample`. The two prompts are the screening instruction plus one framing sentence each. The same model runs both. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_audit.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/audit.py src/prisma/prompts.py src/cli/prisma.py tests/prisma/test_audit.py +git commit -m "feat(prisma): two-pass disagreements and a 10% audit sample" +``` + +### Task 11: PRISMA 2020 flow counts from tags + +**Files:** +- Modify: `src/prisma/flow.py` +- Test: `tests/prisma/test_flow_2020.py` + +**Interfaces:** +- Consumes: Zotero tags written by search (`found-in:`), dedup (`dedup:duplicate`), and fetch misses (`screen:reason:unavailable-fulltext`) +- Produces: extra `FlowCounts` fields `identified_by_source: dict[str, int]`, `duplicates_removed: int`, `reports_sought: int`, `reports_not_retrieved: int`, `other_methods: int`. `mermaid` and `text_summary` print those fields. `report(counts: FlowCounts, *, project: str) -> str` returns markdown whose numbers are taken from the fields. `combined(counts: Sequence[FlowCounts]) -> FlowCounts` sums them. `changed(before: FlowCounts, after: FlowCounts) -> tuple[str, ...]` names the fields whose values differ. The scoping report is `combined` of `lead-history`, `lead-limits`, `lead-scan`, and `lead-external`. The systematic report is `lead-precursors` alone. + +- [ ] **Step 1: Write the failing test** + +Build a db with `zot.schema.create_db`. Tag one item `project:lead-precursors` and `found-in:pubmed`, one `found-in:books` plus `found-in:citation`, one `dedup:duplicate`, one `stage:3-full-text` and `screen:reason:unavailable-fulltext`. `count()` reports `identified_by_source["pubmed"] == 1`, `identified_by_source["arxiv"] == 0` because the renderer asks for a source that has no rows, `duplicates_removed == 1`, `reports_not_retrieved == 1`, `other_methods == 1` (the books/citation item counted once). `text_summary` contains the pubmed count it just read, and contains `arxiv` with 0. `report` contains that same digit and no second conflicting count for pubmed. `combined` of two counts with `duplicates_removed` 1 and 2 is 3. `changed` from a count with `included=0` to `included=1` is `("included",)` and is empty when the counts are equal. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_flow_2020.py -q` +Expected: FAIL, fields missing. + +- [ ] **Step 3: Implement the fields and the report** + +`other_methods` is the set of project items tagged `found-in:citation`, `found-in:books`, or `found-in:expert`. `reports_sought` is items tagged `stage:3-full-text` or `stage:4-included` or `screen:reason:unavailable-fulltext`. `identified_by_source` keys are the `found-in:` stems present, and `text_summary` also prints any stem in `("pubmed", "europepmc", "openalex", "crossref", "semanticscholar", "arxiv")` that is absent, as 0. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_flow_2020.py tests/prisma/test_flow.py -q` +Expected: PASS. If `test_flow.py` asserts the old `FlowCounts` constructor shape, update those call sites in the same step. + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/flow.py tests/prisma/test_flow_2020.py tests/prisma/test_flow.py +git commit -m "feat(prisma): PRISMA 2020 flow counts from tags" +``` + +### Task 12: Zotero export + +**Files:** +- Create: `src/lead/ledger/export.py` +- Modify: `src/cli/lead.py` +- Test: `tests/lead/test_ledger_export.py` + +**Interfaces:** +- Consumes: `open_ledger`, `append`, Zotero `Db` +- Produces: `export_project(zdb: Db, con, project: str, *, extraction_version: str, actor: str) -> dict[str, int]`. Keys of the dict: `source`, `decision`, `extraction`. Idempotency key for a source is the Zotero item key. Idempotency key for an extraction is item key plus `extraction_version`. The `content_hash` includes that key, so a second call inserts nothing. + +- [ ] **Step 1: Write the failing test** + +Create a Zotero db, insert one item with key `ABCD1234`, tags `project:lead-precursors`, `found-in:pubmed`, `stage:4-included`, and a child note whose markdown contains `target_id: Q.ACR`. `export_project` twice returns inserted counts `{source: 1, ...}` then `{source: 0, decision: 0, extraction: 0}`. The `source` row's hash changes if `extraction_version` changes, and a second version inserts one more extraction row. `stack lead ledger export` is registered. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/lead/test_ledger_export.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the export** + +Read items for `project:` excluding `module:prisma` anchors, the same exclusion `flow.count` uses. Write one `source` row per item and one `decision` row per `screen:` decision tag. Write an `extraction` row only for `stage:4-included` items. Do not write claims here. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/lead/test_ledger_export.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/lead/ledger/export.py src/cli/lead.py tests/lead/test_ledger_export.py +git commit -m "feat(lead): export Zotero review rows into the ledger" +``` + +### Task 13: Claims, GRADE start, pool gate, precursor catalog + +**Files:** +- Create: `src/lead/ledger/claims.py` +- Test: `tests/lead/test_ledger_claims.py` + +**Interfaces:** +- Consumes: `append`, `by_id` +- Produces: `claims_from_extraction(extraction: Mapping[str, str], *, project: str, actor: str) -> tuple[Mapping[str, object], ...]`, `grade(design: str, downgrades: Sequence[str]) -> str`, `may_pool(measures: Sequence[str]) -> bool`, `precursor_catalog(con) -> list[Mapping[str, object]]` + +- [ ] **Step 1: Write the failing test** + +A precursors extraction with `outcome_target_id=Q.ACR` becomes one claim of kind `precursor`, tier `literature`, grain as given. A limits extraction becomes kind `limit`. `grade("cohort", ()) == "high"`. `grade("cohort", ("risk of bias",)) == "moderate"`. `grade("cross-sectional", ()) == "low"`. Four downgrades land on `very low` and stop there. `may_pool(["OR", "OR", "OR"])` is true. `may_pool(["OR", "OR"])` is false. `may_pool(["OR", "OR", "RR"])` is false. Exporting the same extraction twice inserts one claim. The catalog row for `Q.ACR` includes that claim's `lead_time` and `certainty`. A claim that cites `target_id` `nope` raises `KeyError`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/lead/test_ledger_claims.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the mapping** + +Cohort, case-control, and quasi-experimental start at `high`. Anything else starts at `low`. Each downgrade steps `high → moderate → low → very low`. `may_pool` is true only when there are at least three measures and they are all the same string. The catalog groups claims by `target_id` and returns lead time, grain, certainty, and the limit claim id for that target when one exists. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/lead/test_ledger_claims.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/lead/ledger/claims.py tests/lead/test_ledger_claims.py +git commit -m "feat(lead): claims, GRADE start, and the precursor catalog" +``` + +### Task 14: Grey-literature parsers + +**Files:** +- Create: `src/prisma/harvest.py` +- Test: `tests/prisma/test_harvest.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `Harvested(title: str, url: str, date: str, source: str)`, `PARSERS: dict[str, Callable[[str], tuple[Harvested, ...]]]`. Keys: `cms`, `medpac`, `macpac`, `aspe`, `gao`, `cbo`, `crs`, `kff`. + +- [ ] **Step 1: Write the failing test** + +Parametrize over the eight keys. Each fixture is an HTML fragment with one `Report title` and a ``. The parser returns one `Harvested` with that title, url, date, and `source` equal to the key. An empty string returns an empty tuple. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_harvest.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the parsers** + +Use `html.parser` from the stdlib. One parser function per source, registered in `PARSERS`. No HTTP. The fixtures are the contract. A source whose real page differs gets a new fixture later, not a live fetch in this task. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_harvest.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/harvest.py tests/prisma/test_harvest.py +git commit -m "feat(prisma): grey-literature harvest parsers" +``` + +### Task 15: Congress.gov client + +**Files:** +- Create: `src/prisma/congress.py` +- Test: `tests/prisma/test_congress.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `Bill(congress: int, bill_type: str, number: int, title: str, latest_action: str)`, `parse_bills(payload: Mapping[str, object]) -> tuple[Bill, ...]`, `fetch_bills(congress: int, *, key: str, transport: Callable[[str], Mapping[str, object]]) -> tuple[Bill, ...]` + +- [ ] **Step 1: Write the failing test** + +A payload `{"bills": [{"congress": 119, "type": "hr", "number": 1, "title": "A bill", "latestAction": {"text": "Introduced"}}]}` parses to one `Bill`. `fetch_bills` calls the transport with a URL that contains `119` and `api_key=test-key`, and returns that bill. `fetch_bills(..., key="")` raises `ValueError` and does not call the transport. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_congress.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement the parser and the client** + +The URL is `https://api.congress.gov/v3/bill/119?api_key=&format=json`. The transport receives that URL and returns the payload. No default HTTP implementation is called from the test. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_congress.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/congress.py tests/prisma/test_congress.py +git commit -m "feat(prisma): Congress.gov bill parser" +``` + +### Task 16: BLS series and the econ fetchers + +**Files:** +- Create: `src/bls/ppi.py`, `src/bls/jolts.py`, `src/bls/qcew.py` +- Create: `src/econ/__init__.py`, `src/econ/fred.py`, `src/econ/irs_soi.py` +- Test: `tests/bls/test_ppi_jolts_qcew.py`, `tests/econ/test_fred_irs.py` + +**Interfaces:** +- Consumes: nothing +- Produces: `parse_ppi(csv_text: str) -> tuple[SeriesRow, ...]`, and the same signature for `parse_jolts` and `parse_qcew`. `SeriesRow(series_id: str, year: int, period: str, value: float | None)`. `parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]` for FRED. `parse_county_income(csv_text: str) -> tuple[CountyIncome, ...]` with `CountyIncome(fips: str, year: int, agi: int | None)`. + +- [ ] **Step 1: Write the failing test** + +A one-row PPI CSV `series_id,year,period,value` of `WPS512,2024,M01,100.5` parses to one row with `value == 100.5`. A blank value parses to `None`, not 0. JOLTS and QCEW use the same columns and the same assertion. A FRED payload `{"observations": [{"date": "2024-01-01", "value": "."}]}` yields `value is None`. IRS CSV `fips,year,agi` of `01001,2022,40000` yields that county. A missing column raises `ValueError`. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/bls/test_ppi_jolts_qcew.py tests/econ/test_fred_irs.py -q` +Expected: FAIL, modules missing. + +- [ ] **Step 3: Implement the parsers** + +CSV via the stdlib `csv` module. Do not download. A blank or `.` value is `None`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/bls/test_ppi_jolts_qcew.py tests/econ/test_fred_irs.py tests/bls/test_oews.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/bls/ppi.py src/bls/jolts.py src/bls/qcew.py src/econ tests/bls/test_ppi_jolts_qcew.py tests/econ +git commit -m "feat: parse PPI, JOLTS, QCEW, FRED, and IRS SOI extracts" +``` + +### Task 17: Nightly and triggered search due dates + +**Files:** +- Create: `src/prisma/search/due.py` +- Modify: `src/cli/prisma.py` +- Test: `tests/prisma/test_search_due.py` + +**Interfaces:** +- Consumes: `lead.ledger.store.open_ledger`, the `search`, `finding`, `claim`, `exogenous`, and `source` tables +- Produces: `due(con, *, project: str, now: str) -> tuple[str, ...]`. Reasons, in this order when several apply: `nightly`, `contradiction`, `exogenous`, `cms`. `now` is `YYYY-MM-DD`. + +- [ ] **Step 1: Write the failing test** + +No `search` row → `("nightly",)`. A `search` row `searched_on` equal to `now` → empty. A row dated the day before `now` → `("nightly",)`. A `finding` with `relation="contradicts"` whose claim `certainty` is `moderate` or `high` adds `contradiction`. Certainty `low` does not. An `exogenous` row whose `target_id` is `Q.ACR` adds `exogenous` when the project has a claim on `Q.ACR`. A `source` row with `database="cms"` and `created_at` after the latest search adds `cms`. `stack prisma search due --help` exits 0. The command exits 0 when nothing is due. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `uv run pytest tests/prisma/test_search_due.py -q` +Expected: FAIL, module missing. + +- [ ] **Step 3: Implement `due`** + +Compare dates as strings in `YYYY-MM-DD`. Nightly means the latest `searched_on` for that project is missing or earlier than `now`. Do not call a searcher from this function. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `uv run pytest tests/prisma/test_search_due.py tests/lead/test_ledger_store.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/prisma/search/due.py src/cli/prisma.py tests/prisma/test_search_due.py +git commit -m "feat(prisma): nightly and triggered search due check" +``` + +## Spec coverage + +Part 1 sections 1.1–1.12 are Tasks 1–17. The target registry and the ledger are Tasks 1, 2, 12, and 13. Appraisal uses the extraction fields in Task 5 and `audit_sample` from Task 10. Parts 2–4, OSF registration, and live database piloting of the query strings are not tasks. The query strings ship as data in `PROTOCOLS` so a later pilot can replace them without changing the parser. + +## Self-review + +- Coverage: every Part 1 behavior in the spec has a task above. Parts 2–4 do not. +- Steps name the function, the file, and the assertion. Parser bodies are not written out. +- Names used later (`append`, `by_id`, `SearchResult`, `fallback_allowed`, `audit_sample`, `FlowCounts`) match the task that produces them. +- The five review-focus cases each have a test: Task 3, Task 7, Task 6, Task 13, Task 11. +- The plan is longer than the spec because it is seventeen tasks, and shorter than the code those tasks will become. diff --git a/docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md b/docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md new file mode 100644 index 0000000..309172e --- /dev/null +++ b/docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md @@ -0,0 +1,484 @@ +# LEAD evidence and modeling chassis — design + +Status: approved 2026-10-31, with the suspecting, Sci-Hub, and nightly-search decisions recorded below. Tracker: none yet. + +This is the overall design for a chassis that turns a living PRISMA review of the LEAD model into models that an agent keeps proposing, running, testing and learning from. There are four parts. Part 1 (the review and the evidence ledger) is designed in full here. Parts 2–4 are outlined only so far as the contracts between the parts need, and each gets its own detailed design before it is built. + +## Why + +CMS's Long-term Enhanced ACO Design (LEAD) model starts in PY 2027. An ACO's money and its quality score are set entirely by published math: + +- **Risk adjustment.** A recalibrated prospective model for Aged & Disabled beneficiaries who don't meet High Needs criteria, and a recalibrated concurrent model for those who do. The concurrent model uses the "Whole Person" approach and Incident High Needs flags. There is a separate ESRD model, plus normalization, coding-intensity handling, mitigation for significant, anomalous and highly suspicious (SAHS) billing and for skin substitutes, and a 4% risk-score growth cap on the High Needs category. +- **Benchmark and settlement.** As set out in the Alignment and Financial Methodology Paper. +- **Quality.** ACR, UAMCC, DAH, TFU and CAHPS. CBP and GSD are optional through dQM/eCQM in PY 2027–28 and required from PY 2029. ACR, UAMCC and DAH are stratified by High Needs status. +- **High Needs eligibility.** Seven criteria, checked quarterly. Once a beneficiary qualifies, the status applies retroactively to the start of their alignment that year and lasts for the rest of their alignment. + +Anything an ACO does matters only to the extent this math can see it. We want to know, with evidence that can be proven: + +1. Which intermediate ("middle layer") measures move before the LEAD quality measures do. +2. Which signals come before a beneficiary meets any one of the seven High Needs criteria. +3. Which public data adds trend and High Needs "suspecting" signal. +4. Which forces outside healthcare shape all of it: labor and prices, legislation and rulemaking, tax and coverage, market structure. + +Then we want a machine that keeps turning that evidence into models, tests them, and learns from what works and what fails. That machine is an agent running on an open model, driven through an MCP server. + +**The methodology is the four corners.** It defines what counts, so the review starts there. But it rests on theory that may or may not be solid, and the math itself sets natural limits. So the review first traces the methodology's history, works out its gaps and limits, and scans the environment around it. Only then does it hunt for precursors, and only inside what the math says can actually move. + +**Public data only.** The chassis is built and proven without PHI, claims or any client data. Someone will later point it at real claims, but it must already be worth using before that happens. + +## What it is + +``` + sources ─────► Part 1: review ─────► evidence ledger ◄─────────────────┐ + literature, 5 strands, claims, each with │ + CMS documents, PRISMA flows a strength tier │ findings flow + books, public │ │ back as evidence + data, Congress… ▼ │ + Part 2: LEAD math kernel + public data │ + (exact rules, limits, loaders) │ + │ │ + ▼ │ + Part 3: modeling loop ─────────────────────┘ + local first → Databricks for what passes the gates + ▲ + Part 4: agent on an open model, via MCP + (reads ledger, proposes, runs, records) +``` + +## Goals + +1. **A defensible five-strand review.** A registered protocol; PRISMA 2020, PRISMA-S and PRISMA-ScR reporting; and flow diagrams whose counts come from the data, never typed by hand. The precursor strand should hold up as a systematic review, and the other strands as a scoping review. +2. **An evidence ledger.** Every claim traces back to its source and extraction, and forward to every feature, model, run and result that used it. Every claim carries a strength tier saying how it is known. +3. **The LEAD math implemented exactly** from public specifications, with each piece checked against a published number or worked example. +4. **A modeling loop** that runs cheaply on the local machine, promotes to Databricks only what passes the gates, and uses Databricks AutoML as the standing baseline. +5. **An agent on an open (FOSS) model**, with the same weights locally and on Databricks. It reads the ledger, proposes features and models, runs them and records the results, all through an MCP server that a person can drive as well. +6. **Value before real claims arrive.** Limits and headroom computed exactly. Population-level findings from real aggregate public data. Full end-to-end runs on synthetic person-level data. + +## Non-goals + +- No PHI, CCLF or client data, and no dependency on the deleted LEAD repository. +- Not a chart-reading or coding assistant. Public data has no clinical notes. +- No person-level conclusions drawn from synthetic data. +- Meta-analysis only where studies are truly comparable. Not the default. +- No product UI. Marimo notebooks and the existing API are enough for inspection. + +## Anti-goals (failure even if it technically works) + +- A finding that can't be traced back to its sources, or a model feature that can't be traced back to a claim. +- Presenting a literature-only or synthetic-only result as if it were confirmed. +- Spending Databricks compute on ideas that local runs, or the failure record, already ruled out. +- An agent that quietly retries something that already failed, or promotes a model without passing the gates. +- A screen that drops records without saying so. The skin-subs run recorded API errors as screening decisions; that must be impossible here. +- **Collapsing suspecting into only coding or only care.** *(User decision.)* A suspect is a hypothesized status transition: the beneficiary moves from one scored status to another (not High Needs to High Needs, an absent HCC to a present one, a quality denominator without the numerator to one with it). Two mechanisms produce that transition, and both are in scope. Code uplift makes a status the person already has visible to the math. Care changes the underlying state. Strand 2 still measures how far each mechanism can move the score, including how demographic reweighting shrinks the coding payoff. The failure is treating the two as the same thing, or forgetting that what is being suspected is the status change. + +## Constraints + +- **Public data and public research only.** +- **Lives inside `stack`.** It reuses existing modules and adds to `stack` where something is missing. It follows the house conventions: `stack ` commands, tests in `tests//`, specs and plans in `docs/superpowers/`, and idempotent, touchless commands. +- **FOSS model.** Open weights under an OSI-approved license, so Llama's community license does not qualify. The same weights must run locally and in Databricks. +- **Local hardware.** `rack` has one RTX 3060 (12 GB), 32 cores and 125 GB RAM. Its Ollama fleet is currently that one host serving `qwen2.5:14b`. In practice that means about 8–14B parameters for local inference (quantized) and about 8B for local QLoRA fine-tuning. +- **Databricks.** The bundle in `databricks.yml` defines catalogs `aco_dev`, `aco_staging` and `aco`, and `stack/.env` has credentials. There is no MLflow, model serving or feature store yet. Which workspace features are available (GPU serving, fine-tuning, AutoML) has not been checked. +- **Licensed reference content must not be published or redistributed.** That covers the AMA CPT and ICD codebooks in the books library, and UMLS, SNOMED and VSAC via `src/nlm`. It may inform the work, but authoritative code lists come from public CMS files. +- **Provability and observability everywhere** (see the cross-cutting section). + +## Sources of truth (in priority order) + +1. **CMS LEAD primary documents** in `lead_corpus/lead_research/cms_primary/`: + - the Risk Adjustment Overview memo (August 2026, v1.0) + - the Risk Adjustment Coefficient Estimates (v1.1) + - the Alignment and Financial Methodology Paper (v1.2), including §2.5 High Needs Eligibility and the Appendix B code lists + - the PY 2027 Quality Measurement Methodology Report (2026-08-19) + + The ACO REACH predecessors sit beside them. +2. **Published CMS software and data:** HCC model software and ICD-10 mappings, measure information forms, and eCQM specs. Most of it is already in `lead_corpus/lead_enrichment/`. +3. **Peer-reviewed and grey literature** found by the review. +4. **The books library** for methods (see the Identification section). + +When sources conflict, CMS primary documents win on *what LEAD does*, and the literature wins on *whether it works*. + +## The spine: target registry and evidence ledger + +### Target registry + +A small versioned table, seeded from the CMS documents, that gives an ID to everything the methodology scores. Every claim, feature, run and limit refers to these IDs. + +| Family | IDs | +|---|---| +| Quality | `Q.ACR`, `Q.UAMCC`, `Q.DAH`, `Q.TFU`, `Q.CAHPS`, `Q.CBP`, `Q.GSD` | +| High Needs (§2.5) | `HN.1` mobility-impairment diagnosis · `HN.2` hospital bed or transfer-equipment DME claim · `HN.3` Kim claims-based frailty index · `HN.4` A&D risk score ≥ 3.0 (V28 prospective or CMMI concurrent V1) · `HN.5` ESRD risk score ≥ 0.35 · `HN.6` A&D score 2.0–3.0 (or ESRD 0.24–0.35) plus ≥ 2 unplanned admissions in 12 months · `HN.7` ≥ 45 covered SNF days in 12 months | +| Risk adjustment | `RA.PROSP` (non-High Needs A&D) · `RA.CONC` (High Needs, with Whole Person and Incident flags) · `RA.ESRD` · `RA.NORM` · `RA.CI` (coding intensity) · `RA.CAP` (4% High Needs growth cap) · `RA.SAHS` (anomalous-billing and skin-substitute mitigation) | +| Benchmark / settlement | `BM.*`, one per component of the Alignment and Financial paper §3 and the settlement tables. Many are already implemented in `src/lead`. | + +### Evidence ledger + +All tables are append-only. Every row records `created_at`, an `actor` (a person, or a model ID plus adapter version) and a content hash. + +| Entity | What it holds | +|---|---| +| `source` | One row per Zotero item: Zotero key, DOI/PMID/URL, file sha256, strand(s), and the database(s) it was found in | +| `search` | The PRISMA-S log: database, interface, date, full query string, filters, hit count, and hash of the export file | +| `decision` | Each screening or eligibility decision: stage, decision, reason code, model, prompt and criteria versions, and whether a human audited it | +| `extraction` | The filled extraction template for a strand, versioned | +| `claim` | The unit the rest of the chassis consumes: kind (`precursor`, `limit`, `lineage`, `context`, `method`), subject (the measurable thing), `target_id`, direction, effect (measure, value, CI), lead time, population and setting, grain (`beneficiary`, `provider`, `ACO`, `area`), design, risk of bias, certainty, source extraction IDs, tier(s) | +| `feature` | An executable definition built from one or more claims: code lists with code-system version, an express/SQL expression, grain, and data source IDs | +| `run` | MLflow run ID, where it ran (`local` or `databricks`), data snapshot hashes, features used, model, metrics, and gate results | +| `finding` | What a run says back about a claim (`confirms`, `contradicts`, `inconclusive`), at what grain, with the numbers. It updates the claim's tier or certainty. | +| `exogenous` | Strand 5 time series and dated events, mapped to target IDs | + +**Strength tiers.** Every claim and finding carries one or more tiers, possibly at different grains: + +- **`rule`** — an exact replication of CMS math that matches a published number. +- **`aggregate`** — shown in real aggregate public data, out of sample, with uncertainty. +- **`literature`** — graded evidence that hasn't been tested locally. +- **`synthetic`** — exercised only on synthetic data. This proves the mechanics and nothing about the world. + +The answer to "what can the chassis tell us, and how convincingly" is simply the current state of this ledger, broken down by tier. + +**Where it lives.** *(My call.)* Zotero stays the source of truth for the review's documents, criteria and decisions, as in the existing PRISMA design. The ledger lives in Delta tables in a Unity Catalog `lead_evidence` schema, with a DuckDB mirror for local runs. Exporting from Zotero to the ledger is deterministic and idempotent, keyed by Zotero item key plus extraction version. The ledger holds public data only and never sits inside a PHI schema. + +## Part 1 — The review (detailed) + +### 1.1 Shape + +- **One protocol, two reports.** Strands 1, 2, 3 and 5 form a scoping review reported under PRISMA-ScR. Strand 4 is a systematic review of prognostic factors reported under PRISMA 2020. Both use PRISMA-S for search reporting. +- **One Zotero PRISMA project per strand**, because each strand has its own criteria: `lead-history`, `lead-limits`, `lead-scan`, `lead-precursors`, `lead-external`. These are added to `STARTERS` in `src/prisma/project.py`. *(My call.)* A record can belong to several strands. Deduplication runs across all of them. +- **Registration.** The protocol is registered on OSF before screening starts. That is an outward-facing step, so it needs the user's go-ahead. PROSPERO is optional for strand 4. + +### 1.2 Questions by strand + +**S1 — History.** For each LEAD component in the target registry, find: +- where it came from +- what assumption it rests on +- what evidence supported it when it was adopted, and what evidence has appeared since +- how it changed through MA, MSSP, Pioneer, NGACO and REACH on the way to LEAD + +Components to cover: +- the CMS-HCC model: Pope 2004, then V24, then V28 +- the CMMI concurrent model: the REACH High Needs track, then the LEAD recalibration +- the ESRD model +- normalization, and coding intensity (Kronick & Welch; Geruso & Layton; the MA statutory minimum) +- growth caps and demographic weighting +- each benchmark component +- each High Needs criterion, including the Kim frailty index +- ACR (from Yale CORE's hospital-wide readmission work), UAMCC, DAH (its technical expert panel), TFU, and CAHPS case-mix adjustment (Zaslavsky 2001) +- hierarchical and shrinkage profiling (Normand & Shahian; COPSS 2012) +- settlement credibility (Mahler & Dean; the relevant ASOPs) + +*Output:* a history map with one row per component: origin, assumptions, evidence, critiques, and what LEAD changed. + +**S2 — Limits and gaps.** For each component, what are the mathematical and empirical limits on: +- (a) predictive accuracy +- (b) how far an ACO's actions can move the scored result +- (c) bias across subgroups +- (d) incentive distortion and gaming + +Limits to look for in the literature or derive from the math: +- individual-level R² for prospective versus concurrent models +- the zero-sum effects of normalization +- how often the growth cap binds +- shrinkage weight as a function of ACO size +- ceiling effects in quality benchmarks +- High Needs as a one-way door, and the effect of applying it retroactively to the whole year +- the memo's statement that High Needs-like spending rarely occurs without meeting the criteria (treat it as a claim to test) +- how much demographic reweighting reduces what coding can achieve + +*Output:* limit claims, each with a statement and a formula or bound. Where possible each also gets a computable expression on public data, which Part 2's limits calculator implements. + +**S3 — Environment scan (2023 onward).** What surrounds LEAD: +- adjacent policy: the V28 phase-in, MSSP changes, the other CMMI models already in `lead_enrichment` +- how the industry identifies High Needs patients and does suspecting +- public data sources, with their refresh cadence and geographic grain +- the current state of methods for risk and prognosis: gradient boosting, GLM and Tweedie models for cost, causal ML, conformal prediction +- GenAI agents for modeling and AutoML +- FOSS model candidates and their licenses +- Databricks capabilities: AutoML, Model Serving, fine-tuning + +*Output:* dated scan items appraised with AACODS, plus the open-model shortlist Part 4 chooses from. + +**S4 — Systematic review of precursors.** Framed as a prognostic factor review (PICOTS): + +- **Population:** Medicare beneficiaries (FFS or MA), or comparable older, disabled or ESRD adults in claims or EHR cohorts, including ACO-attributed populations. +- **Index factors:** anything measurable at time *t*: process measures, utilization patterns, diagnoses, functional status, DME, medications, social risk, area-level factors, and provider or ACO characteristics. +- **Outcomes:** each target ID, or a validated proxy for it, recorded along with how close the proxy is. +- **Timing:** the factor must be measured before the outcome, and the lead time is recorded. +- **Designs:** cohort, case-control, development or validation of a prediction model, and quasi-experimental. Cross-sectional studies without temporal order are excluded with that reason. *(My call.)* +- **Grain:** beneficiary-level and aggregate (ACO, provider, area) analyses are both eligible, and the grain is recorded. + +Sub-questions: +- **4a.** Precursors of quality-measure performance. +- **4b.** Precursors of meeting each High Needs criterion. +- **4c.** Area-level public-data signals for either. + +**S5 — Forces outside healthcare.** How do outside forces shape LEAD's inputs and scores? + +- labor markets and prices (BLS CPI and PPI, JOLTS, QCEW, OEWS, CES) +- legislation and rulemaking: bills, reconciliation, sequestration, the PFS conversion factor, site-neutral payment +- tax and coverage policy: Medicaid eligibility and work requirements feeding dual status, ACA subsidies, IRS county income +- market structure: Medicare Advantage penetration changing who is left in FFS, consolidation, private-equity ownership. The CMS ownership and change-of-ownership files are already in `lead_enrichment`. + +*Output:* the `exogenous` registry (time series plus dated events, mapped to target IDs, each with the citation for its mechanism), and a scoping synthesis of the mechanisms. + +### 1.3 Identification + +**Already in place (reuse):** +- **Zotero "LEAD corpus" collections** (`src/bib/sync.py`, `dev/scripts/import_lead_corpus.py`): + - about 80 staged sources with sha256 hashes (`lead_research/sources.csv`) + - measure specs and dataset documentation + - "Cited works", the backward-citation snowball: 3,550 resolved citation rows in `data/lead_snowball/citations.csv`. In the PRISMA flow these count as citation searching. +- **The books library** (`/srv/pool/media/books`, managed by LazyLibrarian). The "ML library → Books" Zotero collection and `dev/scripts/add_ama_coding_to_zotero.py` already mirror part of it. The books serve two roles: + - the **methods spine** for strands 2–3 and Parts 3–4: Hastie/Tibshirani/Friedman, Murphy, Hernán & Robins, Chernozhukov et al., Wager, Wüthrich & Merz, Goldburd et al. (CAS GLM monograph), Angelopoulos & Bates (conformal), Barocas/Hardt/Narayanan, Gelman, Efron & Hastie, and the Databricks, MLOps and LLMOps titles + - **coding references** (ICD-10-CM codebooks across years, CPT, HCPCS), used to understand how code sets change from year to year. Never as the authoritative code list. + + In the PRISMA flow, books count as "records identified from other methods". Their cited works are chased only for the strand 2 and 3 method questions. *(My call.)* +- **Federal Register, regulations.gov, eCFR, IOM and OIG crawlers** in `src/bib`. +- **Ad-hoc PubMed scripts** (`dev/scripts/search_pubmed_*.py`, `snowball_citations.py`). + +**New in Part 1:** +- **`src/prisma/search/`**: searcher modules behind one interface (query → records plus a `search` log row): + - PubMed (promoting the scripts) + - Europe PMC (which also covers medRxiv) + - OpenAlex (also used for backward and forward citation chasing) + - Crossref + - Semantic Scholar + - arXiv (for the strand 3 methods and agent literature) + + Every hit is tagged `found-in:` in Zotero. +- **Grey-literature harvesters in `src/bib`** for CMS/CMMI evaluation reports, MedPAC, MACPAC, ASPE, GAO, CBO cost estimates, CRS reports and KFF. They use `bib.spider` where they can. +- **Congress.gov API client** (bills, actions, CRS reports). It needs a free api.data.gov key. +- **Strand 5 data series:** extend `src/bls` (PPI, JOLTS, QCEW alongside the existing OEWS and CES, plus the CPI-medical series already in `lead_enrichment`), and add FRED, IRS SOI county and ZIP income, and the CMS MA state/county penetration files. + +**Search strategy.** Each strand's search strings live in the protocol. They are piloted with a *seed recall test*: run the search and report what share of the known seed items indexed in that database it retrieves. Each execution writes a `search` row. A re-run creates a new row, never an overwrite. + +**Known limitation to report.** There is no access to Embase, Scopus, Web of Science or CINAHL. OpenAlex, Europe PMC and citation chasing partly make up for it, and the PRISMA-S report says so. + +### 1.4 Deduplication + +New `src/prisma/dedup.py`. It matches on DOI, then PMID, then normalized title + year + first author. Borderline fuzzy pairs go to a person. Counts per source, before and after deduplication, feed the flow diagram. + +### 1.5 Screening + +- **Criteria** are Zotero anchor notes per strand, the existing pattern. +- **The screening model** is the local FOSS model, through the existing OpenAI-compatible and Ollama fleet path, so no API credits are needed. Anthropic stays available as a configuration option. Each decision records the model, adapter, prompt and criteria versions. +- **An error is never a decision.** If the model or API fails, the item stays unscreened and the run fails loudly, exiting non-zero with a count. This fixes the skin-subs failure. Today `prisma.screen.run` and `run_fleet` already leave a failed item untagged and continue, and `stack prisma screen` prints the error count and exits 0. The change is that a non-zero error count fails the command. +- **Calibration before production:** + - People label a stratified calibration set. *(My call on size: 200 records per strand at title/abstract and 50 at full text.)* + - The model has to reach **sensitivity ≥ 0.95** on include-or-uncertain at the title/abstract stage before the full set runs. *(My call on the threshold.)* + - Sensitivity, specificity and Cohen's κ are reported. +- **Production:** + - Two independent model passes, using different prompt framings or two different models. + - Disagreements go to a person. + - A random **10% of agreed excludes** is audited by a person. *(My call.)* + - "Uncertain" always moves forward. +- **Reporting** follows the RAISE guidance on using AI in evidence synthesis. The builder confirms which version is current. + +### 1.6 Full-text retrieval + +The existing cascade in `src/prisma/fetch.py` and `src/zot/fetch/` (Unpaywall → PMC/Europe PMC → Semantic Scholar open access → the Sci-Hub fallback mirrors). **The Sci-Hub fallback is on for the LEAD projects.** *(User decision.)* A per-project switch still exists, so other projects can turn it off. Anything that can't be retrieved is reported in PRISMA 2020's "reports not retrieved" box. + +### 1.7 Extraction templates (one per strand) + +- **S1 history:** target ID, origin (citation, year, organization), assumptions, supporting evidence, critiques, changes on the way to LEAD, status in LEAD (adopted / modified / dropped), page references. +- **S2 limit:** target ID, type (accuracy / movability / fairness / incentive), statement, formula or bound, any empirical value and its population, a computable expression on public data if one exists, page references. +- **S3 scan:** type (policy / method / data / model / industry), date, summary, target IDs affected, AACODS scores, what it means for the chassis. +- **S4 prognostic** (CHARMS fields plus risk of bias): + - design, data source, population, n, setting, country, years + - factor definition (with codes when the study gives them) + - outcome definition mapped to a target ID, and how close the proxy is + - lead time or horizon, and grain + - effect measure, value, CI, and what it was adjusted for + - C-statistic and calibration for prediction models + - risk of bias: QUIPS for prognostic factors, PROBAST (or PROBAST+AI) for prediction models + - flags for whether the factor is observable in public data and whether it is observable in claims +- **S5 external:** event or series, dates or frequency, geography, source, mechanism linking it to target IDs, evidence on magnitude. + +Each template becomes a forced-tool schema in `src/prisma/tools.py`, the existing pattern. + +### 1.8 Appraisal + +- QUIPS for prognostic-factor studies, PROBAST for prediction models, AACODS for grey literature. +- The scoping strands record source type and AACODS, with no formal risk-of-bias assessment, in line with PRISMA-ScR practice. +- The model drafts each appraisal, and people confirm a sample, using the same audit approach as screening. + +### 1.9 Synthesis into claims + +- Every included extraction yields one or more claims. +- Claims are grouped by subject × target. +- **Certainty** is graded with GRADE for prognostic factors: + - cohort designs start high + - they are downgraded for risk of bias, inconsistency, indirectness (proxy outcomes, non-Medicare populations), imprecision, and publication bias +- **Pooling** (random effects) is done only with three or more comparable studies that use compatible effect measures. Otherwise the synthesis is narrative. +- **The headline output is the *precursor catalog*.** For each target, it ranks claims, giving each one's lead time, grain, certainty, observability, and the strand 2 limit that bounds its payoff. + +### 1.10 Reporting and flow diagrams + +- **Extend `src/prisma/flow.py` to the full PRISMA 2020 layout:** + - counts identified per source + - duplicates removed + - records screened and excluded + - reports sought and not retrieved + - reports assessed, with exclusions by reason + - records included + - a separate "other methods" column for citation searching, books and expert-identified items +- There is one diagram for the scoping review and one for the systematic review. +- Report drafts (markdown) and the PRISMA checklists are generated from ledger counts, so the numbers in the text can never drift from the data. + +### 1.11 Living updates + +**Scheduled re-runs.** Searches re-run nightly. *(User decision.)* Triggered re-runs still fire on top of that schedule. + +**Triggered re-runs.** A search also re-runs when: +- (a) a Part 3 finding contradicts a claim of moderate or higher certainty +- (b) a strand 5 event touches a target +- (c) CMS issues new LEAD guidance + +Each update produces a dated version of the flow diagrams, showing what changed. + +### 1.12 What changes in `stack` for Part 1 + +- **`src/prisma`:** + - new `search/` and `dedup.py` + - the five `lead-*` STARTERS and their per-strand tools + - error-is-not-a-decision handling + - two-pass screening with audit sampling + - a `calibrate` command + - the extended `flow.py` + - a per-project switch for the Sci-Hub fallback; the five `lead-*` projects leave it on +- **`src/bib`:** grey-literature harvesters and the Congress.gov client. +- **`src/bls`:** the extra series. Plus new FRED and IRS SOI fetchers. *(My call: put these in `src/bls` or a new `src/econ`, whichever reads better to the builder.)* +- **New `src/lead/ledger/`:** the ledger schema, the target registry seed, and the Zotero → ledger export. +- **CLI:** + - `stack prisma search | dedup | calibrate | audit` + - `stack lead ledger export | status` + - `stack lead targets` + +## Part 2 — LEAD math kernel and public data (outline) + +**Implemented exactly from public specs.** Each piece is tested against a published number or worked example, following the pattern in `tests/lead/paper.py`. Each one that passes earns the `rule` tier. + +- **V28 prospective scoring**, from the CMS model software and ICD-10 mappings in `lead_enrichment`. +- **The LEAD recalibrated models**, from the coefficient estimates: + - the prospective model for non-High Needs beneficiaries + - the concurrent model for High Needs beneficiaries, with Whole Person and Incident flags +- **The ESRD model, normalization, and the 4% High Needs cap.** +- **The seven High Needs criteria**, using the Appendix B code lists, quarterly rolling lookbacks, retroactive whole-year assignment, and the rule that the status never goes away. +- **The Kim claims-based frailty index.** +- **DAH and TFU.** `src/aco/express/cms_quality_measures.py` already covers UAMCC and ACR; reconcile them to the LEAD spec. +- **CAHPS** from public results only. + +**Limits calculator.** Turns strand 2 limit claims into computations on public data. For example: +- the headroom from catching High Needs earlier, given retroactive assignment and the cap +- how sensitive the benchmark is to normalization +- shrinkage weight by ACO size + +**Public data loader.** Driven by `lead_enrichment/manifest.csv`, which records URL, sha256 and release for each file. The manifest has 651 rows and 140 `source_id`s (several releases per source). The loader loads every row into Unity Catalog and a local DuckDB mirror, joined on a shared geography spine (county FIPS, ZCTA, HSA/HRR, CBSA) and a time spine (year and quarter). Nothing loads these files today. + +**Synthetic person-level layer:** +- the BCDA sandbox (`src/bcda`, already v3) +- the Tuva synthetic seeds +- candidates to evaluate: the CMS synthetic Medicare claims files on data.cms.gov, DE-SynPUF, and Synthea + +Everything in this layer is tagged `synthetic`. + +## Part 3 — Modeling loop (outline) + +- **One record:** MLflow tracking in Databricks, logged from local runs and Databricks jobs alike. There is one experiment per target ID. +- **Local tier** (`rack`): polars/DuckDB with LightGBM, XGBoost and GLM/Tweedie models. This is where most candidates get tried, cheaply. +- **Databricks tier:** promoted candidates, runs at scale, and AutoML baseline runs. +- **Promotion gates.** *(My defaults.)* A candidate is promoted only if: + - it beats both the AutoML baseline and a naive or persistence baseline out of time (train through year Y, test on Y+1) + - calibration is within tolerance + - conformal intervals hold their coverage + - subgroup checks pass (dual status, SVI quartile, rurality) + - every feature has a claim ID +- **Failure record.** Every failed or discarded run becomes a `run` row plus a `finding` row. The agent has to check this record before proposing anything. +- **Drift monitoring** annotates each alarm with the strand 5 events in that window, so "the world changed" can be told apart from "the model broke". +- **Reporting** follows TRIPOD+AI. Each method choice cites its source in the methods spine. + +## Part 4 — Agent, open model, MCP (outline) + +- **A new MCP server in `stack`** (Python MCP SDK). Draft tools: + - `ledger.search` / `ledger.get_claim` + - `targets.list`, `limits.compute` + - `features.propose` / `features.validate` + - `runs.start(local|databricks)` / `runs.get` / `runs.compare` + - `failures.check`, `findings.record`, `review.trigger_search` + + Every call is traced with OpenTelemetry (`src/perf`) and written to the ledger with the actor's identity. +- **Guardrails:** + - promotion only through the gate check + - a daily Databricks spend budget + - the agent writes only to the ledger, MLflow and feature definitions +- **The model:** + - open weights under an OSI license, the same weights in Ollama/vLLM locally and in Databricks Model Serving + - the candidate shortlist comes from strand 3 + - the pick is made with a task benchmark built from ledger work: extraction accuracy against human-audited extractions, feature definitions checked against the Part 2 kernel, and quality of experiment proposals +- **Fine-tuning:** + - LoRA adapters trained on ledger traces: human-confirmed decisions, accepted features, and runs that passed and runs that failed (as negative examples) + - adapters are versioned, and the adapter version is stamped on everything the model produces +- **The same model also does Part 1's screening and extraction.** + +## Provability and observability (cross-cutting) + +- **Everything is hashed:** + - data snapshots by manifest sha256 + - code by git SHA + - models by weights hash plus adapter version + - prompts and criteria by content hash +- **Every chain can be walked in both directions:** + - claim → extractions → sources + - run → features → claims → sources + - finding → the claims it updated +- **Telemetry:** OpenTelemetry via `src/perf`, MLflow, and Unity Catalog lineage. Extend `src/aco/lake/lineage.py` and the `/lineage` API to cover the ledger tables. +- **`stack lead ledger status`** gives, for each target: claims by tier, gaps, last run, and open contradictions. + +## Build order + +1. **Part 1 plus the target registry and ledger.** Detailed above. The next step after approval is a plan. +2. **Part 2: kernel and loaders.** The manifest loader and the target registry can start alongside Part 1. +3. **Part 3: modeling loop.** +4. **Part 4: agent, MCP and model.** The model shortlist comes from strand 3. + +Parts 2–4 each get their own detailed design and plan before they're built. + +## Decided by the user + +- The review powers an MLOps pipeline. It needs provability and observability, and should hold up as a systematic review where possible. +- The end product is an MCP-driven chassis that generates and runs models continuously. +- Public data and research only, standalone from the LEAD repo. It must already be worth using on public data before it is pointed at real claims. +- The methodology is the four corners. Its history, a gap analysis and an environment scan inform everything else. +- Look outside the sector: market forces, BLS, Congressional action, tax changes. +- It lives inside `stack`, reuses what's there, and builds out `stack` where needed. +- The AI is the agent that reads the evidence, proposes models and runs them. +- A FOSS, fine-tunable model that runs both locally and on Databricks and fits with AutoML. Run local-first so Databricks time isn't spent on what already works, or on what's known not to. +- Four parts, one overall design, with the review first. +- **Suspecting is both code uplift and a care hypothesis.** What is suspected is a status transition. Coding makes an existing status visible to the math. Care changes the state. Both count. +- **Sci-Hub fallback is on** for the LEAD projects. Other projects can turn it off. +- **Searches re-run nightly**, plus the triggered re-runs. +- The remaining calls in the next section were confirmed on 2026-10-31. + +## Calls confirmed 2026-10-31 + +- **Where the ledger lives:** Zotero is the source of truth for review artifacts, and the ledger lives in Delta tables in `lead_evidence` with a DuckDB mirror. +- **Five Zotero PRISMA projects**, one per strand. +- **Suspecting is a status transition by either mechanism** — code uplift or care. See the anti-goal. +- **Screening:** a calibration set of 200 records per strand at title/abstract and 50 at full text; sensitivity ≥ 0.95; two passes; a 10% audit of agreed excludes. +- **Sci-Hub fallback on** for the LEAD projects, with a per-project switch. +- **S4 excludes cross-sectional studies** that have no temporal order. +- **Book bibliographies are chased** only for the strand 2 and 3 method questions. +- **Nightly search re-runs**, plus triggered ones. +- **The promotion-gate set** in Part 3. + +## Left to the builder + +- Module and file names inside the areas named above. Whether the new economic fetchers go in `src/bls` or a new module. +- The exact search strings, piloted with the seed recall test and recorded in the protocol. +- The Delta and DuckDB schema details, beyond the entities and fields listed above. +- The catalog that `lead_evidence` sits in, per bundle target. +- How to run the two independent screening passes (different prompts or different models). +- The order of the new searchers and harvesters inside Part 1. + +## Open questions + +Builder check on 2026-10-31 found nothing that changes the plan and is not already under "Calls confirmed" or "Left to the builder." The three calls the user changed that day (suspecting, Sci-Hub, nightly searches) are recorded above. + +Checked against the repo and held: `STARTERS` in `src/prisma/project.py` (today only `skin-subs` and `palliative-rfi`); Sci-Hub as the fallback mirror list in `src/prisma/fetch.py` (no per-project switch yet); `src/aco/express/cms_quality_measures.py` covering UAMCC and ACR; `databricks.yml` catalogs `aco_dev`, `aco_staging` and `aco`; 3,550 rows in `data/lead_snowball/citations.csv`; 79 rows in `lead_research/sources.csv`; the named CMS primary PDFs present. The manifest count above is the one figure the draft had wrong. -- 2.49.1 From 07a0f32afec3fba18c8ec65e71abdf1bec70aedc Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:08:18 -0400 Subject: [PATCH 02/30] feat(lead): add the LEAD target registry --- src/cli/__init__.py | 2 + src/cli/lead.py | 16 +++++ src/lead/targets.py | 109 +++++++++++++++++++++++++++++ tests/lead/test_targets.py | 137 +++++++++++++++++++++++++++++++++++++ 4 files changed, 264 insertions(+) create mode 100644 src/cli/lead.py create mode 100644 src/lead/targets.py create mode 100644 tests/lead/test_targets.py diff --git a/src/cli/__init__.py b/src/cli/__init__.py index 3ead03e..746c8f7 100644 --- a/src/cli/__init__.py +++ b/src/cli/__init__.py @@ -22,6 +22,7 @@ from cli.docs import app as docs_app from cli.generate import app as generate_app from cli.health import health from cli.lake import app as lake_app +from cli.lead import app as lead_app from cli.llm import app as llm_app from cli.load import app as load_app from cli.mail import app as mail_app @@ -60,6 +61,7 @@ app.add_typer( help="CMS rulemaking comment text extraction & analysis.", ) app.add_typer(lake_app, name="lake", help="Lakehouse schema and data.") +app.add_typer(lead_app, name="lead", help="LEAD target registry and evidence ledger.") app.add_typer(llm_app, name="llm", help="Local RAG — index, search, tag.") app.add_typer(db_app, name="db", help="DuckDB utilities.") app.add_typer(docs_app, name="docs", help="Documentation generation.") diff --git a/src/cli/lead.py b/src/cli/lead.py new file mode 100644 index 0000000..d3769c7 --- /dev/null +++ b/src/cli/lead.py @@ -0,0 +1,16 @@ +"""stack lead — target registry and evidence ledger.""" + +from __future__ import annotations + +import typer + +from lead.targets import TARGETS + +app = typer.Typer(no_args_is_help=True) + + +@app.command(name="targets") +def targets() -> None: + """Print every target ID, family, and name.""" + for target in TARGETS: + typer.echo(f"{target.id}\t{target.family}\t{target.name}") diff --git a/src/lead/targets.py b/src/lead/targets.py new file mode 100644 index 0000000..7b90dcb --- /dev/null +++ b/src/lead/targets.py @@ -0,0 +1,109 @@ +"""IDs for everything the LEAD methodology scores. + +Claims, features, runs, and limits refer to these IDs. The set is the +one in the evidence-chassis spec, including the benchmark components +that already have a function in :mod:`lead.benchmark` or +:mod:`lead.settlement`. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class Target: + """One scored component.""" + + id: str + family: str + name: str + + +def _row(family: str, rows: tuple[tuple[str, str], ...]) -> tuple[Target, ...]: + return tuple(Target(id=i, family=family, name=n) for i, n in rows) + + +TARGETS: tuple[Target, ...] = ( + *_row( + "quality", + ( + ("Q.ACR", "ACR"), + ("Q.UAMCC", "UAMCC"), + ("Q.DAH", "DAH"), + ("Q.TFU", "TFU"), + ("Q.CAHPS", "CAHPS"), + ("Q.CBP", "CBP"), + ("Q.GSD", "GSD"), + ), + ), + *_row( + "high_needs", + ( + ("HN.1", "mobility-impairment diagnosis"), + ("HN.2", "hospital bed or transfer-equipment DME claim"), + ("HN.3", "Kim claims-based frailty index"), + ("HN.4", "A&D risk score ≥ 3.0"), + ("HN.5", "ESRD risk score ≥ 0.35"), + ( + "HN.6", + "A&D 2.0–3.0 or ESRD 0.24–0.35 plus ≥ 2 unplanned admissions", + ), + ("HN.7", "≥ 45 covered SNF days"), + ), + ), + *_row( + "risk", + ( + ("RA.PROSP", "non-High Needs A&D"), + ("RA.CONC", "High Needs concurrent"), + ("RA.ESRD", "ESRD"), + ("RA.NORM", "normalization"), + ("RA.CI", "coding intensity"), + ("RA.CAP", "4% High Needs growth cap"), + ("RA.SAHS", "anomalous-billing and skin-substitute mitigation"), + ), + ), + *_row( + "benchmark", + ( + ("BM.HISTORICAL", "historical_benchmark"), + ("BM.VOLUNTARY_GUARDRAIL", "voluntary_guardrail"), + ("BM.REGIONAL", "regional_ffs_expenditure"), + ("BM.SPENDING_DESIGNATION", "spending_designation"), + ("BM.ADJ_CAP", "benchmark_adjustment_cap"), + ("BM.REGIONAL_EFFICIENCY", "regional_efficiency_adjustment"), + ("BM.PRIOR_SAVINGS", "prior_savings_adjustment"), + ("BM.PY", "performance_year_benchmark_pbpm"), + ("BM.DISCOUNT", "apply_discount"), + ("BM.QUALITY_WITHHOLD", "quality_withhold"), + ("BM.QUALITY_EARN_BACK", "quality_earn_back"), + ("BM.RETENTION", "retention_incentive_adjustment"), + ("BM.GROSS", "gross_savings_losses"), + ("BM.SHARED", "shared_savings_losses"), + ("BM.SEQUESTRATION", "sequestration_amount"), + ), + ), +) + +_BY_ID = {t.id: t for t in TARGETS} + + +def by_id(target_id: str) -> Target: + """Return the target with this ID. + + Raises: + KeyError: the ID is not in the registry. + """ + try: + return _BY_ID[target_id] + except KeyError: + raise KeyError(target_id) from None + + +def families() -> dict[str, tuple[Target, ...]]: + """Targets grouped by family, in registry order.""" + out: dict[str, list[Target]] = {} + for target in TARGETS: + out.setdefault(target.family, []).append(target) + return {family: tuple(rows) for family, rows in out.items()} diff --git a/tests/lead/test_targets.py b/tests/lead/test_targets.py new file mode 100644 index 0000000..268dfba --- /dev/null +++ b/tests/lead/test_targets.py @@ -0,0 +1,137 @@ +"""The LEAD target registry matches the spec's IDs.""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +from cli import app +from lead.targets import TARGETS, by_id + +runner = CliRunner() + +EXPECTED_IDS = { + "Q.ACR", + "Q.UAMCC", + "Q.DAH", + "Q.TFU", + "Q.CAHPS", + "Q.CBP", + "Q.GSD", + "HN.1", + "HN.2", + "HN.3", + "HN.4", + "HN.5", + "HN.6", + "HN.7", + "RA.PROSP", + "RA.CONC", + "RA.ESRD", + "RA.NORM", + "RA.CI", + "RA.CAP", + "RA.SAHS", + "BM.HISTORICAL", + "BM.VOLUNTARY_GUARDRAIL", + "BM.REGIONAL", + "BM.SPENDING_DESIGNATION", + "BM.ADJ_CAP", + "BM.REGIONAL_EFFICIENCY", + "BM.PRIOR_SAVINGS", + "BM.PY", + "BM.DISCOUNT", + "BM.QUALITY_WITHHOLD", + "BM.QUALITY_EARN_BACK", + "BM.RETENTION", + "BM.GROSS", + "BM.SHARED", + "BM.SEQUESTRATION", +} + +NAMES = { + "HN.1": "mobility-impairment diagnosis", + "HN.2": "hospital bed or transfer-equipment DME claim", + "HN.3": "Kim claims-based frailty index", + "HN.4": "A&D risk score ≥ 3.0", + "HN.5": "ESRD risk score ≥ 0.35", + "HN.6": "A&D 2.0–3.0 or ESRD 0.24–0.35 plus ≥ 2 unplanned admissions", + "HN.7": "≥ 45 covered SNF days", + "RA.PROSP": "non-High Needs A&D", + "RA.CONC": "High Needs concurrent", + "RA.CAP": "4% High Needs growth cap", + "RA.SAHS": "anomalous-billing and skin-substitute mitigation", + "BM.HISTORICAL": "historical_benchmark", + "BM.VOLUNTARY_GUARDRAIL": "voluntary_guardrail", + "BM.REGIONAL": "regional_ffs_expenditure", + "BM.SPENDING_DESIGNATION": "spending_designation", + "BM.ADJ_CAP": "benchmark_adjustment_cap", + "BM.REGIONAL_EFFICIENCY": "regional_efficiency_adjustment", + "BM.PRIOR_SAVINGS": "prior_savings_adjustment", + "BM.PY": "performance_year_benchmark_pbpm", + "BM.DISCOUNT": "apply_discount", + "BM.QUALITY_WITHHOLD": "quality_withhold", + "BM.QUALITY_EARN_BACK": "quality_earn_back", + "BM.RETENTION": "retention_incentive_adjustment", + "BM.GROSS": "gross_savings_losses", + "BM.SHARED": "shared_savings_losses", + "BM.SEQUESTRATION": "sequestration_amount", +} + + +def test_registry_matches_the_spec(): + assert {t.id for t in TARGETS} == EXPECTED_IDS + assert [t.id for t in TARGETS] == [ + "Q.ACR", + "Q.UAMCC", + "Q.DAH", + "Q.TFU", + "Q.CAHPS", + "Q.CBP", + "Q.GSD", + "HN.1", + "HN.2", + "HN.3", + "HN.4", + "HN.5", + "HN.6", + "HN.7", + "RA.PROSP", + "RA.CONC", + "RA.ESRD", + "RA.NORM", + "RA.CI", + "RA.CAP", + "RA.SAHS", + "BM.HISTORICAL", + "BM.VOLUNTARY_GUARDRAIL", + "BM.REGIONAL", + "BM.SPENDING_DESIGNATION", + "BM.ADJ_CAP", + "BM.REGIONAL_EFFICIENCY", + "BM.PRIOR_SAVINGS", + "BM.PY", + "BM.DISCOUNT", + "BM.QUALITY_WITHHOLD", + "BM.QUALITY_EARN_BACK", + "BM.RETENTION", + "BM.GROSS", + "BM.SHARED", + "BM.SEQUESTRATION", + ] + assert by_id("HN.3").name.startswith("Kim") + assert by_id("Q.ACR").family == "quality" + for target_id, name in NAMES.items(): + assert by_id(target_id).name == name + + +def test_unknown_target_raises(): + with pytest.raises(KeyError): + by_id("nope") + + +def test_targets_command_lists_the_registry(): + result = runner.invoke(app, ["lead", "targets"]) + assert result.exit_code == 0 + assert "HN.3" in result.stdout + assert "BM.SEQUESTRATION" in result.stdout -- 2.49.1 From 4be049c07b10a16ef7cc948138a126ca2b975989 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:16:56 -0400 Subject: [PATCH 03/30] feat(lead): append-only evidence ledger in DuckDB --- src/cli/lead.py | 17 +++ src/lead/ledger/__init__.py | 12 ++ src/lead/ledger/schema.py | 117 +++++++++++++++++ src/lead/ledger/store.py | 131 +++++++++++++++++++ tests/lead/test_ledger_store.py | 221 ++++++++++++++++++++++++++++++++ 5 files changed, 498 insertions(+) create mode 100644 src/lead/ledger/__init__.py create mode 100644 src/lead/ledger/schema.py create mode 100644 src/lead/ledger/store.py create mode 100644 tests/lead/test_ledger_store.py diff --git a/src/cli/lead.py b/src/cli/lead.py index d3769c7..fb65e11 100644 --- a/src/cli/lead.py +++ b/src/cli/lead.py @@ -4,9 +4,13 @@ from __future__ import annotations import typer +from lead.ledger.store import open_ledger +from lead.ledger.store import status as ledger_status from lead.targets import TARGETS app = typer.Typer(no_args_is_help=True) +ledger_app = typer.Typer(no_args_is_help=True) +app.add_typer(ledger_app, name="ledger", help="Evidence ledger.") @app.command(name="targets") @@ -14,3 +18,16 @@ def targets() -> None: """Print every target ID, family, and name.""" for target in TARGETS: typer.echo(f"{target.id}\t{target.family}\t{target.name}") + + +@ledger_app.command(name="status") +def ledger_status_cmd( + db: str = typer.Option(..., help="Path to the DuckDB ledger."), +) -> None: + """Print one status row per target.""" + con = open_ledger(db) + try: + for row in ledger_status(con): + typer.echo(row.target_id) + finally: + con.close() diff --git a/src/lead/ledger/__init__.py b/src/lead/ledger/__init__.py new file mode 100644 index 0000000..d9ce127 --- /dev/null +++ b/src/lead/ledger/__init__.py @@ -0,0 +1,12 @@ +"""Append-only LEAD evidence ledger.""" + +from lead.ledger.schema import schema_sql +from lead.ledger.store import TargetStatus, append, open_ledger, status + +__all__ = [ + "TargetStatus", + "append", + "open_ledger", + "schema_sql", + "status", +] diff --git a/src/lead/ledger/schema.py b/src/lead/ledger/schema.py new file mode 100644 index 0000000..cb5f2bb --- /dev/null +++ b/src/lead/ledger/schema.py @@ -0,0 +1,117 @@ +"""DDL for the lead_evidence ledger. + +DuckDB applies this in :func:`lead.ledger.store.open_ledger`. The Delta +string is the same tables with no catalog, so the bundle target's catalog +wraps it. +""" + +from __future__ import annotations + +_COMMON = ("id", "created_at", "actor", "content_hash") + +COLUMNS: dict[str, tuple[str, ...]] = { + "source": _COMMON + + ( + "zotero_key", + "doi", + "pmid", + "url", + "database", + "file_sha256", + "strands", + "project", + ), + "search": _COMMON + + ( + "project", + "database", + "interface", + "searched_on", + "query", + "filters", + "hit_count", + "export_sha256", + ), + "decision": _COMMON + + ( + "project", + "zotero_key", + "stage", + "decision", + "reason", + "model", + "prompt_hash", + "criteria_hash", + "human_audited", + ), + "extraction": _COMMON + + ("project", "zotero_key", "extraction_version", "body", "target_id"), + "claim": _COMMON + + ( + "project", + "kind", + "subject", + "target_id", + "direction", + "effect_measure", + "effect_value", + "effect_ci_low", + "effect_ci_high", + "lead_time", + "population", + "setting", + "grain", + "design", + "risk_of_bias", + "certainty", + "extraction_ids", + "tiers", + ), + "feature": _COMMON + + ( + "target_id", + "claim_ids", + "expression", + "grain", + "code_lists", + "data_source_ids", + ), + "run": _COMMON + + ( + "target_id", + "mlflow_run_id", + "where_ran", + "finished_on", + "snapshot_hashes", + "feature_ids", + "model", + "metrics", + "gate_results", + ), + "finding": _COMMON + ("claim_id", "relation", "grain", "numbers"), + "exogenous": _COMMON + + ("project", "target_id", "series_or_event", "event_on", "citation"), +} + +_TYPES = { + "hit_count": "BIGINT", + "effect_value": "DOUBLE", + "effect_ci_low": "DOUBLE", + "effect_ci_high": "DOUBLE", + "human_audited": "BOOLEAN", +} + + +def schema_sql(dialect: str) -> str: + """Return DDL for ``duckdb`` or ``delta``. + + Raises: + ValueError: ``dialect`` is neither of those. + """ + if dialect not in {"duckdb", "delta"}: + raise ValueError(dialect) + statements = ["CREATE SCHEMA IF NOT EXISTS lead_evidence"] + for table, columns in COLUMNS.items(): + body = ", ".join(f"{name} {_TYPES.get(name, 'VARCHAR')}" for name in columns) + statements.append(f"CREATE TABLE IF NOT EXISTS lead_evidence.{table} ({body})") + return ";\n".join(statements) + ";" diff --git a/src/lead/ledger/store.py b/src/lead/ledger/store.py new file mode 100644 index 0000000..395e35a --- /dev/null +++ b/src/lead/ledger/store.py @@ -0,0 +1,131 @@ +"""Append-only writes and per-target status for the evidence ledger.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone + +import duckdb + +from lead.ledger.schema import COLUMNS, schema_sql +from lead.targets import TARGETS, by_id + +TIERS = ("rule", "aggregate", "literature", "synthetic") + + +@dataclass(frozen=True) +class TargetStatus: + """How much the ledger knows about one target.""" + + target_id: str + claims_by_tier: dict[str, int] + gaps: int + last_run: str | None + open_contradictions: int + + +def open_ledger(path: str) -> duckdb.DuckDBPyConnection: + """Open a DuckDB file and apply the ledger schema.""" + con = duckdb.connect(path) + for statement in schema_sql("duckdb").split(";"): + text = statement.strip() + if text: + con.execute(text) + return con + + +def append( + con: duckdb.DuckDBPyConnection, table: str, row: Mapping[str, object] +) -> bool: + """Insert ``row`` into ``table``. + + The caller supplies ``content_hash``. A hash already stored in that + table inserts nothing and returns ``False``. ``created_at`` is set + when the row omits it. A claim's ``id`` defaults to its content hash + so a finding can point at it. A claim whose ``target_id`` is unknown + raises ``KeyError`` and writes nothing. + """ + if table not in COLUMNS: + raise ValueError(table) + data = dict(row) + if table == "claim": + by_id(str(data["target_id"])) + content_hash = data.get("content_hash") + if not isinstance(content_hash, str) or not content_hash: + raise ValueError("content_hash is required") + unknown = set(data) - set(COLUMNS[table]) + if unknown: + raise ValueError(f"unknown columns: {sorted(unknown)}") + existing = con.execute( + f"SELECT 1 FROM lead_evidence.{table} WHERE content_hash = ?", + [content_hash], + ).fetchone() + if existing: + return False + if not data.get("created_at"): + data["created_at"] = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + if not data.get("id"): + data["id"] = content_hash + columns = [ + name for name in COLUMNS[table] if name in data and data[name] is not None + ] + placeholders = ", ".join("?" for _ in columns) + names = ", ".join(columns) + con.execute( + f"INSERT INTO lead_evidence.{table} ({names}) VALUES ({placeholders})", + [data[name] for name in columns], + ) + return True + + +def status(con: duckdb.DuckDBPyConnection) -> list[TargetStatus]: + """One row per registry target, including targets with no claims.""" + claims = con.execute( + "SELECT id, target_id, tiers FROM lead_evidence.claim" + ).fetchall() + tier_counts: dict[str, dict[str, int]] = {} + claimed: set[str] = set() + for _claim_id, target_id, tiers in claims: + claimed.add(target_id) + bucket = tier_counts.setdefault(target_id, {}) + for name in str(tiers or "").split(","): + tier = name.strip() + if tier in TIERS: + bucket[tier] = bucket.get(tier, 0) + 1 + contradictions = { + target_id: int(count) + for target_id, count in con.execute( + """ + SELECT c.target_id, count(*) + FROM lead_evidence.finding AS f + JOIN lead_evidence.claim AS c ON c.id = f.claim_id + WHERE f.relation = 'contradicts' + GROUP BY c.target_id + """ + ).fetchall() + } + last_runs = { + target_id: finished_on + for target_id, finished_on in con.execute( + """ + SELECT target_id, max(finished_on) + FROM lead_evidence.run + WHERE target_id IS NOT NULL AND finished_on IS NOT NULL + GROUP BY target_id + """ + ).fetchall() + } + rows: list[TargetStatus] = [] + for target in TARGETS: + counts = {tier: tier_counts.get(target.id, {}).get(tier, 0) for tier in TIERS} + rows.append( + TargetStatus( + target_id=target.id, + claims_by_tier=counts, + gaps=0 if target.id in claimed else 1, + last_run=last_runs.get(target.id), + open_contradictions=contradictions.get(target.id, 0), + ) + ) + return rows diff --git a/tests/lead/test_ledger_store.py b/tests/lead/test_ledger_store.py new file mode 100644 index 0000000..3878333 --- /dev/null +++ b/tests/lead/test_ledger_store.py @@ -0,0 +1,221 @@ +"""Append-only evidence ledger: schema, idempotent insert, per-target status.""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +from cli import app +from lead.ledger.schema import schema_sql +from lead.ledger.store import append, open_ledger, status +from lead.targets import TARGETS + +runner = CliRunner() + +TIERS = ("rule", "aggregate", "literature", "synthetic") + + +def claim_row(**overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "kind": "precursor", + "subject": "readmission", + "target_id": "Q.ACR", + "tiers": "literature", + "content_hash": "abc", + "actor": "tester", + } + row.update(overrides) + return row + + +def test_append_is_idempotent_and_status_covers_every_target(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + row = claim_row(target_id="Q.ACR", tiers="literature", content_hash="abc") + assert append(con, "claim", row) is True + assert append(con, "claim", row) is False + assert ( + con.execute("select count(*) from lead_evidence.claim").fetchone()[0] == 1 + ) + created_at = con.execute( + "select created_at from lead_evidence.claim" + ).fetchone()[0] + assert created_at + found = {s.target_id: s for s in status(con)} + assert set(found) == {t.id for t in TARGETS} + assert found["Q.ACR"].claims_by_tier["literature"] == 1 + assert found["Q.ACR"].gaps == 0 + assert found["HN.1"].gaps == 1 + assert found["HN.1"].claims_by_tier == dict.fromkeys(TIERS, 0) + assert found["HN.1"].open_contradictions == 0 + assert found["HN.1"].last_run is None + finally: + con.close() + + +def test_unknown_claim_target_raises(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + with pytest.raises(KeyError): + append(con, "claim", claim_row(target_id="nope", content_hash="missing")) + finally: + con.close() + + +def test_contradicting_finding_bumps_open_contradictions(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + assert append(con, "claim", claim_row(content_hash="abc", id="abc")) + assert append( + con, + "finding", + { + "claim_id": "abc", + "relation": "confirms", + "grain": "beneficiary", + "content_hash": "f-ok", + "actor": "tester", + }, + ) + found = {s.target_id: s for s in status(con)} + assert found["Q.ACR"].open_contradictions == 0 + assert append( + con, + "finding", + { + "claim_id": "abc", + "relation": "contradicts", + "grain": "beneficiary", + "content_hash": "f-no", + "actor": "tester", + }, + ) + found = {s.target_id: s for s in status(con)} + assert found["Q.ACR"].open_contradictions == 1 + assert found["HN.1"].open_contradictions == 0 + finally: + con.close() + + +def test_last_run_is_the_latest_finished_on_for_that_target(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + assert append( + con, + "run", + { + "target_id": "Q.ACR", + "where_ran": "local", + "finished_on": "2026-10-30", + "content_hash": "run-old", + "actor": "tester", + }, + ) + assert append( + con, + "run", + { + "target_id": "Q.ACR", + "where_ran": "databricks", + "finished_on": "2026-10-31", + "content_hash": "run-new", + "actor": "tester", + }, + ) + found = {s.target_id: s for s in status(con)} + assert found["Q.ACR"].last_run == "2026-10-31" + assert found["HN.1"].last_run is None + finally: + con.close() + + +def test_a_tier_list_counts_each_named_tier(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + assert append( + con, "claim", claim_row(content_hash="both", tiers="rule, synthetic") + ) + found = {s.target_id: s for s in status(con)} + assert found["Q.ACR"].claims_by_tier["rule"] == 1 + assert found["Q.ACR"].claims_by_tier["synthetic"] == 1 + assert found["Q.ACR"].claims_by_tier["literature"] == 0 + finally: + con.close() + + +def test_delta_sql_names_the_schema_and_not_a_catalog(): + sql = schema_sql("delta") + assert "lead_evidence" in sql + assert "aco" not in sql + with pytest.raises(ValueError): + schema_sql("nope") + + +def test_search_exogenous_and_extraction_columns_roundtrip(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + assert append( + con, + "search", + { + "project": "lead-precursors", + "database": "pubmed", + "interface": "eutils", + "searched_on": "2026-10-31", + "query": "frailty", + "hit_count": 2, + "export_sha256": "deadbeef", + "content_hash": "s1", + "actor": "tester", + }, + ) + assert append( + con, + "exogenous", + { + "project": "lead-external", + "target_id": "Q.ACR", + "content_hash": "e1", + "actor": "tester", + }, + ) + assert append( + con, + "extraction", + { + "project": "lead-precursors", + "zotero_key": "ABCD1234", + "extraction_version": "v1", + "body": "target_id: Q.ACR", + "content_hash": "x1", + "actor": "tester", + }, + ) + assert append( + con, + "source", + { + "zotero_key": "ABCD1234", + "doi": "10.1/x", + "database": "pubmed", + "content_hash": "src1", + "actor": "tester", + }, + ) + searched = con.execute( + "select project, searched_on from lead_evidence.search" + ).fetchone() + assert searched == ("lead-precursors", "2026-10-31") + exogenous = con.execute( + "select target_id from lead_evidence.exogenous" + ).fetchone() + assert exogenous == ("Q.ACR",) + finally: + con.close() + + +def test_status_command_prints_every_target(tmp_path): + path = tmp_path / "ledger.duckdb" + result = runner.invoke(app, ["lead", "ledger", "status", "--db", str(path)]) + assert result.exit_code == 0 + assert "Q.ACR" in result.stdout -- 2.49.1 From fbc8fa998d68d57829d6cee050cda7a18fe386e6 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:19:14 -0400 Subject: [PATCH 04/30] fix(prisma): fail the screen command when the model errors --- src/cli/prisma.py | 4 + tests/cli/test_prisma_screen_errors.py | 110 +++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 tests/cli/test_prisma_screen_errors.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 310f6f7..0a52f8e 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -162,6 +162,8 @@ def screen( stats = _hold(_go, hold=hold) for k, v in stats.items(): typer.echo(f" {k}: {v}") + if stats.get("errors", 0) > 0: + raise typer.Exit(1) @app.command(name="eligible") @@ -583,6 +585,8 @@ def run_all( zdb, provider, project, storage_dir=storage, limit=limit or None ) typer.echo(f" {s2}") + if s2.get("errors", 0) > 0: + raise typer.Exit(1) if not skip_fetch: typer.echo("==> fetch (PDFs for non-excluded items)") diff --git a/tests/cli/test_prisma_screen_errors.py b/tests/cli/test_prisma_screen_errors.py new file mode 100644 index 0000000..49f9c1c --- /dev/null +++ b/tests/cli/test_prisma_screen_errors.py @@ -0,0 +1,110 @@ +"""A model error during screening is printed and fails the command.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from cli.prisma import app + +runner = CliRunner() + +_ERR = {"screened": 0, "include": 0, "exclude": 0, "uncertain": 0, "errors": 2} +_OK = {"screened": 0, "include": 0, "exclude": 0, "uncertain": 0, "errors": 0} + + +def _mock_db(): + db = MagicMock() + db.__enter__ = MagicMock(return_value=db) + db.__exit__ = MagicMock(return_value=False) + return db + + +def _ready(mc_db, mc_load, mc_provider): + mc_db.return_value = _mock_db() + mc_load.return_value = MagicMock() + mc_provider.return_value = MagicMock() + + +class TestScreenErrors: + @patch("prisma.screen.run", return_value=_ERR) + @patch("prisma.llm.make_provider") + @patch("prisma.project.load") + @patch("zot.db.Db") + def test_screen_exits_1_when_the_model_errors( + self, mc_db, mc_load, mc_provider, mc_run + ): + _ready(mc_db, mc_load, mc_provider) + result = runner.invoke(app, ["screen", "skin-subs", "--no-hold"]) + assert result.exit_code == 1 + assert "errors" in result.output + + @patch("prisma.screen.run", return_value=_OK) + @patch("prisma.llm.make_provider") + @patch("prisma.project.load") + @patch("zot.db.Db") + def test_screen_exits_0_when_there_are_no_errors( + self, mc_db, mc_load, mc_provider, mc_run + ): + _ready(mc_db, mc_load, mc_provider) + result = runner.invoke(app, ["screen", "skin-subs", "--no-hold"]) + assert result.exit_code == 0 + + @patch("prisma.vpn.status", return_value={"status": "down"}) + @patch("prisma.flow.text_summary", return_value="Summary: OK") + @patch("prisma.flow.count") + @patch("prisma.extract.run", return_value={"extracted": 0}) + @patch("prisma.eligibility.run", return_value={"assessed": 0, "errors": 0}) + @patch("prisma.fetch.run", return_value={"missed": 0, "errors": 0}) + @patch("prisma.screen.run", return_value=_ERR) + @patch("prisma.llm.make_provider") + @patch("prisma.project.load") + @patch("zot.db.Db") + def test_run_exits_1_when_the_screen_stage_errors( + self, + mc_db, + mc_load, + mc_provider, + mc_screen, + mc_fetch, + mc_elig, + mc_extract, + mc_count, + mc_summary, + mc_vpn, + ): + _ready(mc_db, mc_load, mc_provider) + mc_count.return_value = MagicMock() + result = runner.invoke(app, ["run", "skin-subs", "--no-hold"]) + assert result.exit_code == 1 + assert "errors" in result.output + mc_fetch.assert_not_called() + + @patch("prisma.vpn.status", return_value={"status": "down"}) + @patch("prisma.flow.text_summary", return_value="Summary: OK") + @patch("prisma.flow.count") + @patch("prisma.extract.run", return_value={"extracted": 0}) + @patch("prisma.eligibility.run", return_value={"assessed": 0, "errors": 0}) + @patch("prisma.fetch.run", return_value={"missed": 0, "errors": 0}) + @patch("prisma.screen.run", return_value=_OK) + @patch("prisma.llm.make_provider") + @patch("prisma.project.load") + @patch("zot.db.Db") + def test_run_exits_0_when_the_screen_stage_is_clean( + self, + mc_db, + mc_load, + mc_provider, + mc_screen, + mc_fetch, + mc_elig, + mc_extract, + mc_count, + mc_summary, + mc_vpn, + ): + _ready(mc_db, mc_load, mc_provider) + mc_count.return_value = MagicMock() + result = runner.invoke(app, ["run", "skin-subs", "--no-hold"]) + assert result.exit_code == 0 -- 2.49.1 From ae8e46f04682e62e814e28b53743a3aaa0f514df Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:21:43 -0400 Subject: [PATCH 05/30] feat(prisma): per-project Sci-Hub switch, on for LEAD --- src/cli/prisma.py | 2 + src/prisma/fetch.py | 15 ++- tests/prisma/test_fetch_deep.py | 12 +- tests/prisma/test_fetch_fallback_switch.py | 134 +++++++++++++++++++++ 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 tests/prisma/test_fetch_fallback_switch.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 0a52f8e..74bc6c8 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -398,6 +398,7 @@ def fetch( fetch_proxy=fetch_proxy, limit=limit or None, workers=workers, + allow_fallback=_fetch.fallback_allowed(name), ) stats = _hold(_go, hold=hold) @@ -599,6 +600,7 @@ def run_all( email=os.environ.get("UNPAYWALL_EMAIL", "dev@fhirworx.io"), fetch_proxy=fetch_proxy, limit=limit or None, + allow_fallback=_fetch.fallback_allowed(name), ) typer.echo(f" {fetch_stats}") diff --git a/src/prisma/fetch.py b/src/prisma/fetch.py index f521cf9..5497f64 100644 --- a/src/prisma/fetch.py +++ b/src/prisma/fetch.py @@ -56,6 +56,16 @@ _FALLBACK_MIRRORS = ( "https://sci-hub.box", ) +# Projects that must not use the mirror tier. Empty: the five LEAD +# reviews leave the fallback on, and any other project can be named here. +SCIHUB_OFF: frozenset[str] = frozenset() + + +def fallback_allowed(project: str) -> bool: + """True when this project's fetch may use the mirror tier.""" + return project not in SCIHUB_OFF + + # The fallback tier shares one proxied client (altcha solves mutate its # cookie jar) and one VPN exit — hold this across the whole tier. _fallback_lock = threading.Lock() @@ -621,6 +631,7 @@ def fetch_one( scratch: Path, client: httpx.Client, client_proxied: httpx.Client | None = None, + allow_fallback: bool = True, ) -> tuple[str, Path] | None: """Run one item through the source cascade. @@ -668,7 +679,7 @@ def fetch_one( if path: return "pmc", path - if client_proxied is not None: + if allow_fallback and client_proxied is not None: with _fallback_lock: url = fetch_fallback(client_proxied, doi) if url: @@ -723,6 +734,7 @@ def run( limit: int | None = None, progress: bool = True, workers: int = 8, + allow_fallback: bool = True, ) -> dict[str, int]: """Fetch PDFs for every non-excluded item in the project. @@ -763,6 +775,7 @@ def run( scratch=scratch_dir, client=client, client_proxied=client_proxied, + allow_fallback=allow_fallback, ): item for item in pending } diff --git a/tests/prisma/test_fetch_deep.py b/tests/prisma/test_fetch_deep.py index 6587e65..0aa7956 100644 --- a/tests/prisma/test_fetch_deep.py +++ b/tests/prisma/test_fetch_deep.py @@ -105,7 +105,9 @@ class TestRunParallel: def test_all_items_attached_and_counted(self, zotero_db, tmp_path, monkeypatch): _seed_project(zotero_db, 6) - def fake_fetch_one(item, *, email, scratch, client, client_proxied=None): + def fake_fetch_one( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): time.sleep(0.05) p = scratch / f"{item.zot_id}.pdf" p.write_bytes(b"%PDF-1.4 fake") @@ -133,7 +135,9 @@ class TestRunParallel: state = {"active": 0, "max_active": 0} gauge = threading.Lock() - def slow_fetch_one(item, *, email, scratch, client, client_proxied=None): + def slow_fetch_one( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): with gauge: state["active"] += 1 state["max_active"] = max(state["max_active"], state["active"]) @@ -159,7 +163,9 @@ class TestRunParallel: def test_worker_exception_counts_as_error(self, zotero_db, tmp_path, monkeypatch): _seed_project(zotero_db, 3) - def boom(item, *, email, scratch, client, client_proxied=None): + def boom( + item, *, email, scratch, client, client_proxied=None, allow_fallback=True + ): raise RuntimeError("kaput") monkeypatch.setattr(fetch_mod, "fetch_one", boom) diff --git a/tests/prisma/test_fetch_fallback_switch.py b/tests/prisma/test_fetch_fallback_switch.py new file mode 100644 index 0000000..091f632 --- /dev/null +++ b/tests/prisma/test_fetch_fallback_switch.py @@ -0,0 +1,134 @@ +"""Per-project switch for the Sci-Hub fallback. LEAD projects leave it on.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +import prisma.fetch as fetch +from cli.prisma import app +from prisma.fetch import PendingItem, fallback_allowed + +runner = CliRunner() + +LEAD = ( + "lead-history", + "lead-limits", + "lead-scan", + "lead-precursors", + "lead-external", +) + + +def _mock_db(): + db = MagicMock() + db.__enter__ = MagicMock(return_value=db) + db.__exit__ = MagicMock(return_value=False) + return db + + +def test_lead_projects_keep_the_fallback(): + for name in LEAD: + assert fallback_allowed(name) is True + + +def test_named_project_skips_the_mirror(monkeypatch): + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + assert fallback_allowed("skin-subs") is False + assert fallback_allowed("lead-history") is True + + +def test_allow_fallback_false_skips_the_mirror(tmp_path): + item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="") + path = tmp_path / "paper.pdf" + client = MagicMock() + with ( + patch("prisma.fetch.fetch_fallback", return_value="https://mirror/p.pdf") as fb, + patch("prisma.fetch.download", return_value=path), + ): + missed = fetch.fetch_one( + item, + email="e@x", + scratch=tmp_path, + client=client, + client_proxied=client, + allow_fallback=False, + ) + assert missed is None + fb.assert_not_called() + hit = fetch.fetch_one( + item, + email="e@x", + scratch=tmp_path, + client=client, + client_proxied=client, + allow_fallback=True, + ) + assert hit == ("fallback", path) + + +def test_run_threads_allow_fallback(tmp_path, monkeypatch): + item = PendingItem(zot_id=1, doi="10.1/x", title="T", pmcid="", pmid="") + seen: dict[str, object] = {} + + def spy(one, **kwargs): + seen.update(kwargs) + return None + + monkeypatch.setattr(fetch, "pending_queue", lambda *a, **k: [item]) + monkeypatch.setattr(fetch, "fetch_one", spy) + stats = fetch.run( + MagicMock(), + project="skin-subs", + storage_dir=tmp_path, + scratch_dir=tmp_path / "scratch", + email="e@x", + fetch_proxy=None, + allow_fallback=False, + progress=False, + ) + assert seen["allow_fallback"] is False + assert stats["missed"] == 1 + + +def test_fetch_command_passes_the_project_switch(monkeypatch): + with ( + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run, + patch("zot.db.Db", return_value=_mock_db()), + patch("prisma.vpn.status", return_value={"status": "down"}), + ): + result = runner.invoke( + app, ["fetch", "lead-precursors", "--no-hold", "--no-vpn"] + ) + assert result.exit_code == 0 + assert mc_run.call_args.kwargs["allow_fallback"] is True + + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + with ( + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_run, + patch("zot.db.Db", return_value=_mock_db()), + patch("prisma.vpn.status", return_value={"status": "down"}), + ): + result = runner.invoke(app, ["fetch", "skin-subs", "--no-hold", "--no-vpn"]) + assert result.exit_code == 0 + assert mc_run.call_args.kwargs["allow_fallback"] is False + + +def test_orchestrated_run_uses_the_same_switch(monkeypatch): + monkeypatch.setattr(fetch, "SCIHUB_OFF", frozenset({"skin-subs"})) + with ( + patch("prisma.screen.run", return_value={"screened": 0, "errors": 0}), + patch("prisma.fetch.run", return_value={"missed": 0}) as mc_fetch, + patch("prisma.eligibility.run", return_value={"assessed": 0}), + patch("prisma.extract.run", return_value={"extracted": 0}), + patch("prisma.flow.count", return_value=MagicMock()), + patch("prisma.flow.text_summary", return_value="ok"), + patch("prisma.vpn.status", return_value={"status": "down"}), + patch("prisma.llm.make_provider", return_value=MagicMock()), + patch("prisma.project.load", return_value=MagicMock()), + patch("zot.db.Db", return_value=_mock_db()), + ): + result = runner.invoke(app, ["run", "skin-subs", "--no-hold"]) + assert result.exit_code == 0 + assert mc_fetch.call_args.kwargs["allow_fallback"] is False -- 2.49.1 From 44b21f8d8dee64c43df5e538c431641f0f852e12 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:24:24 -0400 Subject: [PATCH 06/30] feat(prisma): add the five LEAD review starters --- src/prisma/project.py | 224 +++++++++++++++++++++++++++++ tests/prisma/test_lead_starters.py | 111 ++++++++++++++ 2 files changed, 335 insertions(+) create mode 100644 tests/prisma/test_lead_starters.py diff --git a/src/prisma/project.py b/src/prisma/project.py index 03dcf0b..73e0983 100644 --- a/src/prisma/project.py +++ b/src/prisma/project.py @@ -369,6 +369,205 @@ Included by default; excluded only with code: """ +_LEAD_SUSPECTING = """\ +Suspecting is in scope as both a code uplift and a care change. Each is a +hypothesized status transition, such as not High Needs to High Needs, an +absent HCC becoming present, or a quality denominator without a numerator +gaining one. A code uplift makes an existing status visible to the math. +Care changes the underlying state. Record how far each can move the score, +including when demographic reweighting shrinks the coding payoff. +""" + +_LEAD_CRITERIA_HISTORY = """\ +# LEAD history — inclusion criteria + +Strand 1. For each target in the LEAD registry, find where the component +came from, what assumption it rests on, what evidence supported it when it +was adopted and what has appeared since, and how it changed through MA, +MSSP, Pioneer, NGACO, and REACH on the way to LEAD. + +Include primary CMS methodology, the papers those methods cite, and +critiques of the same component. Exclude a record that names no registry +target and no predecessor of one. +""" + +_LEAD_EXTRACTION_HISTORY = """\ +# Extraction — lead-history + +- **target_id** — registry id this row is about +- **origin** — where the component came from +- **assumptions** — the assumption it rests on +- **supporting_evidence** — evidence at adoption and since +- **critiques** — published critiques of the component +- **changes_to_lead** — how it changed on the way to LEAD +- **status_in_lead** — adopted / modified / dropped +- **page_refs** — page or section locators +""" + +_LEAD_REASONS_HISTORY = """\ +# Exclusion reasons — lead-history + +- `off-topic` — names no registry target and no predecessor of one. +- `editorial` — editorial, commentary, or letter with no historical claim. +- `not-english` — no English text. +- `unavailable-fulltext` — full text cannot be retrieved. +""" + +_LEAD_CRITERIA_LIMITS = f"""\ +# LEAD limits — inclusion criteria + +Strand 2. For each registry target, the mathematical and empirical limits +on predictive accuracy, how far an ACO's actions can move the scored +result, bias across subgroups, and incentive distortion. + +{_LEAD_SUSPECTING} +Include a limit when it states a bound, a formula, or a number that can +be checked. Exclude commentary that names no limit. +""" + +_LEAD_EXTRACTION_LIMITS = """\ +# Extraction — lead-limits + +- **target_id** — registry id the limit applies to +- **limit_type** — accuracy / movability / fairness / incentive +- **statement** — the limit in one or two sentences +- **formula** — formula or bound, when the source gives one +- **empirical_value** — number the source reports, with its unit +- **computable_expression** — expression that can run on public data +- **page_refs** — page or section locators +""" + +_LEAD_REASONS_LIMITS = """\ +# Exclusion reasons — lead-limits + +- `off-topic` — not a limit on a registry target. +- `no-limit` — on topic but states no bound, formula, or checkable number. +- `editorial` — editorial, commentary, or letter. +- `not-english` — no English text. +- `unavailable-fulltext` — full text cannot be retrieved. +""" + +_LEAD_CRITERIA_SCAN = """\ +# LEAD environment scan — inclusion criteria + +Strand 3, from 2023 onward. What surrounds LEAD: adjacent policy, how the +industry identifies High Needs patients, public data sources and their +grain, current methods for risk and prognosis, and open-model candidates +with their licenses. + +Include a dated item the chassis would have to notice. Exclude undated +background with no consequence for the model. +""" + +_LEAD_EXTRACTION_SCAN = """\ +# Extraction — lead-scan + +- **item_type** — policy / method / data / model / industry +- **item_date** — date or year of the item +- **summary** — what the item says, in a few sentences +- **target_ids** — registry ids it bears on, if any +- **aacods** — AACODS appraisal notes +- **chassis_meaning** — what the chassis should do because of this item +""" + +_LEAD_REASONS_SCAN = """\ +# Exclusion reasons — lead-scan + +- `off-topic` — not about the environment LEAD is built in. +- `undated` — no date, and no way to place it in 2023 or later. +- `editorial` — color with no fact the chassis can use. +- `not-english` — no English text. +- `unavailable-fulltext` — full text cannot be retrieved. +""" + +_LEAD_CRITERIA_PRECURSORS = f"""\ +# LEAD precursors — inclusion criteria + +Strand 4, a prognostic-factor review. The population is Medicare +beneficiaries, or comparable older, disabled, or ESRD adults, in claims +or EHR cohorts. An index factor is measured before the outcome. The +outcome is a registry target or a named proxy, and the lead time is +recorded. Eligible designs are cohort, case-control, prediction-model +development or validation, and quasi-experimental. Grain may be the +beneficiary, the provider, the ACO, or an area. + +A cross-sectional study with no temporal order is excluded. + +{_LEAD_SUSPECTING} +""" + +_LEAD_EXTRACTION_PRECURSORS = """\ +# Extraction — lead-precursors + +- **design** — study design as reported +- **data_source** — claims, EHR, registry, or survey named by the paper +- **population** — who was studied +- **n** — sample size +- **setting** — care setting +- **country** — country of the data +- **years** — years the data cover +- **factor_definition** — how the precursor was defined +- **outcome_target_id** — registry id of the outcome, or of its proxy +- **proxy_closeness** — how close a proxy is to the registry target +- **lead_time** — time from factor to outcome +- **grain** — beneficiary / provider / ACO / area +- **effect_measure** — measure the paper reports +- **effect_value** — point estimate +- **effect_ci** — confidence interval or other uncertainty +- **adjusted_for** — covariates the estimate adjusts for +- **c_statistic** — discrimination, when reported +- **calibration** — calibration, when reported +- **risk_of_bias** — bias appraisal, with the tool named +- **observable_public** — whether public data can supply the factor +- **observable_claims** — whether claims can supply the factor +""" + +_LEAD_REASONS_PRECURSORS = """\ +# Exclusion reasons — lead-precursors + +- `off-topic` — not a precursor of a registry target. +- `wrong-population` — not Medicare or a comparable older, disabled, or ESRD cohort. +- `cross-sectional-no-time-order` — cross-sectional study with no temporal order. +- `editorial` — editorial, commentary, or letter with no study result. +- `not-english` — no English text. +- `unavailable-fulltext` — full text cannot be retrieved. +- `wrong-outcome` — outcome is not a registry target and not a named proxy. +""" + +_LEAD_CRITERIA_EXTERNAL = """\ +# LEAD external forces — inclusion criteria + +Strand 5. How forces outside the ACO move LEAD's inputs and scores: labor +markets and prices, legislation and rulemaking, tax and coverage policy, +and market structure. + +Include a series or a dated event when the record states a mechanism that +reaches a registry target. Exclude a series with no stated path into the +score. +""" + +_LEAD_EXTRACTION_EXTERNAL = """\ +# Extraction — lead-external + +- **series_or_event** — name of the series or the dated event +- **dates** — period or date the record covers +- **geography** — geography the figure applies to +- **source** — agency or publication the figure comes from +- **mechanism** — how this force reaches a LEAD input or score +- **magnitude** — size of the move, with its unit +""" + +_LEAD_REASONS_EXTERNAL = """\ +# Exclusion reasons — lead-external + +- `off-topic` — no path into a LEAD input or score. +- `no-mechanism` — a series or event with no stated mechanism. +- `editorial` — commentary with no series and no dated event. +- `not-english` — no English text. +- `unavailable-fulltext` — full text cannot be retrieved. +""" + + STARTERS = { "skin-subs": ( _STARTER_CRITERIA_SKIN_SUBS, @@ -380,6 +579,31 @@ STARTERS = { _STARTER_EXTRACTION_PALLIATIVE_RFI, _STARTER_REASONS_PALLIATIVE_RFI, ), + "lead-history": ( + _LEAD_CRITERIA_HISTORY, + _LEAD_EXTRACTION_HISTORY, + _LEAD_REASONS_HISTORY, + ), + "lead-limits": ( + _LEAD_CRITERIA_LIMITS, + _LEAD_EXTRACTION_LIMITS, + _LEAD_REASONS_LIMITS, + ), + "lead-scan": ( + _LEAD_CRITERIA_SCAN, + _LEAD_EXTRACTION_SCAN, + _LEAD_REASONS_SCAN, + ), + "lead-precursors": ( + _LEAD_CRITERIA_PRECURSORS, + _LEAD_EXTRACTION_PRECURSORS, + _LEAD_REASONS_PRECURSORS, + ), + "lead-external": ( + _LEAD_CRITERIA_EXTERNAL, + _LEAD_EXTRACTION_EXTERNAL, + _LEAD_REASONS_EXTERNAL, + ), } diff --git a/tests/prisma/test_lead_starters.py b/tests/prisma/test_lead_starters.py new file mode 100644 index 0000000..7cea83c --- /dev/null +++ b/tests/prisma/test_lead_starters.py @@ -0,0 +1,111 @@ +"""The five LEAD review projects init from starter criteria.""" + +from __future__ import annotations + +import pytest + +from prisma.project import init +from prisma.tools import parse_extraction_fields +from zot.db import Db +from zot.schema import create_db + +FIELDS = { + "lead-history": ( + "target_id", + "origin", + "assumptions", + "supporting_evidence", + "critiques", + "changes_to_lead", + "status_in_lead", + "page_refs", + ), + "lead-limits": ( + "target_id", + "limit_type", + "statement", + "formula", + "empirical_value", + "computable_expression", + "page_refs", + ), + "lead-scan": ( + "item_type", + "item_date", + "summary", + "target_ids", + "aacods", + "chassis_meaning", + ), + "lead-precursors": ( + "design", + "data_source", + "population", + "n", + "setting", + "country", + "years", + "factor_definition", + "outcome_target_id", + "proxy_closeness", + "lead_time", + "grain", + "effect_measure", + "effect_value", + "effect_ci", + "adjusted_for", + "c_statistic", + "calibration", + "risk_of_bias", + "observable_public", + "observable_claims", + ), + "lead-external": ( + "series_or_event", + "dates", + "geography", + "source", + "mechanism", + "magnitude", + ), +} + +ENUMS = { + "lead-history": {"status_in_lead": ["adopted", "modified", "dropped"]}, + "lead-limits": {"limit_type": ["accuracy", "movability", "fairness", "incentive"]}, + "lead-scan": {"item_type": ["policy", "method", "data", "model", "industry"]}, + "lead-precursors": {"grain": ["beneficiary", "provider", "ACO", "area"]}, +} + + +def _db(tmp_path) -> str: + path = str(tmp_path / "z.sqlite") + con = create_db(path) + con.close() + return path + + +def test_each_lead_starter_inits(tmp_path): + with Db(_db(tmp_path)) as db: + for name, fields in FIELDS.items(): + project = init(db, name) + assert project.name == name + props = parse_extraction_fields(project.extraction_template) + assert set(props) == set(fields) + for field, members in ENUMS.get(name, {}).items(): + assert props[field]["enum"] == [*members, ""] + with pytest.raises(ValueError): + init(db, "nope") + + +def test_criteria_pin_suspecting_and_temporal_order(tmp_path): + with Db(_db(tmp_path)) as db: + limits = init(db, "lead-limits") + precursors = init(db, "lead-precursors") + for project in (limits, precursors): + assert "code uplift" in project.criteria + assert "care" in project.criteria + assert "status" in project.criteria + assert "cross-sectional" in precursors.criteria + assert "temporal order" in precursors.criteria + assert "cross-sectional-no-time-order" in precursors.reasons -- 2.49.1 From 937a2bbfd11a4594264fcc7ce648982946b67871 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:27:05 -0400 Subject: [PATCH 07/30] feat(prisma): dedup on DOI, PMID, then title and year --- src/cli/prisma.py | 17 ++++ src/prisma/dedup.py | 177 +++++++++++++++++++++++++++++++++++++ tests/prisma/test_dedup.py | 154 ++++++++++++++++++++++++++++++++ 3 files changed, 348 insertions(+) create mode 100644 src/prisma/dedup.py create mode 100644 tests/prisma/test_dedup.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 74bc6c8..0bcd5d3 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -406,6 +406,23 @@ def fetch( typer.echo(f" {k}: {v}") +@app.command(name="dedup") +def dedup_cmd( + db: Path = typer.Option(_DEFAULT_ZOT, "--db"), + hold: bool = typer.Option(True, "--hold/--no-hold"), +) -> None: + """Tag DOI, PMID, and title duplicates. Near-matches stay untagged.""" + from prisma.dedup import mark_duplicates + from zot.db import Db + + def _go() -> int: + with Db(str(db)) as zdb: + return mark_duplicates(zdb) + + tagged = _hold(_go, hold=hold) + typer.echo(f" tagged: {tagged}") + + # ── VPN droplet lifecycle ─────────────────────────────────────── diff --git a/src/prisma/dedup.py b/src/prisma/dedup.py new file mode 100644 index 0000000..678beba --- /dev/null +++ b/src/prisma/dedup.py @@ -0,0 +1,177 @@ +"""Duplicate detection across the LEAD review projects. + +Match order is DOI, then PMID, then normalized title plus year plus the +first author's surname. A near title in the same year stays a borderline +pair and is not merged. +""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from dataclasses import dataclass +from difflib import SequenceMatcher + +from zot.db import TYPE_MAP, Db + +_YEAR = re.compile(r"(\d{4})") +_SIMILAR = 0.85 +DUPLICATE_TAG = "dedup:duplicate" + + +@dataclass(frozen=True) +class Record: + """One bibliographic item, as dedup sees it.""" + + key: str + doi: str + pmid: str + title: str + year: int | None + first_author: str + + +@dataclass(frozen=True) +class DedupResult: + """Groups of the same work, plus near-miss pairs left unmerged.""" + + groups: tuple[tuple[Record, ...], ...] + borderline: tuple[tuple[Record, Record], ...] + + +def normalize(title: str) -> str: + """Lowercase, drop punctuation, and collapse whitespace.""" + chars = [ch if ch.isalnum() or ch.isspace() else " " for ch in title.casefold()] + return " ".join("".join(chars).split()) + + +def _doi_key(doi: str) -> str: + return doi.strip().casefold().rstrip(".") + + +def _title_key(rec: Record) -> tuple[str, int | None, str] | None: + title = normalize(rec.title) + if not title: + return None + return (title, rec.year, rec.first_author.casefold().strip()) + + +def cluster(records: Sequence[Record]) -> DedupResult: + """Group exact duplicates. Near titles become borderline pairs.""" + parent = list(range(len(records))) + + def find(i: int) -> int: + while parent[i] != i: + parent[i] = parent[parent[i]] + i = parent[i] + return i + + def union(i: int, j: int) -> None: + ri, rj = find(i), find(j) + if ri == rj: + return + if ri < rj: + parent[rj] = ri + else: + parent[ri] = rj + + def merge_on(key_of) -> None: + first: dict[object, int] = {} + for i, rec in enumerate(records): + key = key_of(rec) + if not key: + continue + prior = first.get(key) + if prior is None: + first[key] = i + else: + union(prior, i) + + merge_on(lambda rec: _doi_key(rec.doi)) + merge_on(lambda rec: rec.pmid.strip()) + merge_on(_title_key) + + members: dict[int, list[Record]] = {} + order: list[int] = [] + for i, rec in enumerate(records): + root = find(i) + if root not in members: + members[root] = [] + order.append(root) + members[root].append(rec) + groups = tuple(tuple(members[root]) for root in order) + + singles = [group[0] for group in groups if len(group) == 1] + borderline: list[tuple[Record, Record]] = [] + for i, left in enumerate(singles): + left_title = normalize(left.title) + if not left_title: + continue + for right in singles[i + 1 :]: + if left.year != right.year: + continue + right_title = normalize(right.title) + if not right_title: + continue + ratio = SequenceMatcher(None, left_title, right_title).ratio() + if ratio >= _SIMILAR: + borderline.append((left, right)) + return DedupResult(groups=groups, borderline=tuple(borderline)) + + +def _year(value: str | None) -> int | None: + if not value: + return None + match = _YEAR.search(value) + return int(match.group(1)) if match else None + + +def load_records(db: Db) -> tuple[Record, ...]: + """Bibliographic items, anchors excluded, in item order.""" + rows = db.con.execute( + """ + SELECT i.itemID, i.key + FROM items i + WHERE i.itemTypeID NOT IN (?, ?) + AND NOT EXISTS ( + SELECT 1 FROM itemTags it + JOIN tags t ON t.tagID = it.tagID + WHERE it.itemID = i.itemID AND t.name = 'module:prisma' + ) + ORDER BY i.itemID + """, + (TYPE_MAP["note"], TYPE_MAP["attachment"]), + ).fetchall() + records: list[Record] = [] + for item_id, key in rows: + creators = db.get_creators(item_id) + author = "" + if creators: + author = str(creators[0]["lastName"] or creators[0]["firstName"] or "") + records.append( + Record( + key=str(key), + doi=db.get_field(item_id, "DOI") or "", + pmid=db.get_field(item_id, "PMID") or "", + title=db.get_field(item_id, "title") or "", + year=_year(db.get_field(item_id, "date")), + first_author=author, + ) + ) + return tuple(records) + + +def mark_duplicates(db: Db) -> int: + """Tag every record in a group except the first. Return how many were new.""" + written = 0 + for group in cluster(load_records(db)).groups: + for rec in group[1:]: + item_id = db.find_item_by_key(rec.key) + if item_id is None: + continue + if DUPLICATE_TAG in db.get_tags(item_id): + continue + db.tag_item(item_id, DUPLICATE_TAG) + written += 1 + db.commit() + return written diff --git a/tests/prisma/test_dedup.py b/tests/prisma/test_dedup.py new file mode 100644 index 0000000..67083ee --- /dev/null +++ b/tests/prisma/test_dedup.py @@ -0,0 +1,154 @@ +"""Dedup on DOI, then PMID, then normalized title, year, and author.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from cli.prisma import app +from prisma.dedup import Record, cluster, mark_duplicates, normalize +from zot.db import TYPE_MAP, Db +from zot.schema import create_db + +runner = CliRunner() + + +def _rec(**kwargs: object) -> Record: + base: dict[str, object] = { + "key": "AAAAAAAA", + "doi": "", + "pmid": "", + "title": "", + "year": 2020, + "first_author": "", + } + base.update(kwargs) + return Record(**base) # type: ignore[arg-type] + + +def test_normalize_lowercases_and_drops_punctuation(): + assert normalize("Risk Adjustment!") == "risk adjustment" + + +def test_exact_doi_ignores_case_and_a_trailing_period(): + first = _rec(key="AAAAAAAA", doi="10.1000/ABC.", title="One", first_author="Kim") + second = _rec(key="BBBBBBBB", doi="10.1000/abc", title="Two", first_author="Lee") + result = cluster([first, second]) + assert len(result.groups) == 1 + assert {rec.key for rec in result.groups[0]} == {"AAAAAAAA", "BBBBBBBB"} + assert result.borderline == () + + +def test_same_pmid_is_one_group(): + first = _rec(key="AAAAAAAA", pmid="12345", title="One") + second = _rec(key="BBBBBBBB", pmid="12345", title="Two") + result = cluster([first, second]) + assert len(result.groups) == 1 + assert {rec.key for rec in result.groups[0]} == {"AAAAAAAA", "BBBBBBBB"} + + +def test_normalized_title_year_and_author_is_one_group(): + first = _rec( + key="AAAAAAAA", title="Risk Adjustment!", year=2020, first_author="Pope" + ) + second = _rec( + key="BBBBBBBB", title="risk adjustment", year=2020, first_author="Pope" + ) + result = cluster([first, second]) + assert len(result.groups) == 1 + assert {rec.key for rec in result.groups[0]} == {"AAAAAAAA", "BBBBBBBB"} + + +def test_near_titles_stay_borderline_and_unmerged(): + first = _rec( + key="AAAAAAAA", + title="High needs frailty index", + year=2024, + first_author="Kim", + ) + second = _rec( + key="BBBBBBBB", + title="High-need frailty index", + year=2024, + first_author="Lee", + ) + result = cluster([first, second]) + assert {tuple(rec.key for rec in group) for group in result.groups} == { + ("AAAAAAAA",), + ("BBBBBBBB",), + } + assert result.borderline == ((first, second),) + + +def test_dissimilar_titles_are_neither_merged_nor_borderline(): + first = _rec(key="AAAAAAAA", title="zzzz unique alpha", first_author="Kim") + second = _rec(key="BBBBBBBB", title="qqqq other beta", first_author="Lee") + result = cluster([first, second]) + assert len(result.groups) == 2 + assert result.borderline == () + + +def _db(tmp_path) -> str: + path = str(tmp_path / "z.sqlite") + con = create_db(path) + con.close() + return path + + +def _add( + db: Db, *, key: str, doi: str, title: str, year: int, author: str, pmid: str = "" +) -> None: + item_id = db.create_item(TYPE_MAP["journalArticle"], key=key) + db.set_fields( + item_id, + {"title": title, "DOI": doi, "date": str(year), "PMID": pmid}, + ) + db.add_creators(item_id, [("", author)]) + db.commit() + + +def _dup_count(db: Db) -> int: + row = db.con.execute( + """ + SELECT count(*) FROM itemTags it + JOIN tags t ON t.tagID = it.tagID + WHERE t.name = 'dedup:duplicate' + """ + ).fetchone() + return int(row[0]) + + +def test_shared_doi_writes_one_duplicate_tag(tmp_path): + with Db(_db(tmp_path)) as db: + _add(db, key="ABCD2345", doi="10.1/ABC.", title="One", year=2020, author="Kim") + _add(db, key="EFGH5678", doi="10.1/abc", title="Two", year=2021, author="Lee") + mark_duplicates(db) + assert _dup_count(db) == 1 + assert "dedup:duplicate" not in db.get_tags(db.find_item_by_key("ABCD2345")) + assert "dedup:duplicate" in db.get_tags(db.find_item_by_key("EFGH5678")) + + +def test_borderline_items_are_not_tagged(tmp_path): + with Db(_db(tmp_path)) as db: + _add( + db, + key="ABCD2345", + doi="", + title="High needs frailty index", + year=2024, + author="Kim", + ) + _add( + db, + key="EFGH5678", + doi="", + title="High-need frailty index", + year=2024, + author="Lee", + ) + mark_duplicates(db) + assert _dup_count(db) == 0 + + +def test_dedup_command_is_registered(): + result = runner.invoke(app, ["dedup", "--help"]) + assert result.exit_code == 0 -- 2.49.1 From 43d8e51886244cb00be45c24430654bf92426406 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:29:18 -0400 Subject: [PATCH 08/30] feat(prisma): PubMed search log and seed recall --- src/cli/prisma.py | 14 +++ src/prisma/search/__init__.py | 23 +++++ src/prisma/search/base.py | 119 ++++++++++++++++++++++ src/prisma/search/protocols.py | 27 +++++ src/prisma/search/pubmed.py | 57 +++++++++++ tests/prisma/test_search_pubmed.py | 155 +++++++++++++++++++++++++++++ 6 files changed, 395 insertions(+) create mode 100644 src/prisma/search/__init__.py create mode 100644 src/prisma/search/base.py create mode 100644 src/prisma/search/protocols.py create mode 100644 src/prisma/search/pubmed.py create mode 100644 tests/prisma/test_search_pubmed.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 0bcd5d3..84e72d9 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -423,6 +423,20 @@ def dedup_cmd( typer.echo(f" tagged: {tagged}") +search_app = typer.Typer(no_args_is_help=True) +app.add_typer(search_app, name="search", help="Search logs and strand protocols.") + + +@search_app.command("protocols") +def search_protocols() -> None: + """Print the starting PubMed query for each LEAD project.""" + from prisma.search.protocols import PROTOCOLS + + for name in sorted(PROTOCOLS): + for query in PROTOCOLS[name]: + typer.echo(f"{name}\t{query}") + + # ── VPN droplet lifecycle ─────────────────────────────────────── diff --git a/src/prisma/search/__init__.py b/src/prisma/search/__init__.py new file mode 100644 index 0000000..2370684 --- /dev/null +++ b/src/prisma/search/__init__.py @@ -0,0 +1,23 @@ +"""Literature search logs for the LEAD review. Parsers do not fetch.""" + +from prisma.search.base import ( + SearchHit, + SearchLog, + SearchResult, + import_hits, + record, + seed_recall, +) +from prisma.search.protocols import PROTOCOLS +from prisma.search.pubmed import parse_pubmed_xml + +__all__ = [ + "PROTOCOLS", + "SearchHit", + "SearchLog", + "SearchResult", + "import_hits", + "parse_pubmed_xml", + "record", + "seed_recall", +] diff --git a/src/prisma/search/base.py b/src/prisma/search/base.py new file mode 100644 index 0000000..7b10701 --- /dev/null +++ b/src/prisma/search/base.py @@ -0,0 +1,119 @@ +"""Search hits, the PRISMA-S log row, and import into Zotero.""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass + +from lead.ledger.store import append +from zot.db import FIELD_MAP, TYPE_MAP, Db + + +@dataclass(frozen=True) +class SearchHit: + """One retrieved record.""" + + title: str + year: int | None + doi: str + pmid: str + first_author: str + database: str + + +@dataclass(frozen=True) +class SearchLog: + """One execution of one query against one database.""" + + database: str + interface: str + searched_on: str + query: str + filters: str + hit_count: int + export_sha256: str + + +@dataclass(frozen=True) +class SearchResult: + """Hits plus the log row that produced them.""" + + hits: tuple[SearchHit, ...] + log: SearchLog + + +def seed_recall(retrieved: set[str], seeds: set[str]) -> float: + """Share of the seed set present in the retrieved set. + + Raises: + ValueError: ``seeds`` is empty, so the ratio is undefined. + """ + if not seeds: + raise ValueError("seeds must not be empty") + return len(retrieved & seeds) / len(seeds) + + +def record(con, result: SearchResult, *, actor: str) -> bool: + """Append one search row. The same export on a later day is a new row.""" + log = result.log + raw = "\n".join( + [ + log.database, + log.interface, + log.searched_on, + log.query, + log.filters, + str(log.hit_count), + log.export_sha256, + actor, + ] + ) + return append( + con, + "search", + { + "database": log.database, + "interface": log.interface, + "searched_on": log.searched_on, + "query": log.query, + "filters": log.filters, + "hit_count": log.hit_count, + "export_sha256": log.export_sha256, + "content_hash": hashlib.sha256(raw.encode()).hexdigest(), + "actor": actor, + }, + ) + + +def import_hits(zdb: Db, project: str, result: SearchResult) -> int: + """Create one Zotero item per new DOI. Return how many were new.""" + inserted = 0 + for hit in result.hits: + if hit.doi and _doi_exists(zdb, hit.doi): + continue + item_id = zdb.create_item(TYPE_MAP["journalArticle"]) + fields = {"title": hit.title, "DOI": hit.doi, "PMID": hit.pmid} + if hit.year is not None: + fields["date"] = str(hit.year) + zdb.set_fields(item_id, fields) + if hit.first_author: + zdb.add_creators(item_id, [("", hit.first_author)]) + zdb.tag_item(item_id, f"project:{project}") + zdb.tag_item(item_id, f"found-in:{hit.database}") + inserted += 1 + zdb.commit() + return inserted + + +def _doi_exists(zdb: Db, doi: str) -> bool: + row = zdb.con.execute( + """ + SELECT 1 + FROM itemData id + JOIN itemDataValues idv ON id.valueID = idv.valueID + WHERE id.fieldID = ? AND lower(idv.value) = lower(?) + LIMIT 1 + """, + (FIELD_MAP["DOI"], doi.strip()), + ).fetchone() + return row is not None diff --git a/src/prisma/search/protocols.py b/src/prisma/search/protocols.py new file mode 100644 index 0000000..634bc8d --- /dev/null +++ b/src/prisma/search/protocols.py @@ -0,0 +1,27 @@ +"""Starting PubMed query strings for the five LEAD strands. + +These are the strings a nightly run fires. They are not a measured +recall, and they are not a claim that the query is finished. +""" + +from __future__ import annotations + +PROTOCOLS: dict[str, tuple[str, ...]] = { + "lead-history": ( + '("CMS-HCC"[tiab] OR "hierarchical condition category"[tiab]) AND Medicare[tiab]', + '("Accountable Care Organization"[tiab]) AND (REACH[tiab] OR MSSP[tiab])', + ), + "lead-limits": ( + '("risk adjustment"[tiab]) AND Medicare[tiab] AND (accuracy[tiab] OR cap[tiab])', + ), + "lead-scan": ( + '("Medicare Shared Savings"[tiab] OR "ACO REACH"[tiab]) AND 2023:3000[dp]', + ), + "lead-precursors": ( + "(frailty[tiab] OR readmission[tiab]) AND Medicare[tiab] " + "AND (cohort[tiab] OR prognostic[tiab])", + ), + "lead-external": ( + '("Medicare Advantage"[tiab]) AND (penetration[tiab] OR sequestration[tiab])', + ), +} diff --git a/src/prisma/search/pubmed.py b/src/prisma/search/pubmed.py new file mode 100644 index 0000000..3edd7ef --- /dev/null +++ b/src/prisma/search/pubmed.py @@ -0,0 +1,57 @@ +"""PubMed efetch XML → search hits. The caller supplies the file.""" + +from __future__ import annotations + +import hashlib +import xml.etree.ElementTree as ET + +from prisma.search.base import SearchHit, SearchLog, SearchResult + + +def _text(node: ET.Element | None) -> str: + if node is None: + return "" + return "".join(node.itertext()).strip() + + +def _year(article: ET.Element) -> int | None: + raw = _text(article.find(".//PubDate/Year")) + if not raw: + raw = _text(article.find(".//PubDate/MedlineDate"))[:4] + return int(raw) if raw.isdigit() else None + + +def _doi(article: ET.Element) -> str: + for node in article.findall(".//ArticleId"): + if (node.get("IdType") or "").casefold() == "doi": + return (node.text or "").strip() + return "" + + +def parse_pubmed_xml(xml: str, *, query: str, searched_on: str) -> SearchResult: + """Read one PubmedArticleSet. Missing fields become empty or None.""" + root = ET.fromstring(xml) + hits: list[SearchHit] = [] + for article in root.iter("PubmedArticle"): + author = article.find(".//Author/LastName") + hits.append( + SearchHit( + title=_text(article.find(".//ArticleTitle")), + year=_year(article), + doi=_doi(article), + pmid=_text(article.find("./MedlineCitation/PMID")), + first_author=_text(author), + database="pubmed", + ) + ) + found = tuple(hits) + log = SearchLog( + database="pubmed", + interface="eutils", + searched_on=searched_on, + query=query, + filters="", + hit_count=len(found), + export_sha256=hashlib.sha256(xml.encode()).hexdigest(), + ) + return SearchResult(hits=found, log=log) diff --git a/tests/prisma/test_search_pubmed.py b/tests/prisma/test_search_pubmed.py new file mode 100644 index 0000000..778fa0b --- /dev/null +++ b/tests/prisma/test_search_pubmed.py @@ -0,0 +1,155 @@ +"""PubMed search log, parser, and seed recall. No live HTTP.""" + +from __future__ import annotations + +import hashlib + +import pytest +from typer.testing import CliRunner + +from cli.prisma import app +from lead.ledger.store import open_ledger +from prisma.search.base import ( + SearchHit, + SearchLog, + SearchResult, + import_hits, + record, + seed_recall, +) +from prisma.search.protocols import PROTOCOLS +from prisma.search.pubmed import parse_pubmed_xml +from zot.db import TYPE_MAP, Db +from zot.schema import create_db + +runner = CliRunner() + +_XML = """\ + + + + + 111 +
+ Alpha frailty + 2020 + Kim +
+ + + + 10.1/alpha + + + + + + 222 +
+ Beta readmission + 2021 + Lee +
+ + + + 10.1/beta + + + + +""" + + +def test_parse_two_articles_and_hash_the_xml(): + result = parse_pubmed_xml(_XML, query="frailty", searched_on="2026-10-31") + assert result.log.database == "pubmed" + assert result.log.hit_count == 2 + assert result.log.export_sha256 == hashlib.sha256(_XML.encode()).hexdigest() + assert {hit.database for hit in result.hits} == {"pubmed"} + assert {hit.title for hit in result.hits} == {"Alpha frailty", "Beta readmission"} + assert {hit.year for hit in result.hits} == {2020, 2021} + assert {hit.doi for hit in result.hits} == {"10.1/alpha", "10.1/beta"} + assert {hit.pmid for hit in result.hits} == {"111", "222"} + assert {hit.first_author for hit in result.hits} == {"Kim", "Lee"} + + +def test_seed_recall(): + assert seed_recall({"a", "b"}, {"a", "c"}) == 0.5 + with pytest.raises(ValueError): + seed_recall(set(), set()) + + +def test_same_xml_on_two_days_appends_two_search_rows(tmp_path): + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + first = parse_pubmed_xml(_XML, query="frailty", searched_on="2026-10-31") + second = parse_pubmed_xml(_XML, query="frailty", searched_on="2026-11-01") + assert record(con, first, actor="tester") is True + assert record(con, second, actor="tester") is True + count = con.execute("select count(*) from lead_evidence.search").fetchone()[0] + assert count == 2 + finally: + con.close() + + +def test_import_hits_is_idempotent_on_doi(tmp_path): + path = str(tmp_path / "z.sqlite") + con = create_db(path) + con.close() + hit = SearchHit( + title="Alpha frailty", + year=2020, + doi="10.1/alpha", + pmid="111", + first_author="Kim", + database="pubmed", + ) + result = SearchResult( + hits=(hit,), + log=SearchLog( + database="pubmed", + interface="eutils", + searched_on="2026-10-31", + query="frailty", + filters="", + hit_count=1, + export_sha256="abc", + ), + ) + with Db(path) as db: + assert import_hits(db, "lead-precursors", result) == 1 + names = { + row[0] + for row in db.con.execute( + """ + SELECT t.name FROM tags t + JOIN itemTags it ON it.tagID = t.tagID + """ + ) + } + assert "found-in:pubmed" in names + assert "project:lead-precursors" in names + assert import_hits(db, "lead-precursors", result) == 0 + articles = db.con.execute( + "select count(*) from items where itemTypeID = ?", + (TYPE_MAP["journalArticle"],), + ).fetchone()[0] + assert articles == 1 + + +def test_protocols_cover_the_five_projects(): + assert set(PROTOCOLS) == { + "lead-history", + "lead-limits", + "lead-scan", + "lead-precursors", + "lead-external", + } + assert PROTOCOLS["lead-precursors"] + assert all(isinstance(q, str) and q for q in PROTOCOLS["lead-precursors"]) + + +def test_search_command_is_registered(): + result = runner.invoke(app, ["search", "--help"]) + assert result.exit_code == 0 -- 2.49.1 From aa1d31db247951e87ff0ae57416ec7d26b140839 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:31:21 -0400 Subject: [PATCH 09/30] feat(prisma): parsers for Europe PMC, OpenAlex, Crossref, Semantic Scholar, arXiv --- src/prisma/search/arxiv.py | 52 +++++++++++++ src/prisma/search/base.py | 48 ++++++++++++ src/prisma/search/crossref.py | 43 +++++++++++ src/prisma/search/europepmc.py | 33 ++++++++ src/prisma/search/openalex.py | 36 +++++++++ src/prisma/search/semanticscholar.py | 39 ++++++++++ tests/prisma/test_search_others.py | 108 +++++++++++++++++++++++++++ 7 files changed, 359 insertions(+) create mode 100644 src/prisma/search/arxiv.py create mode 100644 src/prisma/search/crossref.py create mode 100644 src/prisma/search/europepmc.py create mode 100644 src/prisma/search/openalex.py create mode 100644 src/prisma/search/semanticscholar.py create mode 100644 tests/prisma/test_search_others.py diff --git a/src/prisma/search/arxiv.py b/src/prisma/search/arxiv.py new file mode 100644 index 0000000..7b0bb17 --- /dev/null +++ b/src/prisma/search/arxiv.py @@ -0,0 +1,52 @@ +"""arXiv Atom XML. The caller supplies the file.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from prisma.search.base import ( + SearchHit, + SearchResult, + bare_doi, + finish, + surname, + year_of, +) + +_ATOM = "http://www.w3.org/2005/Atom" +_ARXIV = "http://arxiv.org/schemas/atom" + + +def _child(node: ET.Element, tag: str, ns: str = _ATOM) -> ET.Element | None: + found = node.find(f"{{{ns}}}{tag}") + return node.find(tag) if found is None else found + + +def _text(node: ET.Element | None) -> str: + if node is None: + return "" + return "".join(node.itertext()).strip() + + +def parse_arxiv(payload: str, *, query: str, searched_on: str) -> SearchResult: + """Read one arXiv Atom feed. A blank payload has no hits.""" + if not payload.strip(): + return finish("arxiv", [], payload, query=query, searched_on=searched_on) + root = ET.fromstring(payload) + entries = list(root.iter(f"{{{_ATOM}}}entry")) or list(root.iter("entry")) + hits: list[SearchHit] = [] + for entry in entries: + author = _child(entry, "author") + name = _text(_child(author, "name")) if author is not None else "" + doi = _text(_child(entry, "doi", _ARXIV)) or _text(_child(entry, "doi")) + hits.append( + SearchHit( + title=_text(_child(entry, "title")), + year=year_of(_text(_child(entry, "published"))), + doi=bare_doi(doi), + pmid="", + first_author=surname(name), + database="arxiv", + ) + ) + return finish("arxiv", hits, payload, query=query, searched_on=searched_on) diff --git a/src/prisma/search/base.py b/src/prisma/search/base.py index 7b10701..c57d609 100644 --- a/src/prisma/search/base.py +++ b/src/prisma/search/base.py @@ -42,6 +42,54 @@ class SearchResult: log: SearchLog +def finish( + database: str, + hits: list[SearchHit], + payload: str, + *, + query: str, + searched_on: str, +) -> SearchResult: + """Wrap hits in a log row. ``hit_count`` is the number of hits.""" + found = tuple(hits) + return SearchResult( + hits=found, + log=SearchLog( + database=database, + interface=database, + searched_on=searched_on, + query=query, + filters="", + hit_count=len(found), + export_sha256=hashlib.sha256(payload.encode()).hexdigest(), + ), + ) + + +def surname(name: str) -> str: + """Last token of a display name. One token is returned as-is.""" + parts = name.split() + return parts[-1] if parts else "" + + +def bare_doi(value: str) -> str: + """Drop a ``doi.org`` or ``doi:`` prefix.""" + text = value.strip() + lowered = text.casefold() + for prefix in ("https://doi.org/", "http://doi.org/", "doi:"): + if lowered.startswith(prefix): + return text[len(prefix) :] + return text + + +def year_of(value: object) -> int | None: + """First four digits of a year, or None when the field is blank.""" + if value is None: + return None + head = str(value).strip()[:4] + return int(head) if head.isdigit() and len(head) == 4 else None + + def seed_recall(retrieved: set[str], seeds: set[str]) -> float: """Share of the seed set present in the retrieved set. diff --git a/src/prisma/search/crossref.py b/src/prisma/search/crossref.py new file mode 100644 index 0000000..e0a4228 --- /dev/null +++ b/src/prisma/search/crossref.py @@ -0,0 +1,43 @@ +"""Crossref works JSON. The caller supplies the file.""" + +from __future__ import annotations + +import json + +from prisma.search.base import SearchHit, SearchResult, bare_doi, finish, year_of + + +def _title(row: dict) -> str: + title = row.get("title") or "" + if isinstance(title, list): + return str(title[0]) if title else "" + return str(title) + + +def _year(row: dict) -> int | None: + parts = (row.get("issued") or {}).get("date-parts") or [] + if not parts or not parts[0]: + return None + return year_of(parts[0][0]) + + +def parse_crossref(payload: str, *, query: str, searched_on: str) -> SearchResult: + """Read one Crossref message. A blank payload has no hits.""" + rows = [] + if payload.strip(): + rows = ((json.loads(payload).get("message") or {}).get("items")) or [] + hits: list[SearchHit] = [] + for row in rows: + authors = row.get("author") or [] + family = (authors[0].get("family") or "") if authors else "" + hits.append( + SearchHit( + title=_title(row), + year=_year(row), + doi=bare_doi(row.get("DOI") or ""), + pmid="", + first_author=family, + database="crossref", + ) + ) + return finish("crossref", hits, payload, query=query, searched_on=searched_on) diff --git a/src/prisma/search/europepmc.py b/src/prisma/search/europepmc.py new file mode 100644 index 0000000..beaec9c --- /dev/null +++ b/src/prisma/search/europepmc.py @@ -0,0 +1,33 @@ +"""Europe PMC REST XML. The caller supplies the file.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from prisma.search.base import SearchHit, SearchResult, bare_doi, finish, year_of + + +def _text(node: ET.Element | None) -> str: + if node is None: + return "" + return "".join(node.itertext()).strip() + + +def parse_europepmc(payload: str, *, query: str, searched_on: str) -> SearchResult: + """Read one Europe PMC result list. A blank payload has no hits.""" + if not payload.strip(): + return finish("europepmc", [], payload, query=query, searched_on=searched_on) + root = ET.fromstring(payload) + hits: list[SearchHit] = [] + for result in root.iter("result"): + hits.append( + SearchHit( + title=_text(result.find("title")), + year=year_of(_text(result.find("pubYear"))), + doi=bare_doi(_text(result.find("doi"))), + pmid=_text(result.find("pmid")), + first_author=_text(result.find("./authorList/author/lastName")), + database="europepmc", + ) + ) + return finish("europepmc", hits, payload, query=query, searched_on=searched_on) diff --git a/src/prisma/search/openalex.py b/src/prisma/search/openalex.py new file mode 100644 index 0000000..65c7d3d --- /dev/null +++ b/src/prisma/search/openalex.py @@ -0,0 +1,36 @@ +"""OpenAlex works JSON. The caller supplies the file.""" + +from __future__ import annotations + +import json + +from prisma.search.base import ( + SearchHit, + SearchResult, + bare_doi, + finish, + surname, + year_of, +) + + +def parse_openalex(payload: str, *, query: str, searched_on: str) -> SearchResult: + """Read one OpenAlex works page. A blank payload has no hits.""" + rows = [] if not payload.strip() else (json.loads(payload).get("results") or []) + hits: list[SearchHit] = [] + for row in rows: + authorships = row.get("authorships") or [] + name = "" + if authorships: + name = ((authorships[0].get("author") or {}).get("display_name")) or "" + hits.append( + SearchHit( + title=row.get("title") or "", + year=year_of(row.get("publication_year")), + doi=bare_doi(row.get("doi") or ""), + pmid="", + first_author=surname(name), + database="openalex", + ) + ) + return finish("openalex", hits, payload, query=query, searched_on=searched_on) diff --git a/src/prisma/search/semanticscholar.py b/src/prisma/search/semanticscholar.py new file mode 100644 index 0000000..ee70bb0 --- /dev/null +++ b/src/prisma/search/semanticscholar.py @@ -0,0 +1,39 @@ +"""Semantic Scholar paper-search JSON. The caller supplies the file.""" + +from __future__ import annotations + +import json + +from prisma.search.base import ( + SearchHit, + SearchResult, + bare_doi, + finish, + surname, + year_of, +) + + +def parse_semanticscholar( + payload: str, *, query: str, searched_on: str +) -> SearchResult: + """Read one Semantic Scholar data page. A blank payload has no hits.""" + rows = [] if not payload.strip() else (json.loads(payload).get("data") or []) + hits: list[SearchHit] = [] + for row in rows: + authors = row.get("authors") or [] + name = (authors[0].get("name") or "") if authors else "" + doi = ((row.get("externalIds") or {}).get("DOI")) or "" + hits.append( + SearchHit( + title=row.get("title") or "", + year=year_of(row.get("year")), + doi=bare_doi(doi), + pmid="", + first_author=surname(name), + database="semanticscholar", + ) + ) + return finish( + "semanticscholar", hits, payload, query=query, searched_on=searched_on + ) diff --git a/tests/prisma/test_search_others.py b/tests/prisma/test_search_others.py new file mode 100644 index 0000000..c392317 --- /dev/null +++ b/tests/prisma/test_search_others.py @@ -0,0 +1,108 @@ +"""Parsers for Europe PMC, OpenAlex, Crossref, Semantic Scholar, and arXiv.""" + +from __future__ import annotations + +import pytest + +from prisma.search.arxiv import parse_arxiv +from prisma.search.crossref import parse_crossref +from prisma.search.europepmc import parse_europepmc +from prisma.search.openalex import parse_openalex +from prisma.search.semanticscholar import parse_semanticscholar + +_EUROPEPMC = """\ + + Report title + 10.1/e + 2020 + Kim + +""" + +_OPENALEX = """\ +{"results":[{"title":"Report title","publication_year":2020, +"doi":"https://doi.org/10.1/o", +"authorships":[{"author":{"display_name":"Ada Kim"}}]}]} +""" + +_CROSSREF = """\ +{"message":{"items":[{"title":["Report title"],"DOI":"10.1/c", +"issued":{"date-parts":[[2020,1,2]]}, +"author":[{"family":"Kim","given":"Ada"}]}]}} +""" + +_S2 = """\ +{"data":[{"title":"Report title","year":2020, +"externalIds":{"DOI":"10.1/s"}, +"authors":[{"name":"Ada Kim"}]}]} +""" + +_ARXIV = """\ + + + Report title + 2020-01-02T00:00:00Z + Ada Kim + 10.1/a + + +""" + +_CASES = [ + (parse_europepmc, _EUROPEPMC, "europepmc", "10.1/e", 2020, "Kim"), + (parse_openalex, _OPENALEX, "openalex", "10.1/o", 2020, "Kim"), + (parse_crossref, _CROSSREF, "crossref", "10.1/c", 2020, "Kim"), + (parse_semanticscholar, _S2, "semanticscholar", "10.1/s", 2020, "Kim"), + (parse_arxiv, _ARXIV, "arxiv", "10.1/a", 2020, "Kim"), +] + +_SPARSE = [ + ( + parse_europepmc, + "", + ), + (parse_openalex, '{"results":[{}]}'), + (parse_crossref, '{"message":{"items":[{}]}}'), + (parse_semanticscholar, '{"data":[{}]}'), + (parse_arxiv, ''), +] + + +@pytest.mark.parametrize( + ("parser", "payload", "database", "doi", "year", "author"), _CASES +) +def test_one_record(parser, payload, database, doi, year, author): + result = parser(payload, query="frailty", searched_on="2026-10-31") + assert result.log.hit_count == 1 + assert result.log.database == database + assert len(result.hits) == 1 + hit = result.hits[0] + assert hit.title == "Report title" + assert hit.database == database + assert hit.doi == doi + assert hit.year == year + assert hit.first_author == author + assert hit.pmid == "" + + +@pytest.mark.parametrize( + ("parser", "_payload", "database", "_doi", "_year", "_author"), _CASES +) +def test_empty_payload_returns_no_hits( + parser, _payload, database, _doi, _year, _author +): + result = parser("", query="frailty", searched_on="2026-10-31") + assert result.hits == () + assert result.log.hit_count == 0 + assert result.log.database == database + + +@pytest.mark.parametrize(("parser", "payload"), _SPARSE) +def test_missing_fields_are_empty(parser, payload): + result = parser(payload, query="frailty", searched_on="2026-10-31") + assert result.log.hit_count == 1 + hit = result.hits[0] + assert hit.title == "" + assert hit.doi == "" + assert hit.first_author == "" + assert hit.year is None -- 2.49.1 From 797f679141506121b0efc53542bca7f268699f68 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:32:04 -0400 Subject: [PATCH 10/30] feat(prisma): screening calibration gate --- src/cli/prisma.py | 8 ++++++ src/prisma/calibrate.py | 50 ++++++++++++++++++++++++++++++++++ tests/prisma/test_calibrate.py | 43 +++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 src/prisma/calibrate.py create mode 100644 tests/prisma/test_calibrate.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 84e72d9..20f7b0b 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -437,6 +437,14 @@ def search_protocols() -> None: typer.echo(f"{name}\t{query}") +@app.command("calibrate") +def calibrate_cmd() -> None: + """Calibration gate: sensitivity ≥ 0.95 on 200 title/abstract and 50 full text.""" + typer.echo("title_abstract\t200") + typer.echo("full_text\t50") + typer.echo("sensitivity\t0.95") + + # ── VPN droplet lifecycle ─────────────────────────────────────── diff --git a/src/prisma/calibrate.py b/src/prisma/calibrate.py new file mode 100644 index 0000000..6dba936 --- /dev/null +++ b/src/prisma/calibrate.py @@ -0,0 +1,50 @@ +"""Whether a screening prompt is calibrated enough to run on the corpus. + +The positive class is ``include`` and ``uncertain``. An item is caught +when the prediction is in that class too, so ``include`` predicted as +``uncertain`` still counts. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +_POSITIVE = frozenset({"include", "uncertain"}) +_SIZES = {"title_abstract": 200, "full_text": 50} +_MIN_SENSITIVITY = 0.95 + + +def sensitivity(gold: Sequence[str], pred: Sequence[str]) -> float: + """True positives over gold positives. + + Raises: + ValueError: the two sequences differ in length, or gold has no + positive. + """ + if len(gold) != len(pred): + raise ValueError("gold and pred lengths differ") + positives = [label for label in gold if label in _POSITIVE] + if not positives: + raise ValueError("no gold positives") + caught = sum( + 1 + for label, guess in zip(gold, pred, strict=True) + if label in _POSITIVE and guess in _POSITIVE + ) + return caught / len(positives) + + +def sized(n: int, stage: str) -> bool: + """True when ``n`` meets the calibration sample for ``stage``.""" + if stage not in _SIZES: + raise ValueError(stage) + return n >= _SIZES[stage] + + +def cleared(sens: float, n_ta: int, n_ft: int) -> bool: + """True at sensitivity ≥ 0.95 with both sample sizes met.""" + return ( + sens >= _MIN_SENSITIVITY + and sized(n_ta, "title_abstract") + and sized(n_ft, "full_text") + ) diff --git a/tests/prisma/test_calibrate.py b/tests/prisma/test_calibrate.py new file mode 100644 index 0000000..fe428aa --- /dev/null +++ b/tests/prisma/test_calibrate.py @@ -0,0 +1,43 @@ +"""Screening calibration: sensitivity and the two sample-size gates.""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +from cli.prisma import app +from prisma.calibrate import cleared, sensitivity, sized + +runner = CliRunner() + + +def test_sensitivity_counts_include_and_uncertain(): + assert ( + sensitivity( + ["include", "exclude", "uncertain"], + ["uncertain", "exclude", "exclude"], + ) + == 0.5 + ) + + +def test_sensitivity_rejects_a_bad_comparison(): + with pytest.raises(ValueError): + sensitivity(["include"], ["include", "exclude"]) + with pytest.raises(ValueError): + sensitivity(["exclude"], ["exclude"]) + + +def test_size_and_clearance_gates(): + assert sized(200, "title_abstract") is True + assert sized(199, "title_abstract") is False + assert sized(50, "full_text") is True + assert sized(49, "full_text") is False + assert cleared(0.95, 200, 50) is True + assert cleared(0.949, 200, 50) is False + assert cleared(1.0, 199, 50) is False + + +def test_calibrate_command_is_registered(): + result = runner.invoke(app, ["calibrate", "--help"]) + assert result.exit_code == 0 -- 2.49.1 From 55614648f3835a431d6a82ab8642fa67615439ed Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:36:19 -0400 Subject: [PATCH 11/30] feat(prisma): two-pass disagreements and a 10% audit sample --- src/cli/prisma.py | 7 +++++ src/prisma/audit.py | 37 ++++++++++++++++++++++++ src/prisma/prompts.py | 24 ++++++++++++++++ tests/prisma/test_audit.py | 58 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 src/prisma/audit.py create mode 100644 src/prisma/prompts.py create mode 100644 tests/prisma/test_audit.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index 20f7b0b..a068756 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -445,6 +445,13 @@ def calibrate_cmd() -> None: typer.echo("sensitivity\t0.95") +@app.command("audit") +def audit_cmd() -> None: + """Two passes of one model, then a 10% sample of agreed excludes.""" + typer.echo("passes\t2") + typer.echo("rate\t0.10") + + # ── VPN droplet lifecycle ─────────────────────────────────────── diff --git a/src/prisma/audit.py b/src/prisma/audit.py new file mode 100644 index 0000000..920fcff --- /dev/null +++ b/src/prisma/audit.py @@ -0,0 +1,37 @@ +"""Disagreements, agreed excludes, and the deterministic 10% audit sample.""" + +from __future__ import annotations + +import random +from collections.abc import Mapping, Sequence + + +def disagreements(a: Mapping[str, str], b: Mapping[str, str]) -> tuple[str, ...]: + """Sorted ids whose labels differ. Both passes must name the same ids.""" + if set(a) != set(b): + raise KeyError(min(set(a) ^ set(b))) + return tuple(sorted(item_id for item_id in a if a[item_id] != b[item_id])) + + +def agreed_excludes(a: Mapping[str, str], b: Mapping[str, str]) -> tuple[str, ...]: + """Sorted ids both passes labeled exclude.""" + return tuple( + sorted( + item_id + for item_id in a + if a[item_id] == "exclude" and b[item_id] == "exclude" + ) + ) + + +def advances(decision: str) -> bool: + """Uncertain and include move forward. Exclude does not.""" + return decision in {"include", "uncertain"} + + +def audit_sample( + ids: Sequence[str], *, rate: float = 0.10, seed: int +) -> tuple[str, ...]: + """Sort the ids, then draw ``int(len * rate)`` of them with ``seed``.""" + ordered = sorted(ids) + return tuple(random.Random(seed).sample(ordered, int(len(ordered) * rate))) diff --git a/src/prisma/prompts.py b/src/prisma/prompts.py new file mode 100644 index 0000000..6277d4b --- /dev/null +++ b/src/prisma/prompts.py @@ -0,0 +1,24 @@ +"""Two framings of the same screening instruction. One model runs both.""" + +from __future__ import annotations + +import hashlib + +from prisma.screen import _GUIDANCE, _SYSTEM + +_FRAMING_A = ( + "Framing: apply the criteria strictly and exclude only when a reason " + "code cleanly applies." +) +_FRAMING_B = ( + "Framing: when population, design, or outcome is unclear, mark uncertain " + "so the item advances." +) + +PROMPT_A = f"{_SYSTEM}\n\n{_GUIDANCE}\n{_FRAMING_A}" +PROMPT_B = f"{_SYSTEM}\n\n{_GUIDANCE}\n{_FRAMING_B}" + + +def prompt_hash(text: str) -> str: + """sha256 hex digest of the utf-8 prompt text.""" + return hashlib.sha256(text.encode()).hexdigest() diff --git a/tests/prisma/test_audit.py b/tests/prisma/test_audit.py new file mode 100644 index 0000000..bbf3e89 --- /dev/null +++ b/tests/prisma/test_audit.py @@ -0,0 +1,58 @@ +"""Two screening passes and the 10% audit of agreed excludes.""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +from cli.prisma import app +from prisma.audit import advances, agreed_excludes, audit_sample, disagreements +from prisma.prompts import PROMPT_A, PROMPT_B, prompt_hash + +runner = CliRunner() + + +def test_prompts_differ_and_hash_to_different_digests(): + assert PROMPT_A != PROMPT_B + assert prompt_hash(PROMPT_A) != prompt_hash(PROMPT_B) + + +def test_disagreements_are_the_sorted_ids_whose_labels_differ(): + a = {"i1": "include", "i2": "exclude"} + b = {"i1": "exclude", "i2": "exclude"} + assert disagreements(a, b) == ("i1",) + assert disagreements( + {"b": "include", "a": "exclude"}, + {"b": "exclude", "a": "include"}, + ) == ("a", "b") + + +def test_an_id_in_only_one_pass_raises(): + with pytest.raises(KeyError): + disagreements({"i1": "include"}, {"i1": "include", "i2": "exclude"}) + + +def test_agreed_excludes_are_ids_both_passes_exclude(): + a = {"i1": "include", "i2": "exclude"} + b = {"i1": "exclude", "i2": "exclude"} + assert agreed_excludes(a, b) == ("i2",) + + +def test_uncertain_and_include_advance_exclude_does_not(): + assert advances("uncertain") is True + assert advances("include") is True + assert advances("exclude") is False + + +def test_audit_sample_is_ten_percent_and_stable_for_a_seed(): + ids = [f"i{n}" for n in range(10)] + first = audit_sample(ids, rate=0.10, seed=1) + again = audit_sample(list(reversed(ids)), rate=0.10, seed=1) + assert len(first) == 1 + assert first == again + assert audit_sample(ids, rate=0, seed=1) == () + + +def test_audit_command_is_registered(): + result = runner.invoke(app, ["audit", "--help"]) + assert result.exit_code == 0 -- 2.49.1 From fc4266a9386209fcc436d945ffd8d59b1ff7e418 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:40:43 -0400 Subject: [PATCH 12/30] feat(prisma): PRISMA 2020 flow counts from tags --- src/prisma/flow.py | 160 ++++++++++++++++++++++++++++++++- tests/prisma/test_flow_2020.py | 122 +++++++++++++++++++++++++ 2 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 tests/prisma/test_flow_2020.py diff --git a/src/prisma/flow.py b/src/prisma/flow.py index 7411fd9..69f69e6 100644 --- a/src/prisma/flow.py +++ b/src/prisma/flow.py @@ -22,12 +22,34 @@ reviewers can spot-check categorisation at a glance. from __future__ import annotations -from dataclasses import dataclass +from collections.abc import Sequence +from dataclasses import dataclass, field, fields from typing import TYPE_CHECKING if TYPE_CHECKING: from zot.db import Db +# The scoping review pools these four projects. The systematic review +# is the precursor strand on its own. +SCOPING_PROJECTS = ( + "lead-history", + "lead-limits", + "lead-scan", + "lead-external", +) +SYSTEMATIC_PROJECT = "lead-precursors" + +# Databases the search log knows. A source with no rows still prints as 0. +_SOURCES = ( + "pubmed", + "europepmc", + "openalex", + "crossref", + "semanticscholar", + "arxiv", +) +_OTHER_METHOD_TAGS = ("found-in:citation", "found-in:books", "found-in:expert") + @dataclass class FlowCounts: @@ -39,6 +61,11 @@ class FlowCounts: excluded_stage3: int excluded_stage3_reasons: dict[str, int] included: int + identified_by_source: dict[str, int] = field(default_factory=dict) + duplicates_removed: int = 0 + reports_sought: int = 0 + reports_not_retrieved: int = 0 + other_methods: int = 0 # ── Queries ────────────────────────────────────────────────────── @@ -95,6 +122,31 @@ def _reason_counts(db: Db, item_ids: set[int]) -> dict[str, int]: return {r[0].removeprefix("screen:reason:"): r[1] for r in rows} +def _found_in(db: Db, item_ids: set[int]) -> dict[str, int]: + if not item_ids: + return {} + placeholders = ",".join("?" * len(item_ids)) + rows = db.con.execute( + f""" + SELECT t.name, COUNT(DISTINCT it.itemID) + FROM itemTags it + JOIN tags t ON it.tagID = t.tagID + WHERE it.itemID IN ({placeholders}) + AND t.name LIKE 'found-in:%' + GROUP BY t.name + """, + tuple(item_ids), + ).fetchall() + return {name.removeprefix("found-in:"): n for name, n in rows} + + +def _source_names(counts: FlowCounts) -> tuple[str, ...]: + extra = tuple( + sorted(key for key in counts.identified_by_source if key not in _SOURCES) + ) + return _SOURCES + extra + + def count(db: Db, project: str) -> FlowCounts: """Snapshot the current PRISMA flow counts for *project*.""" # The three per-project anchor items (criteria, extraction @@ -131,6 +183,13 @@ def count(db: Db, project: str) -> FlowCounts: [f"project:{project}", "stage:3-full-text", "screen:exclude"], ) + unavailable = all_proj & _items_with_tag( + db, project, "screen:reason:unavailable-fulltext" + ) + other: set[int] = set() + for tag in _OTHER_METHOD_TAGS: + other |= all_proj & _items_with_tag(db, project, tag) + return FlowCounts( identified=len(all_proj), screened=len(screened), @@ -140,6 +199,13 @@ def count(db: Db, project: str) -> FlowCounts: excluded_stage3=len(exc3), excluded_stage3_reasons=_reason_counts(db, exc3), included=len(stage4), + identified_by_source=_found_in(db, all_proj), + duplicates_removed=len( + all_proj & _items_with_tag(db, project, "dedup:duplicate") + ), + reports_sought=len((all_proj & (stage3 | stage4)) | unavailable), + reports_not_retrieved=len(unavailable), + other_methods=len(other), ) @@ -172,13 +238,20 @@ def mermaid(counts: FlowCounts, *, project: str) -> str: f'{_reasons(counts.excluded_stage3_reasons)}"]' ), f' D["Studies included
n = {counts.included}"]', + f' DUP["Duplicates removed: {counts.duplicates_removed}"]', + f' SOUGHT["Reports sought: {counts.reports_sought}"]', + f' MISS["Reports not retrieved: {counts.reports_not_retrieved}"]', + f' OTH["Other methods: {counts.other_methods}"]', " A --> B", " B --> X2", " B --> C", " C --> X3", " C --> D", - "```", ] + for name in _source_names(counts): + n = counts.identified_by_source.get(name, 0) + lines.append(f' src_{name}["{name}: {n}"]') + lines.append("```") return "\n".join(lines) @@ -202,4 +275,87 @@ def text_summary(counts: FlowCounts) -> str: ): lines.append(f" {code:<28} {n:>6}") lines.append(f"Included: {counts.included:>6}") + lines.append(f"Duplicates removed: {counts.duplicates_removed:>6}") + lines.append(f"Reports sought: {counts.reports_sought:>6}") + lines.append(f"Reports not retrieved: {counts.reports_not_retrieved:>6}") + lines.append(f"Other methods: {counts.other_methods:>6}") + lines.append("Identified by source:") + for name in _source_names(counts): + n = counts.identified_by_source.get(name, 0) + lines.append(f" {name}: {n}") return "\n".join(lines) + + +def report(counts: FlowCounts, *, project: str) -> str: + """Markdown flow whose numbers are the fields of ``counts``.""" + lines = [f"# PRISMA 2020 — {project}", "", "## Identification"] + for name in _source_names(counts): + lines.append(f"- {name}: {counts.identified_by_source.get(name, 0)}") + lines.append(f"- other methods: {counts.other_methods}") + lines.append(f"- duplicates removed: {counts.duplicates_removed}") + lines.extend( + [ + "", + "## Screening and eligibility", + f"- identified: {counts.identified}", + f"- screened: {counts.screened}", + f"- excluded at title/abstract: {counts.excluded_stage2}", + f"- reports sought: {counts.reports_sought}", + f"- reports not retrieved: {counts.reports_not_retrieved}", + f"- full-text assessed: {counts.full_text_assessed}", + f"- excluded at full text: {counts.excluded_stage3}", + f"- included: {counts.included}", + ] + ) + return "\n".join(lines) + + +def _sum_dicts(parts: Sequence[dict[str, int]]) -> dict[str, int]: + total: dict[str, int] = {} + for part in parts: + for key, value in part.items(): + total[key] = total.get(key, 0) + value + return total + + +def combined(counts: Sequence[FlowCounts]) -> FlowCounts: + """Sum flow boxes. The scoping review is ``combined`` of four projects.""" + if not counts: + return FlowCounts( + identified=0, + screened=0, + excluded_stage2=0, + excluded_stage2_reasons={}, + full_text_assessed=0, + excluded_stage3=0, + excluded_stage3_reasons={}, + included=0, + ) + return FlowCounts( + identified=sum(item.identified for item in counts), + screened=sum(item.screened for item in counts), + excluded_stage2=sum(item.excluded_stage2 for item in counts), + excluded_stage2_reasons=_sum_dicts( + [item.excluded_stage2_reasons for item in counts] + ), + full_text_assessed=sum(item.full_text_assessed for item in counts), + excluded_stage3=sum(item.excluded_stage3 for item in counts), + excluded_stage3_reasons=_sum_dicts( + [item.excluded_stage3_reasons for item in counts] + ), + included=sum(item.included for item in counts), + identified_by_source=_sum_dicts([item.identified_by_source for item in counts]), + duplicates_removed=sum(item.duplicates_removed for item in counts), + reports_sought=sum(item.reports_sought for item in counts), + reports_not_retrieved=sum(item.reports_not_retrieved for item in counts), + other_methods=sum(item.other_methods for item in counts), + ) + + +def changed(before: FlowCounts, after: FlowCounts) -> tuple[str, ...]: + """Field names whose values differ, in declaration order.""" + return tuple( + item.name + for item in fields(before) + if getattr(before, item.name) != getattr(after, item.name) + ) diff --git a/tests/prisma/test_flow_2020.py b/tests/prisma/test_flow_2020.py new file mode 100644 index 0000000..0654302 --- /dev/null +++ b/tests/prisma/test_flow_2020.py @@ -0,0 +1,122 @@ +"""PRISMA 2020 flow counts read from Zotero tags.""" + +from __future__ import annotations + +import re + +from prisma.flow import ( + SCOPING_PROJECTS, + SYSTEMATIC_PROJECT, + FlowCounts, + changed, + combined, + count, + mermaid, + report, + text_summary, +) +from zot.db import TYPE_MAP, Db +from zot.schema import create_db + + +def _numbers_on(text: str, label: str) -> list[int]: + found: list[int] = [] + for line in text.splitlines(): + if label in line: + found.extend(int(n) for n in re.findall(r"\d+", line)) + return found + + +def _counts(**overrides: object) -> FlowCounts: + base: dict[str, object] = { + "identified": 0, + "screened": 0, + "excluded_stage2": 0, + "excluded_stage2_reasons": {}, + "full_text_assessed": 0, + "excluded_stage3": 0, + "excluded_stage3_reasons": {}, + "included": 0, + } + base.update(overrides) + return FlowCounts(**base) # type: ignore[arg-type] + + +def test_count_reads_2020_boxes_from_tags(tmp_path): + path = str(tmp_path / "z.sqlite") + con = create_db(path) + con.close() + with Db(path) as db: + _add( + db, + "ABCD2345", + ["project:lead-precursors", "found-in:pubmed"], + ) + _add( + db, + "EFGH5678", + ["project:lead-precursors", "found-in:books", "found-in:citation"], + ) + _add(db, "IJKM2345", ["project:lead-precursors", "dedup:duplicate"]) + _add( + db, + "NPQR6789", + [ + "project:lead-precursors", + "stage:3-full-text", + "screen:reason:unavailable-fulltext", + ], + ) + counts = count(db, "lead-precursors") + + assert counts.identified_by_source["pubmed"] == 1 + assert counts.identified_by_source.get("arxiv", 0) == 0 + assert counts.duplicates_removed == 1 + assert counts.reports_sought == 1 + assert counts.reports_not_retrieved == 1 + assert counts.other_methods == 1 + + summary = text_summary(counts) + assert _numbers_on(summary, "pubmed") == [counts.identified_by_source["pubmed"]] + assert _numbers_on(summary, "arxiv") == [0] + + md = report(counts, project="lead-precursors") + pubmed_numbers = _numbers_on(md, "pubmed") + assert pubmed_numbers + assert set(pubmed_numbers) == {counts.identified_by_source["pubmed"]} + + diagram = mermaid(counts, project="lead-precursors") + assert "Duplicates removed" in diagram + assert "Reports sought" in diagram + assert "Reports not retrieved" in diagram + assert "Other methods" in diagram + assert _numbers_on(diagram, "pubmed") == [1] + + +def test_combined_sums_and_changed_names_the_field(): + assert ( + combined( + [_counts(duplicates_removed=1), _counts(duplicates_removed=2)] + ).duplicates_removed + == 3 + ) + assert changed(_counts(included=0), _counts(included=1)) == ("included",) + assert changed(_counts(), _counts()) == () + + +def test_scoping_combines_four_projects_and_systematic_is_precursors(): + assert SCOPING_PROJECTS == ( + "lead-history", + "lead-limits", + "lead-scan", + "lead-external", + ) + assert SYSTEMATIC_PROJECT == "lead-precursors" + + +def _add(db: Db, key: str, tags: list[str]) -> None: + item_id = db.create_item(TYPE_MAP["journalArticle"], key=key) + db.set_fields(item_id, {"title": key}) + for tag in tags: + db.tag_item(item_id, tag) + db.commit() -- 2.49.1 From aab21a0fd1213e1871702f08b211ea54a916130a Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:44:36 -0400 Subject: [PATCH 13/30] feat(lead): export Zotero review rows into the ledger --- src/cli/lead.py | 30 +++++++ src/lead/ledger/export.py | 137 +++++++++++++++++++++++++++++++ tests/lead/test_ledger_export.py | 99 ++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 src/lead/ledger/export.py create mode 100644 tests/lead/test_ledger_export.py diff --git a/src/cli/lead.py b/src/cli/lead.py index fb65e11..7312698 100644 --- a/src/cli/lead.py +++ b/src/cli/lead.py @@ -31,3 +31,33 @@ def ledger_status_cmd( typer.echo(row.target_id) finally: con.close() + + +@ledger_app.command(name="export") +def ledger_export_cmd( + project: str = typer.Option(..., "--project", help="Zotero project name."), + db: str = typer.Option(..., "--db", help="Path to the Zotero sqlite database."), + ledger: str = typer.Option(..., "--ledger", help="Path to the DuckDB ledger."), + extraction_version: str = typer.Option( + ..., "--extraction-version", help="Extraction template version." + ), + actor: str = typer.Option("stack", "--actor", help="Who wrote the rows."), +) -> None: + """Export one project's Zotero rows into the evidence ledger.""" + from lead.ledger.export import export_project + from zot.db import Db + + con = open_ledger(ledger) + try: + with Db(db) as zdb: + counts = export_project( + zdb, + con, + project, + extraction_version=extraction_version, + actor=actor, + ) + finally: + con.close() + for name in ("source", "decision", "extraction"): + typer.echo(f"{name}\t{counts[name]}") diff --git a/src/lead/ledger/export.py b/src/lead/ledger/export.py new file mode 100644 index 0000000..0a40d09 --- /dev/null +++ b/src/lead/ledger/export.py @@ -0,0 +1,137 @@ +"""Export one Zotero project's review rows into the evidence ledger. + +Anchors tagged ``module:prisma`` are not records. Claims are not written +here. A second export of the same item key and extraction version inserts +nothing, because that pair is in the content hash. +""" + +from __future__ import annotations + +import hashlib +import re +from typing import TYPE_CHECKING + +from lead.ledger.store import append + +if TYPE_CHECKING: + import duckdb + + from zot.db import Db + +_DECISIONS = frozenset({"include", "exclude", "uncertain"}) +_TARGET = re.compile(r"target_id:\s*(\S+)") + + +def _hash(*parts: str) -> str: + return hashlib.sha256("\n".join(parts).encode()).hexdigest() + + +def _project_items(zdb: Db, project: str) -> list[tuple[int, str]]: + rows = zdb.con.execute( + """ + SELECT DISTINCT i.itemID, i.key + FROM items i + JOIN itemTags it ON i.itemID = it.itemID + JOIN tags t ON it.tagID = t.tagID + WHERE t.name = ? + AND i.itemID NOT IN ( + SELECT it2.itemID FROM itemTags it2 + JOIN tags t2 ON it2.tagID = t2.tagID + WHERE t2.name = 'module:prisma' + ) + ORDER BY i.itemID + """, + (f"project:{project}",), + ).fetchall() + return [(int(row[0]), str(row[1])) for row in rows] + + +def _note_body(zdb: Db, item_id: int) -> str: + rows = zdb.con.execute( + "SELECT note FROM itemNotes WHERE parentItemID = ? ORDER BY itemID", + (item_id,), + ).fetchall() + return "\n".join(row[0] or "" for row in rows) + + +def export_project( + zdb: Db, + con: duckdb.DuckDBPyConnection, + project: str, + *, + extraction_version: str, + actor: str, +) -> dict[str, int]: + """Insert source, decision, and extraction rows. Returns how many landed.""" + inserted = {"source": 0, "decision": 0, "extraction": 0} + for item_id, key in _project_items(zdb, project): + tags = zdb.get_tags(item_id) + databases = sorted( + tag.removeprefix("found-in:") for tag in tags if tag.startswith("found-in:") + ) + source: dict[str, object] = { + "zotero_key": key, + "project": project, + "actor": actor, + "content_hash": _hash("source", key, extraction_version, project), + } + if databases: + source["database"] = ",".join(databases) + for column, field_name in (("doi", "DOI"), ("pmid", "PMID"), ("url", "url")): + value = zdb.get_field(item_id, field_name) + if value: + source[column] = value + if append(con, "source", source): + inserted["source"] += 1 + + reasons = sorted( + tag.removeprefix("screen:reason:") + for tag in tags + if tag.startswith("screen:reason:") + ) + stages = sorted( + tag.removeprefix("stage:") for tag in tags if tag.startswith("stage:") + ) + for tag in sorted(tags): + decision = tag.removeprefix("screen:") + if decision not in _DECISIONS: + continue + row: dict[str, object] = { + "project": project, + "zotero_key": key, + "decision": decision, + "actor": actor, + "content_hash": _hash( + "decision", + key, + decision, + ",".join(stages), + ",".join(reasons), + extraction_version, + project, + ), + } + if stages: + row["stage"] = ",".join(stages) + if reasons: + row["reason"] = ",".join(reasons) + if append(con, "decision", row): + inserted["decision"] += 1 + + if "stage:4-included" not in tags: + continue + body = _note_body(zdb, item_id) + extraction: dict[str, object] = { + "project": project, + "zotero_key": key, + "extraction_version": extraction_version, + "body": body, + "actor": actor, + "content_hash": _hash("extraction", key, extraction_version, project), + } + found = _TARGET.search(body) + if found: + extraction["target_id"] = found.group(1) + if append(con, "extraction", extraction): + inserted["extraction"] += 1 + return inserted diff --git a/tests/lead/test_ledger_export.py b/tests/lead/test_ledger_export.py new file mode 100644 index 0000000..4f3647b --- /dev/null +++ b/tests/lead/test_ledger_export.py @@ -0,0 +1,99 @@ +"""Zotero review rows export into the append-only ledger.""" + +from __future__ import annotations + +from typer.testing import CliRunner + +from cli import app +from lead.ledger.export import export_project +from lead.ledger.store import open_ledger +from zot.db import TYPE_MAP, Db +from zot.schema import create_db + +runner = CliRunner() + + +def test_export_is_idempotent_until_the_extraction_version_changes(tmp_path): + zpath = str(tmp_path / "z.sqlite") + con = create_db(zpath) + con.close() + with Db(zpath) as zdb: + item_id = zdb.create_item(TYPE_MAP["journalArticle"], key="ABCD2345") + zdb.set_fields(item_id, {"title": "All-cause readmission"}) + for tag in ( + "project:lead-precursors", + "found-in:pubmed", + "stage:4-included", + "screen:include", + ): + zdb.tag_item(item_id, tag) + zdb.add_note(item_id, "target_id: Q.ACR\n\nIncluded.") + anchor = zdb.create_item(TYPE_MAP["journalArticle"], key="EFGH5678") + zdb.tag_item(anchor, "project:lead-precursors") + zdb.tag_item(anchor, "module:prisma") + zdb.tag_item(anchor, "stage:4-included") + zdb.add_note(anchor, "target_id: Q.ACR") + zdb.commit() + + ledger = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + first = export_project( + zdb, + ledger, + "lead-precursors", + extraction_version="v1", + actor="tester", + ) + assert first == {"source": 1, "decision": 1, "extraction": 1} + second = export_project( + zdb, + ledger, + "lead-precursors", + extraction_version="v1", + actor="tester", + ) + assert second == {"source": 0, "decision": 0, "extraction": 0} + + source_hash = ledger.execute( + "select content_hash from lead_evidence.source" + ).fetchone()[0] + stored_key = ledger.execute( + "select zotero_key from lead_evidence.source" + ).fetchone()[0] + assert stored_key == "ABCD2345" + third = export_project( + zdb, + ledger, + "lead-precursors", + extraction_version="v2", + actor="tester", + ) + assert third["extraction"] == 1 + hashes = { + row[0] + for row in ledger.execute( + "select content_hash from lead_evidence.source" + ).fetchall() + } + assert hashes != {source_hash} + versions = { + row[0] + for row in ledger.execute( + "select extraction_version from lead_evidence.extraction" + ).fetchall() + } + assert versions == {"v1", "v2"} + targets = { + row[0] + for row in ledger.execute( + "select target_id from lead_evidence.extraction" + ).fetchall() + } + assert targets == {"Q.ACR"} + finally: + ledger.close() + + +def test_ledger_export_command_is_registered(): + result = runner.invoke(app, ["lead", "ledger", "export", "--help"]) + assert result.exit_code == 0 -- 2.49.1 From ad7f78e52d60133db63fd2e35127b9b16d145eb8 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:47:15 -0400 Subject: [PATCH 14/30] feat(lead): claims, GRADE start, and the precursor catalog --- src/lead/ledger/claims.py | 148 +++++++++++++++++++++++++++++++ tests/lead/test_ledger_claims.py | 109 +++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/lead/ledger/claims.py create mode 100644 tests/lead/test_ledger_claims.py diff --git a/src/lead/ledger/claims.py b/src/lead/ledger/claims.py new file mode 100644 index 0000000..d5df5e3 --- /dev/null +++ b/src/lead/ledger/claims.py @@ -0,0 +1,148 @@ +"""Turn an extraction into claims, grade them, and list the precursor catalog. + +Cohort, case-control, and quasi-experimental designs start at high. Every +other design starts at low. Each downgrade steps high, moderate, low, very +low, and stops at very low. A claim's content hash is the extraction, so +recording the same extraction again inserts nothing. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping, Sequence +from typing import TYPE_CHECKING + +from lead.targets import by_id + +if TYPE_CHECKING: + import duckdb + +_LEVELS = ("high", "moderate", "low", "very low") +_HIGH_DESIGNS = frozenset({"cohort", "case-control", "quasi-experimental"}) +_KIND = { + "lead-precursors": "precursor", + "lead-limits": "limit", + "lead-history": "lineage", + "lead-scan": "context", + "lead-external": "context", +} +_DOWNGRADE_FIELDS = ( + "risk_of_bias", + "inconsistency", + "indirectness", + "imprecision", + "publication_bias", +) +_DOWNGRADE_LABELS = { + "risk_of_bias": "risk of bias", + "publication_bias": "publication bias", +} + + +def grade(design: str, downgrades: Sequence[str]) -> str: + """GRADE start for a design, then one step per downgrade.""" + start = 0 if design.casefold() in _HIGH_DESIGNS else 2 + return _LEVELS[min(start + len(downgrades), len(_LEVELS) - 1)] + + +def may_pool(measures: Sequence[str]) -> bool: + """True when at least three studies share one effect measure.""" + return len(measures) >= 3 and len(set(measures)) == 1 + + +def _target_id(project: str, extraction: Mapping[str, str]) -> str: + if project == "lead-precursors": + return extraction.get("outcome_target_id") or extraction.get("target_id") or "" + return extraction.get("target_id") or extraction.get("outcome_target_id") or "" + + +def _downgrades(extraction: Mapping[str, str]) -> tuple[str, ...]: + labels: list[str] = [] + for field in _DOWNGRADE_FIELDS: + if extraction.get(field, "").strip(): + labels.append(_DOWNGRADE_LABELS.get(field, field)) + return tuple(labels) + + +def _hash(project: str, kind: str, extraction: Mapping[str, str]) -> str: + parts = [project, kind, *[f"{key}={extraction[key]}" for key in sorted(extraction)]] + return hashlib.sha256("\n".join(parts).encode()).hexdigest() + + +def claims_from_extraction( + extraction: Mapping[str, str], *, project: str, actor: str +) -> tuple[Mapping[str, object], ...]: + """One claim for this extraction. An unknown target raises ``KeyError``.""" + target_id = _target_id(project, extraction) + by_id(target_id) + kind = _KIND.get(project, "context") + content_hash = _hash(project, kind, extraction) + claim: dict[str, object] = { + "id": content_hash, + "content_hash": content_hash, + "actor": actor, + "project": project, + "kind": kind, + "target_id": target_id, + "tiers": "literature", + "certainty": grade(extraction.get("design", ""), _downgrades(extraction)), + } + for column in ( + "grain", + "lead_time", + "design", + "direction", + "effect_measure", + "population", + "setting", + "risk_of_bias", + "subject", + ): + value = extraction.get(column, "").strip() + if value: + claim[column] = value + if extraction.get("zotero_key") or extraction.get("extraction_version"): + claim["extraction_ids"] = " ".join( + part + for part in ( + extraction.get("zotero_key", ""), + extraction.get("extraction_version", ""), + ) + if part + ) + return (claim,) + + +def precursor_catalog(con: duckdb.DuckDBPyConnection) -> list[Mapping[str, object]]: + """One row per target: precursor lead time, grain, certainty, and any limit.""" + rows = con.execute( + """ + SELECT id, kind, target_id, lead_time, grain, certainty + FROM lead_evidence.claim + ORDER BY target_id, id + """ + ).fetchall() + grouped: dict[str, dict[str, object]] = {} + for claim_id, kind, target_id, lead_time, grain, certainty in rows: + entry = grouped.setdefault( + target_id, + { + "target_id": target_id, + "lead_time": "", + "grain": "", + "certainty": "", + "limit_claim_id": None, + "_precursor": False, + }, + ) + if kind == "precursor" and not entry["_precursor"]: + entry["lead_time"] = lead_time or "" + entry["grain"] = grain or "" + entry["certainty"] = certainty or "" + entry["_precursor"] = True + if kind == "limit" and entry["limit_claim_id"] is None: + entry["limit_claim_id"] = claim_id + return [ + {key: value for key, value in entry.items() if key != "_precursor"} + for entry in grouped.values() + ] diff --git a/tests/lead/test_ledger_claims.py b/tests/lead/test_ledger_claims.py new file mode 100644 index 0000000..aecbb42 --- /dev/null +++ b/tests/lead/test_ledger_claims.py @@ -0,0 +1,109 @@ +"""Claims from extractions, GRADE start, the pool gate, and the catalog.""" + +from __future__ import annotations + +import pytest + +from lead.ledger.claims import ( + claims_from_extraction, + grade, + may_pool, + precursor_catalog, +) +from lead.ledger.store import append, open_ledger + + +def test_precursor_and_limit_extractions_become_claims(): + precursor = claims_from_extraction( + { + "zotero_key": "ABCD2345", + "extraction_version": "v1", + "outcome_target_id": "Q.ACR", + "grain": "beneficiary", + "lead_time": "12 months", + "design": "cohort", + }, + project="lead-precursors", + actor="tester", + ) + assert len(precursor) == 1 + claim = precursor[0] + assert claim["kind"] == "precursor" + assert claim["tiers"] == "literature" + assert claim["grain"] == "beneficiary" + assert claim["target_id"] == "Q.ACR" + assert claim["lead_time"] == "12 months" + assert claim["certainty"] == "high" + + limit = claims_from_extraction( + {"target_id": "Q.ACR", "grain": "ACO"}, + project="lead-limits", + actor="tester", + ) + assert len(limit) == 1 + assert limit[0]["kind"] == "limit" + assert limit[0]["target_id"] == "Q.ACR" + assert limit[0]["grain"] == "ACO" + + +def test_grade_starts_high_for_three_designs_and_stops_at_very_low(): + assert grade("cohort", ()) == "high" + assert grade("case-control", ()) == "high" + assert grade("quasi-experimental", ()) == "high" + assert grade("cohort", ("risk of bias",)) == "moderate" + assert grade("cross-sectional", ()) == "low" + assert grade("cohort", ("a", "b", "c", "d")) == "very low" + assert grade("cohort", ("a", "b", "c", "d", "e")) == "very low" + + +def test_may_pool_needs_three_identical_measures(): + assert may_pool(["OR", "OR", "OR"]) is True + assert may_pool(["OR", "OR"]) is False + assert may_pool(["OR", "OR", "RR"]) is False + + +def test_same_extraction_inserts_one_claim_and_the_catalog_reads_it(tmp_path): + extraction = { + "zotero_key": "ABCD2345", + "extraction_version": "v1", + "outcome_target_id": "Q.ACR", + "grain": "beneficiary", + "lead_time": "12 months", + "design": "cohort", + } + first = claims_from_extraction( + extraction, project="lead-precursors", actor="tester" + )[0] + second = claims_from_extraction( + extraction, project="lead-precursors", actor="tester" + )[0] + limit = claims_from_extraction( + {"target_id": "Q.ACR", "grain": "ACO", "zotero_key": "EFGH5678"}, + project="lead-limits", + actor="tester", + )[0] + con = open_ledger(str(tmp_path / "ledger.duckdb")) + try: + assert append(con, "claim", first) is True + assert append(con, "claim", second) is False + assert ( + con.execute("select count(*) from lead_evidence.claim").fetchone()[0] == 1 + ) + assert append(con, "claim", limit) is True + catalog = precursor_catalog(con) + row = next(item for item in catalog if item["target_id"] == "Q.ACR") + assert row["lead_time"] == first["lead_time"] + assert row["certainty"] == first["certainty"] + assert row["grain"] == first["grain"] + assert row["limit_claim_id"] == limit["content_hash"] + finally: + con.close() + + +def test_unknown_target_raises_key_error(): + with pytest.raises(KeyError): + claims_from_extraction( + {"outcome_target_id": "nope", "grain": "beneficiary"}, + project="lead-precursors", + actor="tester", + ) -- 2.49.1 From 9e9908dd9f0da5a5ee47ea5d3613e04d132e09ba Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:48:16 -0400 Subject: [PATCH 15/30] feat(prisma): grey-literature harvest parsers --- src/prisma/harvest.py | 111 +++++++++++++++++++++++++++++++++++ tests/prisma/test_harvest.py | 27 +++++++++ 2 files changed, 138 insertions(+) create mode 100644 src/prisma/harvest.py create mode 100644 tests/prisma/test_harvest.py diff --git a/src/prisma/harvest.py b/src/prisma/harvest.py new file mode 100644 index 0000000..1edf9d1 --- /dev/null +++ b/src/prisma/harvest.py @@ -0,0 +1,111 @@ +"""Grey-literature HTML parsers. The caller supplies the page. No HTTP. + +Each source has its own function. They share one stdlib parser because +the fixture contract is the same: one report link and one date. A source +whose real page differs gets its own fixture later. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from html.parser import HTMLParser + + +@dataclass(frozen=True) +class Harvested: + title: str + url: str + date: str + source: str + + +class _Page(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.links: list[tuple[str, str]] = [] + self.times: list[str] = [] + self._href: str | None = None + self._link_text: list[str] = [] + self._in_time = False + self._time_text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {name: value or "" for name, value in attrs} + if tag == "a" and values.get("href"): + self._href = values["href"] + self._link_text = [] + elif tag == "time": + self._in_time = True + self._time_text = [] + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._href is not None: + self.links.append((self._href, "".join(self._link_text).strip())) + self._href = None + elif tag == "time" and self._in_time: + self.times.append("".join(self._time_text).strip()) + self._in_time = False + + def handle_data(self, data: str) -> None: + if self._href is not None: + self._link_text.append(data) + if self._in_time: + self._time_text.append(data) + + +def _parse(html: str, source: str) -> tuple[Harvested, ...]: + if not html: + return () + page = _Page() + page.feed(html) + page.close() + date = page.times[0] if page.times else "" + return tuple( + Harvested(title=title, url=url, date=date, source=source) + for url, title in page.links + ) + + +def parse_cms(html: str) -> tuple[Harvested, ...]: + return _parse(html, "cms") + + +def parse_medpac(html: str) -> tuple[Harvested, ...]: + return _parse(html, "medpac") + + +def parse_macpac(html: str) -> tuple[Harvested, ...]: + return _parse(html, "macpac") + + +def parse_aspe(html: str) -> tuple[Harvested, ...]: + return _parse(html, "aspe") + + +def parse_gao(html: str) -> tuple[Harvested, ...]: + return _parse(html, "gao") + + +def parse_cbo(html: str) -> tuple[Harvested, ...]: + return _parse(html, "cbo") + + +def parse_crs(html: str) -> tuple[Harvested, ...]: + return _parse(html, "crs") + + +def parse_kff(html: str) -> tuple[Harvested, ...]: + return _parse(html, "kff") + + +PARSERS: dict[str, Callable[[str], tuple[Harvested, ...]]] = { + "cms": parse_cms, + "medpac": parse_medpac, + "macpac": parse_macpac, + "aspe": parse_aspe, + "gao": parse_gao, + "cbo": parse_cbo, + "crs": parse_crs, + "kff": parse_kff, +} diff --git a/tests/prisma/test_harvest.py b/tests/prisma/test_harvest.py new file mode 100644 index 0000000..944b9f6 --- /dev/null +++ b/tests/prisma/test_harvest.py @@ -0,0 +1,27 @@ +"""Grey-literature HTML parsers. No HTTP.""" + +from __future__ import annotations + +import pytest + +from prisma.harvest import PARSERS, Harvested + +_SOURCES = ("cms", "medpac", "macpac", "aspe", "gao", "cbo", "crs", "kff") +_HTML = 'Report title' + + +@pytest.mark.parametrize("source", _SOURCES) +def test_parser_reads_the_report_link_and_date(source: str): + assert PARSERS[source](_HTML) == ( + Harvested( + title="Report title", + url="https://example.test/r", + date="2026-01-02", + source=source, + ), + ) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_empty_html_returns_nothing(source: str): + assert PARSERS[source]("") == () -- 2.49.1 From 9a85d766d7c40da4746cef21904eef6df1ff10bf Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:49:08 -0400 Subject: [PATCH 16/30] feat(prisma): Congress.gov bill parser --- src/prisma/congress.py | 53 +++++++++++++++++++++++++++++++++++ tests/prisma/test_congress.py | 51 +++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 src/prisma/congress.py create mode 100644 tests/prisma/test_congress.py diff --git a/src/prisma/congress.py b/src/prisma/congress.py new file mode 100644 index 0000000..b15bc35 --- /dev/null +++ b/src/prisma/congress.py @@ -0,0 +1,53 @@ +"""Congress.gov bill list. The HTTP call is a transport the caller supplies.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass + +_BILL_URL = "https://api.congress.gov/v3/bill/{congress}?api_key={key}&format=json" + + +@dataclass(frozen=True) +class Bill: + congress: int + bill_type: str + number: int + title: str + latest_action: str + + +def parse_bills(payload: Mapping[str, object]) -> tuple[Bill, ...]: + """Read the ``bills`` array. Missing action text becomes an empty string.""" + raw = payload.get("bills") or () + bills: list[Bill] = [] + if not isinstance(raw, list): + return () + for item in raw: + if not isinstance(item, Mapping): + continue + action = item.get("latestAction") or {} + text = action.get("text") if isinstance(action, Mapping) else "" + bills.append( + Bill( + congress=int(item["congress"]), + bill_type=str(item.get("type") or ""), + number=int(item["number"]), + title=str(item.get("title") or ""), + latest_action=str(text or ""), + ) + ) + return tuple(bills) + + +def fetch_bills( + congress: int, + *, + key: str, + transport: Callable[[str], Mapping[str, object]], +) -> tuple[Bill, ...]: + """GET the bill list for one congress. An empty key raises and does not fetch.""" + if not key: + raise ValueError("Congress.gov API key is required") + payload = transport(_BILL_URL.format(congress=congress, key=key)) + return parse_bills(payload) diff --git a/tests/prisma/test_congress.py b/tests/prisma/test_congress.py new file mode 100644 index 0000000..6b8c967 --- /dev/null +++ b/tests/prisma/test_congress.py @@ -0,0 +1,51 @@ +"""Congress.gov bill payloads. The transport is injected. No HTTP.""" + +from __future__ import annotations + +import pytest + +from prisma.congress import Bill, fetch_bills, parse_bills + +_PAYLOAD = { + "bills": [ + { + "congress": 119, + "type": "hr", + "number": 1, + "title": "A bill", + "latestAction": {"text": "Introduced"}, + } + ] +} +_BILL = Bill( + congress=119, + bill_type="hr", + number=1, + title="A bill", + latest_action="Introduced", +) + + +def test_parse_bills_reads_one_bill(): + assert parse_bills(_PAYLOAD) == (_BILL,) + + +def test_fetch_bills_passes_the_congress_url_to_the_transport(): + seen: list[str] = [] + + def transport(url: str) -> dict[str, object]: + seen.append(url) + return _PAYLOAD + + assert fetch_bills(119, key="test-key", transport=transport) == (_BILL,) + assert len(seen) == 1 + assert "119" in seen[0] + assert "api_key=test-key" in seen[0] + + +def test_empty_key_does_not_call_the_transport(): + def transport(url: str) -> dict[str, object]: + raise AssertionError(url) + + with pytest.raises(ValueError): + fetch_bills(119, key="", transport=transport) -- 2.49.1 From 2153094b77bcb67ae1a46ff51a6b5362377d4b0a Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:51:10 -0400 Subject: [PATCH 17/30] feat: parse PPI, JOLTS, QCEW, FRED, and IRS SOI extracts --- src/bls/jolts.py | 11 +++++++ src/bls/ppi.py | 52 ++++++++++++++++++++++++++++++++ src/bls/qcew.py | 11 +++++++ src/econ/__init__.py | 1 + src/econ/fred.py | 45 +++++++++++++++++++++++++++ src/econ/irs_soi.py | 42 ++++++++++++++++++++++++++ tests/bls/test_ppi_jolts_qcew.py | 29 ++++++++++++++++++ tests/econ/test_fred_irs.py | 31 +++++++++++++++++++ 8 files changed, 222 insertions(+) create mode 100644 src/bls/jolts.py create mode 100644 src/bls/ppi.py create mode 100644 src/bls/qcew.py create mode 100644 src/econ/__init__.py create mode 100644 src/econ/fred.py create mode 100644 src/econ/irs_soi.py create mode 100644 tests/bls/test_ppi_jolts_qcew.py create mode 100644 tests/econ/test_fred_irs.py diff --git a/src/bls/jolts.py b/src/bls/jolts.py new file mode 100644 index 0000000..452fd20 --- /dev/null +++ b/src/bls/jolts.py @@ -0,0 +1,11 @@ +"""JOLTS extract. Same columns as PPI. The caller supplies the CSV.""" + +from __future__ import annotations + +from bls.ppi import SeriesRow, parse_series + +__all__ = ["SeriesRow", "parse_jolts"] + + +def parse_jolts(csv_text: str) -> tuple[SeriesRow, ...]: + return parse_series(csv_text) diff --git a/src/bls/ppi.py b/src/bls/ppi.py new file mode 100644 index 0000000..e3fb198 --- /dev/null +++ b/src/bls/ppi.py @@ -0,0 +1,52 @@ +"""PPI extract. The caller supplies the CSV. No download. + +JOLTS and QCEW use this same reader. A later file with different columns +should stop calling :func:`parse_series`. +""" + +from __future__ import annotations + +import csv +import io +from dataclasses import dataclass + +_COLUMNS = ("series_id", "year", "period", "value") + + +@dataclass(frozen=True) +class SeriesRow: + series_id: str + year: int + period: str + value: float | None + + +def _value(text: str) -> float | None: + raw = text.strip() + if raw in {"", "."}: + return None + return float(raw) + + +def parse_series(csv_text: str) -> tuple[SeriesRow, ...]: + """Read ``series_id,year,period,value``. A missing column raises.""" + reader = csv.DictReader(io.StringIO(csv_text)) + names = set(reader.fieldnames or ()) + missing = [name for name in _COLUMNS if name not in names] + if missing: + raise ValueError(f"missing columns: {missing}") + rows: list[SeriesRow] = [] + for record in reader: + rows.append( + SeriesRow( + series_id=(record.get("series_id") or "").strip(), + year=int(record["year"]), + period=(record.get("period") or "").strip(), + value=_value(record.get("value") or ""), + ) + ) + return tuple(rows) + + +def parse_ppi(csv_text: str) -> tuple[SeriesRow, ...]: + return parse_series(csv_text) diff --git a/src/bls/qcew.py b/src/bls/qcew.py new file mode 100644 index 0000000..e84852a --- /dev/null +++ b/src/bls/qcew.py @@ -0,0 +1,11 @@ +"""QCEW extract. Same columns as PPI. The caller supplies the CSV.""" + +from __future__ import annotations + +from bls.ppi import SeriesRow, parse_series + +__all__ = ["SeriesRow", "parse_qcew"] + + +def parse_qcew(csv_text: str) -> tuple[SeriesRow, ...]: + return parse_series(csv_text) diff --git a/src/econ/__init__.py b/src/econ/__init__.py new file mode 100644 index 0000000..96fa79b --- /dev/null +++ b/src/econ/__init__.py @@ -0,0 +1 @@ +"""Public economic series that are not BLS products: FRED and IRS SOI.""" diff --git a/src/econ/fred.py b/src/econ/fred.py new file mode 100644 index 0000000..62ba971 --- /dev/null +++ b/src/econ/fred.py @@ -0,0 +1,45 @@ +"""FRED observations. The caller supplies the payload. No download.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SeriesRow: + series_id: str + year: int + period: str + value: float | None + + +def _value(raw: object) -> float | None: + if raw is None: + return None + text = str(raw).strip() + if text in {"", "."}: + return None + return float(text) + + +def parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]: + """One row per observation. A ``.`` value is missing, not zero.""" + raw = payload.get("observations") or () + if not isinstance(raw, list): + return () + rows: list[SeriesRow] = [] + for item in raw: + if not isinstance(item, Mapping): + continue + date = str(item.get("date") or "") + year = int(date[:4]) if len(date) >= 4 and date[:4].isdigit() else 0 + rows.append( + SeriesRow( + series_id=str(item.get("series_id") or ""), + year=year, + period=date, + value=_value(item.get("value")), + ) + ) + return tuple(rows) diff --git a/src/econ/irs_soi.py b/src/econ/irs_soi.py new file mode 100644 index 0000000..833e45d --- /dev/null +++ b/src/econ/irs_soi.py @@ -0,0 +1,42 @@ +"""IRS SOI county income. The caller supplies the CSV. No download.""" + +from __future__ import annotations + +import csv +import io +from dataclasses import dataclass + +_COLUMNS = ("fips", "year", "agi") + + +@dataclass(frozen=True) +class CountyIncome: + fips: str + year: int + agi: int | None + + +def _agi(text: str) -> int | None: + raw = text.strip() + if raw in {"", "."}: + return None + return int(raw) + + +def parse_county_income(csv_text: str) -> tuple[CountyIncome, ...]: + """Read ``fips,year,agi``. FIPS stays text. A missing column raises.""" + reader = csv.DictReader(io.StringIO(csv_text)) + names = set(reader.fieldnames or ()) + missing = [name for name in _COLUMNS if name not in names] + if missing: + raise ValueError(f"missing columns: {missing}") + rows: list[CountyIncome] = [] + for record in reader: + rows.append( + CountyIncome( + fips=(record.get("fips") or "").strip(), + year=int(record["year"]), + agi=_agi(record.get("agi") or ""), + ) + ) + return tuple(rows) diff --git a/tests/bls/test_ppi_jolts_qcew.py b/tests/bls/test_ppi_jolts_qcew.py new file mode 100644 index 0000000..a51b4bd --- /dev/null +++ b/tests/bls/test_ppi_jolts_qcew.py @@ -0,0 +1,29 @@ +"""PPI, JOLTS, and QCEW extracts share one CSV shape. No download.""" + +from __future__ import annotations + +import pytest + +from bls.jolts import parse_jolts +from bls.ppi import SeriesRow, parse_ppi +from bls.qcew import parse_qcew + +_PARSERS = (parse_ppi, parse_jolts, parse_qcew) + + +@pytest.mark.parametrize("parse", _PARSERS) +def test_one_row_keeps_the_value(parse): + text = "series_id,year,period,value\nWPS512,2024,M01,100.5\n" + assert parse(text) == (SeriesRow("WPS512", 2024, "M01", 100.5),) + + +@pytest.mark.parametrize("parse", _PARSERS) +def test_blank_value_is_none_not_zero(parse): + text = "series_id,year,period,value\nWPS512,2024,M01,\n" + assert parse(text) == (SeriesRow("WPS512", 2024, "M01", None),) + + +@pytest.mark.parametrize("parse", _PARSERS) +def test_missing_column_raises(parse): + with pytest.raises(ValueError): + parse("series_id,year,period\nWPS512,2024,M01\n") diff --git a/tests/econ/test_fred_irs.py b/tests/econ/test_fred_irs.py new file mode 100644 index 0000000..ddfee62 --- /dev/null +++ b/tests/econ/test_fred_irs.py @@ -0,0 +1,31 @@ +"""FRED observations and IRS SOI county income. No download.""" + +from __future__ import annotations + +import pytest + +from econ.fred import parse_observations +from econ.irs_soi import CountyIncome, parse_county_income + + +def test_fred_dot_value_is_missing(): + rows = parse_observations({"observations": [{"date": "2024-01-01", "value": "."}]}) + assert len(rows) == 1 + assert rows[0].value is None + assert rows[0].year == 2024 + assert rows[0].period == "2024-01-01" + + +def test_county_income_keeps_the_fips_text(): + rows = parse_county_income("fips,year,agi\n01001,2022,40000\n") + assert rows == (CountyIncome(fips="01001", year=2022, agi=40000),) + + +def test_blank_agi_is_none(): + rows = parse_county_income("fips,year,agi\n01001,2022,\n") + assert rows[0].agi is None + + +def test_missing_column_raises(): + with pytest.raises(ValueError): + parse_county_income("fips,year\n01001,2022\n") -- 2.49.1 From 32a1c5a0b8ac4bc63e496b84ddfd94ad81e8c52f Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 20:53:52 -0400 Subject: [PATCH 18/30] feat(prisma): nightly and triggered search due check --- src/cli/prisma.py | 19 +++ src/prisma/search/due.py | 93 +++++++++++++ tests/prisma/test_search_due.py | 226 ++++++++++++++++++++++++++++++++ 3 files changed, 338 insertions(+) create mode 100644 src/prisma/search/due.py create mode 100644 tests/prisma/test_search_due.py diff --git a/src/cli/prisma.py b/src/cli/prisma.py index a068756..aad8385 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -437,6 +437,25 @@ def search_protocols() -> None: typer.echo(f"{name}\t{query}") +@search_app.command("due") +def search_due( + project: str = typer.Option(..., "--project", help="Zotero project name."), + db: str = typer.Option(..., "--db", help="Path to the DuckDB ledger."), + now: str = typer.Option(..., "--now", help="Today as YYYY-MM-DD."), +) -> None: + """Print why a search is due. Prints nothing when it is current.""" + from lead.ledger.store import open_ledger + from prisma.search.due import due + + con = open_ledger(db) + try: + reasons = due(con, project=project, now=now) + finally: + con.close() + for reason in reasons: + typer.echo(reason) + + @app.command("calibrate") def calibrate_cmd() -> None: """Calibration gate: sensitivity ≥ 0.95 on 200 title/abstract and 50 full text.""" diff --git a/src/prisma/search/due.py b/src/prisma/search/due.py new file mode 100644 index 0000000..6ae4b31 --- /dev/null +++ b/src/prisma/search/due.py @@ -0,0 +1,93 @@ +"""Whether a project's searches are due. This does not run a search. + +Dates are ``YYYY-MM-DD`` strings, compared as text. Nightly means the +latest ``searched_on`` for the project is missing or earlier than today. +The other reasons are a contradiction of moderate or high certainty, a +strand-5 event on a target this project has a claim about, and a CMS +source dated after that latest search. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import duckdb + + +def _latest(con: duckdb.DuckDBPyConnection, project: str) -> str | None: + row = con.execute( + """ + SELECT max(searched_on) + FROM lead_evidence.search + WHERE project = ? + """, + [project], + ).fetchone() + if row is None or row[0] is None: + return None + return str(row[0]) + + +def _contradiction(con: duckdb.DuckDBPyConnection, project: str) -> bool: + row = con.execute( + """ + SELECT 1 + FROM lead_evidence.finding AS f + JOIN lead_evidence.claim AS c ON c.id = f.claim_id + WHERE c.project = ? + AND f.relation = 'contradicts' + AND c.certainty IN ('moderate', 'high') + LIMIT 1 + """, + [project], + ).fetchone() + return row is not None + + +def _exogenous(con: duckdb.DuckDBPyConnection, project: str) -> bool: + row = con.execute( + """ + SELECT 1 + FROM lead_evidence.exogenous AS e + WHERE EXISTS ( + SELECT 1 + FROM lead_evidence.claim AS c + WHERE c.project = ? + AND c.target_id = e.target_id + ) + LIMIT 1 + """, + [project], + ).fetchone() + return row is not None + + +def _cms(con: duckdb.DuckDBPyConnection, project: str, latest: str) -> bool: + row = con.execute( + """ + SELECT 1 + FROM lead_evidence.source + WHERE project = ? + AND database = 'cms' + AND created_at > ? + LIMIT 1 + """, + [project, latest], + ).fetchone() + return row is not None + + +def due(con: duckdb.DuckDBPyConnection, *, project: str, now: str) -> tuple[str, ...]: + """Reasons a search should run, in a fixed order.""" + reasons: list[str] = [] + latest = _latest(con, project) + if latest is None or latest < now: + reasons.append("nightly") + if _contradiction(con, project): + reasons.append("contradiction") + if _exogenous(con, project): + reasons.append("exogenous") + if latest is not None and _cms(con, project, latest): + reasons.append("cms") + return tuple(reasons) diff --git a/tests/prisma/test_search_due.py b/tests/prisma/test_search_due.py new file mode 100644 index 0000000..a15858a --- /dev/null +++ b/tests/prisma/test_search_due.py @@ -0,0 +1,226 @@ +"""Search is due nightly, and also when a trigger lands. No search is run.""" + +from __future__ import annotations + +import pytest +from typer.testing import CliRunner + +from cli.prisma import app +from lead.ledger.store import append, open_ledger +from prisma.search.due import due + +runner = CliRunner() +PROJECT = "lead-precursors" +NOW = "2026-10-31" +YESTERDAY = "2026-10-30" + + +def _ledger(tmp_path): + return open_ledger(str(tmp_path / "ledger.duckdb")) + + +def _search( + con, *, searched_on: str, project: str = PROJECT, suffix: str = "s" +) -> None: + assert append( + con, + "search", + { + "project": project, + "database": "pubmed", + "interface": "eutils", + "searched_on": searched_on, + "query": "lead", + "filters": "", + "hit_count": 1, + "export_sha256": suffix, + "actor": "tester", + "content_hash": f"search-{suffix}", + }, + ) + + +def _claim(con, *, certainty: str, suffix: str, target_id: str = "Q.ACR") -> None: + assert append( + con, + "claim", + { + "id": suffix, + "project": PROJECT, + "kind": "precursor", + "target_id": target_id, + "certainty": certainty, + "actor": "tester", + "content_hash": f"claim-{suffix}", + }, + ) + + +def _finding(con, *, claim_id: str, suffix: str) -> None: + assert append( + con, + "finding", + { + "claim_id": claim_id, + "relation": "contradicts", + "grain": "beneficiary", + "actor": "tester", + "content_hash": f"finding-{suffix}", + }, + ) + + +def test_missing_or_stale_search_is_nightly(tmp_path): + con = _ledger(tmp_path) + try: + assert due(con, project=PROJECT, now=NOW) == ("nightly",) + _search(con, searched_on=NOW, suffix="today") + assert due(con, project=PROJECT, now=NOW) == () + _search(con, searched_on=YESTERDAY, suffix="yesterday", project="lead-history") + assert due(con, project=PROJECT, now=NOW) == () + finally: + con.close() + + con = open_ledger(str(tmp_path / "stale.duckdb")) + try: + _search(con, searched_on=YESTERDAY, suffix="old") + assert due(con, project=PROJECT, now=NOW) == ("nightly",) + finally: + con.close() + + +def test_another_project_does_not_clear_nightly(tmp_path): + con = _ledger(tmp_path) + try: + _search(con, searched_on=NOW, project="lead-history", suffix="other") + assert due(con, project=PROJECT, now=NOW) == ("nightly",) + finally: + con.close() + + +@pytest.mark.parametrize( + ("certainty", "expected"), + [("low", ()), ("moderate", ("contradiction",)), ("high", ("contradiction",))], +) +def test_contradiction_respects_certainty(tmp_path, certainty, expected): + con = _ledger(tmp_path) + try: + _search(con, searched_on=NOW) + _claim(con, certainty=certainty, suffix=certainty) + _finding(con, claim_id=certainty, suffix=certainty) + assert due(con, project=PROJECT, now=NOW) == expected + finally: + con.close() + + +def test_exogenous_event_on_a_claimed_target(tmp_path): + con = _ledger(tmp_path) + try: + _search(con, searched_on=NOW) + assert append( + con, + "exogenous", + { + "project": PROJECT, + "target_id": "Q.ACR", + "series_or_event": "cms guidance", + "event_on": NOW, + "citation": "cms", + "actor": "tester", + "content_hash": "exo-1", + }, + ) + assert due(con, project=PROJECT, now=NOW) == () + _claim(con, certainty="low", suffix="qacr") + assert due(con, project=PROJECT, now=NOW) == ("exogenous",) + finally: + con.close() + + +def test_cms_source_newer_than_the_search(tmp_path): + con = _ledger(tmp_path) + try: + _search(con, searched_on=NOW) + assert append( + con, + "source", + { + "project": PROJECT, + "database": "cms", + "zotero_key": "ABCD2345", + "created_at": NOW, + "actor": "tester", + "content_hash": "src-same-day", + }, + ) + assert due(con, project=PROJECT, now=NOW) == () + assert append( + con, + "source", + { + "project": PROJECT, + "database": "cms", + "zotero_key": "EFGH5678", + "created_at": "2026-11-01", + "actor": "tester", + "content_hash": "src-later", + }, + ) + assert due(con, project=PROJECT, now=NOW) == ("cms",) + finally: + con.close() + + +def test_reasons_stay_in_order(tmp_path): + con = _ledger(tmp_path) + try: + _search(con, searched_on=YESTERDAY, suffix="old") + _claim(con, certainty="high", suffix="high") + _finding(con, claim_id="high", suffix="high") + assert append( + con, + "exogenous", + { + "target_id": "Q.ACR", + "series_or_event": "cms guidance", + "event_on": NOW, + "actor": "tester", + "content_hash": "exo-order", + }, + ) + assert append( + con, + "source", + { + "project": PROJECT, + "database": "cms", + "created_at": "2026-11-01", + "actor": "tester", + "content_hash": "src-order", + }, + ) + assert due(con, project=PROJECT, now=NOW) == ( + "nightly", + "contradiction", + "exogenous", + "cms", + ) + finally: + con.close() + + +def test_due_command_is_registered_and_quiet_when_current(tmp_path): + result = runner.invoke(app, ["search", "due", "--help"]) + assert result.exit_code == 0 + path = tmp_path / "ledger.duckdb" + con = open_ledger(str(path)) + try: + _search(con, searched_on=NOW, suffix="cli") + finally: + con.close() + result = runner.invoke( + app, + ["search", "due", "--db", str(path), "--project", PROJECT, "--now", NOW], + ) + assert result.exit_code == 0 + assert result.stdout.strip() == "" -- 2.49.1 From fe99b6bbf233b1083517ece1f0c354ba4195bde7 Mon Sep 17 00:00:00 2001 From: kert Date: Sat, 31 Oct 2026 21:31:11 -0400 Subject: [PATCH 19/30] fix: correct the part 1 review findings Export reads the extraction JSON instead of a tag fragment and writes the claim. A logged search stores its project so the nightly check can clear. Claims keep the effect and the subject, a later strand tags an existing DOI, the dedup command prints borderline pairs, and due dates compare the calendar day. --- src/cli/prisma.py | 12 ++- src/lead/ledger/claims.py | 75 ++++++++++---- src/lead/ledger/export.py | 69 +++++++++++-- src/prisma/search/base.py | 83 ++++++++------- src/prisma/search/due.py | 68 +++++++----- tests/lead/test_ledger_claims.py | 76 ++++++++++++++ tests/lead/test_ledger_export.py | 135 ++++++++++++++++++++++++ tests/prisma/test_dedup.py | 29 ++++++ tests/prisma/test_search_due.py | 161 ++++++++++++++++++++++++++--- tests/prisma/test_search_pubmed.py | 84 +++++++++++++++ 10 files changed, 678 insertions(+), 114 deletions(-) diff --git a/src/cli/prisma.py b/src/cli/prisma.py index aad8385..84ea3f7 100644 --- a/src/cli/prisma.py +++ b/src/cli/prisma.py @@ -412,15 +412,19 @@ def dedup_cmd( hold: bool = typer.Option(True, "--hold/--no-hold"), ) -> None: """Tag DOI, PMID, and title duplicates. Near-matches stay untagged.""" - from prisma.dedup import mark_duplicates + from prisma.dedup import cluster, load_records, mark_duplicates from zot.db import Db - def _go() -> int: + def _go(): with Db(str(db)) as zdb: - return mark_duplicates(zdb) + tagged = mark_duplicates(zdb) + borderline = cluster(load_records(zdb)).borderline + return tagged, borderline - tagged = _hold(_go, hold=hold) + tagged, borderline = _hold(_go, hold=hold) typer.echo(f" tagged: {tagged}") + for left, right in borderline: + typer.echo(f" borderline: {left.key} {left.title} | {right.key} {right.title}") search_app = typer.Typer(no_args_is_help=True) diff --git a/src/lead/ledger/claims.py b/src/lead/ledger/claims.py index d5df5e3..1ff6d1a 100644 --- a/src/lead/ledger/claims.py +++ b/src/lead/ledger/claims.py @@ -9,6 +9,7 @@ recording the same extraction again inserts nothing. from __future__ import annotations import hashlib +import re from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING @@ -37,6 +38,7 @@ _DOWNGRADE_LABELS = { "risk_of_bias": "risk of bias", "publication_bias": "publication bias", } +_INTERVAL = re.compile(r"\s*(?:,|to)\s*", re.IGNORECASE) def grade(design: str, downgrades: Sequence[str]) -> str: @@ -56,6 +58,26 @@ def _target_id(project: str, extraction: Mapping[str, str]) -> str: return extraction.get("target_id") or extraction.get("outcome_target_id") or "" +def _number(value: str) -> float | None: + text = value.strip() + if not text: + return None + try: + return float(text) + except ValueError: + return None + + +def _interval(value: str) -> tuple[float | None, float | None]: + text = value.strip() + if not text: + return None, None + parts = _INTERVAL.split(text, maxsplit=1) + if len(parts) != 2: + return None, None + return _number(parts[0]), _number(parts[1]) + + def _downgrades(extraction: Mapping[str, str]) -> tuple[str, ...]: labels: list[str] = [] for field in _DOWNGRADE_FIELDS: @@ -110,11 +132,26 @@ def claims_from_extraction( ) if part ) + measured = _number(extraction.get("effect_value", "")) + if measured is not None: + claim["effect_value"] = measured + low, high = _interval(extraction.get("effect_ci", "")) + if low is not None: + claim["effect_ci_low"] = low + if high is not None: + claim["effect_ci_high"] = high + if "subject" not in claim: + subject = ( + extraction.get("factor_definition", "").strip() + or extraction.get("statement", "").strip() + ) + if subject: + claim["subject"] = subject return (claim,) def precursor_catalog(con: duckdb.DuckDBPyConnection) -> list[Mapping[str, object]]: - """One row per target: precursor lead time, grain, certainty, and any limit.""" + """One row per precursor claim, plus that target's first limit claim.""" rows = con.execute( """ SELECT id, kind, target_id, lead_time, grain, certainty @@ -122,27 +159,21 @@ def precursor_catalog(con: duckdb.DuckDBPyConnection) -> list[Mapping[str, objec ORDER BY target_id, id """ ).fetchall() - grouped: dict[str, dict[str, object]] = {} + limits: dict[str, str] = {} + precursors: list[dict[str, object]] = [] for claim_id, kind, target_id, lead_time, grain, certainty in rows: - entry = grouped.setdefault( - target_id, - { - "target_id": target_id, - "lead_time": "", - "grain": "", - "certainty": "", - "limit_claim_id": None, - "_precursor": False, - }, - ) - if kind == "precursor" and not entry["_precursor"]: - entry["lead_time"] = lead_time or "" - entry["grain"] = grain or "" - entry["certainty"] = certainty or "" - entry["_precursor"] = True - if kind == "limit" and entry["limit_claim_id"] is None: - entry["limit_claim_id"] = claim_id + if kind == "limit" and target_id not in limits: + limits[target_id] = claim_id + elif kind == "precursor": + precursors.append( + { + "target_id": target_id, + "lead_time": lead_time or "", + "grain": grain or "", + "certainty": certainty or "", + } + ) return [ - {key: value for key, value in entry.items() if key != "_precursor"} - for entry in grouped.values() + {**entry, "limit_claim_id": limits.get(str(entry["target_id"]))} + for entry in precursors ] diff --git a/src/lead/ledger/export.py b/src/lead/ledger/export.py index 0a40d09..86aa7fc 100644 --- a/src/lead/ledger/export.py +++ b/src/lead/ledger/export.py @@ -1,17 +1,22 @@ """Export one Zotero project's review rows into the evidence ledger. -Anchors tagged ``module:prisma`` are not records. Claims are not written -here. A second export of the same item key and extraction version inserts -nothing, because that pair is in the content hash. +Anchors tagged ``module:prisma`` are not records. An included extraction +also appends one claim from the note fields. A second export of the same +item key and extraction version inserts nothing, because that pair is in +the content hash. """ from __future__ import annotations import hashlib +import html +import json import re from typing import TYPE_CHECKING +from lead.ledger.claims import claims_from_extraction from lead.ledger.store import append +from lead.targets import by_id if TYPE_CHECKING: import duckdb @@ -19,7 +24,9 @@ if TYPE_CHECKING: from zot.db import Db _DECISIONS = frozenset({"include", "exclude", "uncertain"}) -_TARGET = re.compile(r"target_id:\s*(\S+)") +_PRE = re.compile(r"
(.*?)
", re.DOTALL) +_TAG = re.compile(r"<[^>]+>") +_FIELD = re.compile(r"(? str: @@ -54,6 +61,32 @@ def _note_body(zdb: Db, item_id: int) -> str: return "\n".join(row[0] or "" for row in rows) +def _fields_from_note(body: str) -> dict[str, str]: + """JSON inside ``
`` wins. A plain line must not capture a tag."""
+    match = _PRE.search(body)
+    if match:
+        try:
+            payload = json.loads(html.unescape(match.group(1)).strip())
+        except json.JSONDecodeError:
+            payload = None
+        if isinstance(payload, dict):
+            return {
+                str(key): "" if value is None else str(value)
+                for key, value in payload.items()
+            }
+    text = _TAG.sub(" ", html.unescape(body))
+    found: dict[str, str] = {}
+    for key, value in _FIELD.findall(text):
+        if value.startswith(("<", "&")):
+            continue
+        found.setdefault(key, value)
+    return found
+
+
+def _target(fields: dict[str, str]) -> str:
+    return (fields.get("outcome_target_id") or fields.get("target_id") or "").strip()
+
+
 def export_project(
     zdb: Db,
     con: duckdb.DuckDBPyConnection,
@@ -62,10 +95,20 @@ def export_project(
     extraction_version: str,
     actor: str,
 ) -> dict[str, int]:
-    """Insert source, decision, and extraction rows. Returns how many landed."""
+    """Insert source, decision, and extraction rows.
+
+    A note that names a known target also appends one claim. The returned
+    counts are the three review tables.
+    """
     inserted = {"source": 0, "decision": 0, "extraction": 0}
     for item_id, key in _project_items(zdb, project):
         tags = zdb.get_tags(item_id)
+        included = "stage:4-included" in tags
+        body = _note_body(zdb, item_id) if included else ""
+        fields = _fields_from_note(body) if included else {}
+        target = _target(fields)
+        if target:
+            by_id(target)
         databases = sorted(
             tag.removeprefix("found-in:") for tag in tags if tag.startswith("found-in:")
         )
@@ -118,9 +161,8 @@ def export_project(
             if append(con, "decision", row):
                 inserted["decision"] += 1
 
-        if "stage:4-included" not in tags:
+        if not included:
             continue
-        body = _note_body(zdb, item_id)
         extraction: dict[str, object] = {
             "project": project,
             "zotero_key": key,
@@ -129,9 +171,16 @@ def export_project(
             "actor": actor,
             "content_hash": _hash("extraction", key, extraction_version, project),
         }
-        found = _TARGET.search(body)
-        if found:
-            extraction["target_id"] = found.group(1)
+        if target:
+            extraction["target_id"] = target
         if append(con, "extraction", extraction):
             inserted["extraction"] += 1
+        if target:
+            claim_fields = dict(fields)
+            claim_fields["zotero_key"] = key
+            claim_fields["extraction_version"] = extraction_version
+            claim = claims_from_extraction(claim_fields, project=project, actor=actor)[
+                0
+            ]
+            append(con, "claim", dict(claim))
     return inserted
diff --git a/src/prisma/search/base.py b/src/prisma/search/base.py
index c57d609..f5db393 100644
--- a/src/prisma/search/base.py
+++ b/src/prisma/search/base.py
@@ -101,44 +101,55 @@ def seed_recall(retrieved: set[str], seeds: set[str]) -> float:
     return len(retrieved & seeds) / len(seeds)
 
 
-def record(con, result: SearchResult, *, actor: str) -> bool:
-    """Append one search row. The same export on a later day is a new row."""
+def record(con, result: SearchResult, *, actor: str, project: str = "") -> bool:
+    """Append one search row. The same export on a later day is a new row.
+
+    A nonempty ``project`` is stored and hashed, so two projects on the
+    same day stay two rows and a search logged today clears ``due``.
+    """
     log = result.log
-    raw = "\n".join(
-        [
-            log.database,
-            log.interface,
-            log.searched_on,
-            log.query,
-            log.filters,
-            str(log.hit_count),
-            log.export_sha256,
-            actor,
-        ]
-    )
-    return append(
-        con,
-        "search",
-        {
-            "database": log.database,
-            "interface": log.interface,
-            "searched_on": log.searched_on,
-            "query": log.query,
-            "filters": log.filters,
-            "hit_count": log.hit_count,
-            "export_sha256": log.export_sha256,
-            "content_hash": hashlib.sha256(raw.encode()).hexdigest(),
-            "actor": actor,
-        },
-    )
+    parts = [
+        log.database,
+        log.interface,
+        log.searched_on,
+        log.query,
+        log.filters,
+        str(log.hit_count),
+        log.export_sha256,
+        actor,
+    ]
+    if project:
+        parts.append(project)
+    row: dict[str, object] = {
+        "database": log.database,
+        "interface": log.interface,
+        "searched_on": log.searched_on,
+        "query": log.query,
+        "filters": log.filters,
+        "hit_count": log.hit_count,
+        "export_sha256": log.export_sha256,
+        "content_hash": hashlib.sha256("\n".join(parts).encode()).hexdigest(),
+        "actor": actor,
+    }
+    if project:
+        row["project"] = project
+    return append(con, "search", row)
 
 
 def import_hits(zdb: Db, project: str, result: SearchResult) -> int:
-    """Create one Zotero item per new DOI. Return how many were new."""
+    """Create one Zotero item per new DOI. Return how many were new.
+
+    A DOI already in the library is not inserted again. The existing item
+    gains this project's tag and this database's ``found-in`` tag.
+    """
     inserted = 0
     for hit in result.hits:
-        if hit.doi and _doi_exists(zdb, hit.doi):
-            continue
+        if hit.doi:
+            existing = _doi_item_id(zdb, hit.doi)
+            if existing is not None:
+                zdb.tag_item(existing, f"project:{project}")
+                zdb.tag_item(existing, f"found-in:{hit.database}")
+                continue
         item_id = zdb.create_item(TYPE_MAP["journalArticle"])
         fields = {"title": hit.title, "DOI": hit.doi, "PMID": hit.pmid}
         if hit.year is not None:
@@ -153,10 +164,10 @@ def import_hits(zdb: Db, project: str, result: SearchResult) -> int:
     return inserted
 
 
-def _doi_exists(zdb: Db, doi: str) -> bool:
+def _doi_item_id(zdb: Db, doi: str) -> int | None:
     row = zdb.con.execute(
         """
-        SELECT 1
+        SELECT id.itemID
         FROM itemData id
         JOIN itemDataValues idv ON id.valueID = idv.valueID
         WHERE id.fieldID = ? AND lower(idv.value) = lower(?)
@@ -164,4 +175,6 @@ def _doi_exists(zdb: Db, doi: str) -> bool:
         """,
         (FIELD_MAP["DOI"], doi.strip()),
     ).fetchone()
-    return row is not None
+    if row is None:
+        return None
+    return int(row[0])
diff --git a/src/prisma/search/due.py b/src/prisma/search/due.py
index 6ae4b31..800bd74 100644
--- a/src/prisma/search/due.py
+++ b/src/prisma/search/due.py
@@ -1,10 +1,12 @@
 """Whether a project's searches are due. This does not run a search.
 
-Dates are ``YYYY-MM-DD`` strings, compared as text. Nightly means the
-latest ``searched_on`` for the project is missing or earlier than today.
-The other reasons are a contradiction of moderate or high certainty, a
-strand-5 event on a target this project has a claim about, and a CMS
-source dated after that latest search.
+Dates are compared by the ``YYYY-MM-DD`` prefix, so an ISO timestamp and a
+date on the same day are the same day. Nightly means the latest
+``searched_on`` for the project is missing or earlier than today.
+Contradiction and exogenous fire only when their day is after that latest
+search. A project with no search row reports nightly only. CMS matches
+when ``cms`` is one comma-separated database token and the source day is
+after that latest search.
 """
 
 from __future__ import annotations
@@ -29,26 +31,32 @@ def _latest(con: duckdb.DuckDBPyConnection, project: str) -> str | None:
     return str(row[0])
 
 
-def _contradiction(con: duckdb.DuckDBPyConnection, project: str) -> bool:
-    row = con.execute(
+def _day(value: object) -> str:
+    if value is None:
+        return ""
+    return str(value)[:10]
+
+
+def _contradiction(con: duckdb.DuckDBPyConnection, project: str, latest: str) -> bool:
+    rows = con.execute(
         """
-        SELECT 1
+        SELECT f.created_at
         FROM lead_evidence.finding AS f
         JOIN lead_evidence.claim AS c ON c.id = f.claim_id
         WHERE c.project = ?
           AND f.relation = 'contradicts'
           AND c.certainty IN ('moderate', 'high')
-        LIMIT 1
         """,
         [project],
-    ).fetchone()
-    return row is not None
+    ).fetchall()
+    latest_day = _day(latest)
+    return any(_day(created_at) > latest_day for (created_at,) in rows)
 
 
-def _exogenous(con: duckdb.DuckDBPyConnection, project: str) -> bool:
-    row = con.execute(
+def _exogenous(con: duckdb.DuckDBPyConnection, project: str, latest: str) -> bool:
+    rows = con.execute(
         """
-        SELECT 1
+        SELECT e.event_on
         FROM lead_evidence.exogenous AS e
         WHERE EXISTS (
             SELECT 1
@@ -56,37 +64,41 @@ def _exogenous(con: duckdb.DuckDBPyConnection, project: str) -> bool:
             WHERE c.project = ?
               AND c.target_id = e.target_id
         )
-        LIMIT 1
         """,
         [project],
-    ).fetchone()
-    return row is not None
+    ).fetchall()
+    latest_day = _day(latest)
+    return any(_day(event_on) > latest_day for (event_on,) in rows)
 
 
 def _cms(con: duckdb.DuckDBPyConnection, project: str, latest: str) -> bool:
-    row = con.execute(
+    rows = con.execute(
         """
-        SELECT 1
+        SELECT database, created_at
         FROM lead_evidence.source
         WHERE project = ?
-          AND database = 'cms'
-          AND created_at > ?
-        LIMIT 1
         """,
-        [project, latest],
-    ).fetchone()
-    return row is not None
+        [project],
+    ).fetchall()
+    latest_day = _day(latest)
+    for database, created_at in rows:
+        tokens = [
+            part.strip() for part in str(database or "").split(",") if part.strip()
+        ]
+        if "cms" in tokens and _day(created_at) > latest_day:
+            return True
+    return False
 
 
 def due(con: duckdb.DuckDBPyConnection, *, project: str, now: str) -> tuple[str, ...]:
     """Reasons a search should run, in a fixed order."""
     reasons: list[str] = []
     latest = _latest(con, project)
-    if latest is None or latest < now:
+    if latest is None or _day(latest) < _day(now):
         reasons.append("nightly")
-    if _contradiction(con, project):
+    if latest is not None and _contradiction(con, project, latest):
         reasons.append("contradiction")
-    if _exogenous(con, project):
+    if latest is not None and _exogenous(con, project, latest):
         reasons.append("exogenous")
     if latest is not None and _cms(con, project, latest):
         reasons.append("cms")
diff --git a/tests/lead/test_ledger_claims.py b/tests/lead/test_ledger_claims.py
index aecbb42..b1a4b75 100644
--- a/tests/lead/test_ledger_claims.py
+++ b/tests/lead/test_ledger_claims.py
@@ -100,6 +100,82 @@ def test_same_extraction_inserts_one_claim_and_the_catalog_reads_it(tmp_path):
         con.close()
 
 
+def test_claim_keeps_effect_value_interval_and_subject():
+    claim = claims_from_extraction(
+        {
+            "target_id": "Q.ACR",
+            "effect_measure": "OR",
+            "effect_value": "1.2",
+            "effect_ci": "1.1, 1.4",
+            "factor_definition": "frailty status",
+            "design": "cohort",
+        },
+        project="lead-limits",
+        actor="tester",
+    )[0]
+    assert claim["effect_measure"] == "OR"
+    assert claim["effect_value"] == 1.2
+    assert claim["effect_ci_low"] == 1.1
+    assert claim["effect_ci_high"] == 1.4
+    assert claim["subject"] == "frailty status"
+
+    stated = claims_from_extraction(
+        {
+            "target_id": "Q.ACR",
+            "effect_ci": "1.1 to 1.4",
+            "statement": "care changes the status",
+        },
+        project="lead-limits",
+        actor="tester",
+    )[0]
+    assert stated["effect_ci_low"] == 1.1
+    assert stated["effect_ci_high"] == 1.4
+    assert stated["subject"] == "care changes the status"
+
+
+def test_catalog_lists_each_precursor_claim(tmp_path):
+    first = claims_from_extraction(
+        {
+            "zotero_key": "ABCD2345",
+            "extraction_version": "v1",
+            "outcome_target_id": "Q.ACR",
+            "grain": "beneficiary",
+            "lead_time": "12 months",
+            "design": "cohort",
+        },
+        project="lead-precursors",
+        actor="tester",
+    )[0]
+    second = claims_from_extraction(
+        {
+            "zotero_key": "IJKM2345",
+            "extraction_version": "v1",
+            "outcome_target_id": "Q.ACR",
+            "grain": "beneficiary",
+            "lead_time": "6 months",
+            "design": "cohort",
+        },
+        project="lead-precursors",
+        actor="tester",
+    )[0]
+    limit = claims_from_extraction(
+        {"target_id": "Q.ACR", "grain": "ACO", "zotero_key": "EFGH5678"},
+        project="lead-limits",
+        actor="tester",
+    )[0]
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        assert append(con, "claim", first) is True
+        assert append(con, "claim", second) is True
+        assert append(con, "claim", limit) is True
+        rows = [item for item in precursor_catalog(con) if item["target_id"] == "Q.ACR"]
+        assert len(rows) == 2
+        assert {row["lead_time"] for row in rows} == {"12 months", "6 months"}
+        assert {row["limit_claim_id"] for row in rows} == {limit["content_hash"]}
+    finally:
+        con.close()
+
+
 def test_unknown_target_raises_key_error():
     with pytest.raises(KeyError):
         claims_from_extraction(
diff --git a/tests/lead/test_ledger_export.py b/tests/lead/test_ledger_export.py
index 4f3647b..04be6bf 100644
--- a/tests/lead/test_ledger_export.py
+++ b/tests/lead/test_ledger_export.py
@@ -2,11 +2,13 @@
 
 from __future__ import annotations
 
+import pytest
 from typer.testing import CliRunner
 
 from cli import app
 from lead.ledger.export import export_project
 from lead.ledger.store import open_ledger
+from prisma.ingest import apply_extraction
 from zot.db import TYPE_MAP, Db
 from zot.schema import create_db
 
@@ -97,3 +99,136 @@ def test_export_is_idempotent_until_the_extraction_version_changes(tmp_path):
 def test_ledger_export_command_is_registered():
     result = runner.invoke(app, ["lead", "ledger", "export", "--help"])
     assert result.exit_code == 0
+
+
+def test_html_extraction_note_stores_the_target_and_a_claim(tmp_path):
+    zpath = str(tmp_path / "z.sqlite")
+    con = create_db(zpath)
+    con.close()
+    with Db(zpath) as zdb:
+        item_id = zdb.create_item(TYPE_MAP["journalArticle"], key="ABCD2345")
+        zdb.set_fields(item_id, {"title": "All-cause readmission"})
+        zdb.tag_item(item_id, "stage:4-included")
+        apply_extraction(
+            zdb,
+            item_id,
+            "lead-precursors",
+            {
+                "outcome_target_id": "Q.ACR",
+                "effect_measure": "OR",
+                "effect_value": "1.2",
+                "effect_ci": "1.1, 1.4",
+                "factor_definition": "frailty",
+                "design": "cohort",
+            },
+        )
+        zdb.commit()
+        ledger = open_ledger(str(tmp_path / "ledger.duckdb"))
+        try:
+            export_project(
+                zdb,
+                ledger,
+                "lead-precursors",
+                extraction_version="v1",
+                actor="tester",
+            )
+            target = ledger.execute(
+                "select target_id from lead_evidence.extraction"
+            ).fetchone()[0]
+            assert target == "Q.ACR"
+            row = ledger.execute(
+                """
+                select effect_value, effect_ci_low, effect_ci_high, subject
+                from lead_evidence.claim
+                """
+            ).fetchone()
+            assert row == (1.2, 1.1, 1.4, "frailty")
+        finally:
+            ledger.close()
+
+
+def test_html_target_label_is_not_stored_as_a_tag(tmp_path):
+    zpath = str(tmp_path / "z.sqlite")
+    con = create_db(zpath)
+    con.close()
+    with Db(zpath) as zdb:
+        item_id = zdb.create_item(TYPE_MAP["journalArticle"], key="ABCD2345")
+        zdb.tag_item(item_id, "project:lead-precursors")
+        zdb.tag_item(item_id, "stage:4-included")
+        zdb.add_note(item_id, "
  • outcome_target_id: Q.ACR
    • ")
+        zdb.commit()
+        ledger = open_ledger(str(tmp_path / "ledger.duckdb"))
+        try:
+            export_project(
+                zdb,
+                ledger,
+                "lead-precursors",
+                extraction_version="v1",
+                actor="tester",
+            )
+            target = ledger.execute(
+                "select target_id from lead_evidence.extraction"
+            ).fetchone()[0]
+            assert target == "Q.ACR"
+        finally:
+            ledger.close()
+
+
+def test_unknown_target_in_a_note_is_not_stored(tmp_path):
+    zpath = str(tmp_path / "z.sqlite")
+    con = create_db(zpath)
+    con.close()
+    with Db(zpath) as zdb:
+        item_id = zdb.create_item(TYPE_MAP["journalArticle"], key="ABCD2345")
+        zdb.tag_item(item_id, "project:lead-limits")
+        zdb.tag_item(item_id, "stage:4-included")
+        zdb.add_note(item_id, "target_id: nope")
+        zdb.commit()
+        ledger = open_ledger(str(tmp_path / "ledger.duckdb"))
+        try:
+            with pytest.raises(KeyError):
+                export_project(
+                    zdb,
+                    ledger,
+                    "lead-limits",
+                    extraction_version="v1",
+                    actor="tester",
+                )
+            count = ledger.execute(
+                "select count(*) from lead_evidence.extraction"
+            ).fetchone()[0]
+            assert count == 0
+        finally:
+            ledger.close()
+
+
+def test_extraction_without_a_target_stores_the_row_and_skips_the_claim(tmp_path):
+    zpath = str(tmp_path / "z.sqlite")
+    con = create_db(zpath)
+    con.close()
+    with Db(zpath) as zdb:
+        item_id = zdb.create_item(TYPE_MAP["journalArticle"], key="ABCD2345")
+        zdb.tag_item(item_id, "project:lead-limits")
+        zdb.tag_item(item_id, "stage:4-included")
+        zdb.add_note(item_id, "Included with no target.")
+        zdb.commit()
+        ledger = open_ledger(str(tmp_path / "ledger.duckdb"))
+        try:
+            counts = export_project(
+                zdb,
+                ledger,
+                "lead-limits",
+                extraction_version="v1",
+                actor="tester",
+            )
+            assert counts["extraction"] == 1
+            target = ledger.execute(
+                "select target_id from lead_evidence.extraction"
+            ).fetchone()[0]
+            assert target is None
+            claims = ledger.execute(
+                "select count(*) from lead_evidence.claim"
+            ).fetchone()[0]
+            assert claims == 0
+        finally:
+            ledger.close()
diff --git a/tests/prisma/test_dedup.py b/tests/prisma/test_dedup.py
index 67083ee..f9bc886 100644
--- a/tests/prisma/test_dedup.py
+++ b/tests/prisma/test_dedup.py
@@ -152,3 +152,32 @@ def test_borderline_items_are_not_tagged(tmp_path):
 def test_dedup_command_is_registered():
     result = runner.invoke(app, ["dedup", "--help"])
     assert result.exit_code == 0
+
+
+def test_dedup_command_prints_borderline_keys_and_titles(tmp_path):
+    path = _db(tmp_path)
+    with Db(path) as db:
+        _add(
+            db,
+            key="ABCD2345",
+            doi="",
+            title="High needs frailty index",
+            year=2024,
+            author="Kim",
+        )
+        _add(
+            db,
+            key="EFGH5678",
+            doi="",
+            title="High-need frailty index",
+            year=2024,
+            author="Lee",
+        )
+    result = runner.invoke(app, ["dedup", "--db", path, "--no-hold"])
+    assert result.exit_code == 0
+    assert "ABCD2345" in result.stdout
+    assert "EFGH5678" in result.stdout
+    assert "High needs frailty index" in result.stdout
+    assert "High-need frailty index" in result.stdout
+    with Db(path) as db:
+        assert _dup_count(db) == 0
diff --git a/tests/prisma/test_search_due.py b/tests/prisma/test_search_due.py
index a15858a..d79c83c 100644
--- a/tests/prisma/test_search_due.py
+++ b/tests/prisma/test_search_due.py
@@ -56,18 +56,17 @@ def _claim(con, *, certainty: str, suffix: str, target_id: str = "Q.ACR") -> Non
     )
 
 
-def _finding(con, *, claim_id: str, suffix: str) -> None:
-    assert append(
-        con,
-        "finding",
-        {
-            "claim_id": claim_id,
-            "relation": "contradicts",
-            "grain": "beneficiary",
-            "actor": "tester",
-            "content_hash": f"finding-{suffix}",
-        },
-    )
+def _finding(con, *, claim_id: str, suffix: str, created_at: str | None = None) -> None:
+    row: dict[str, object] = {
+        "claim_id": claim_id,
+        "relation": "contradicts",
+        "grain": "beneficiary",
+        "actor": "tester",
+        "content_hash": f"finding-{suffix}",
+    }
+    if created_at is not None:
+        row["created_at"] = created_at
+    assert append(con, "finding", row)
 
 
 def test_missing_or_stale_search_is_nightly(tmp_path):
@@ -107,12 +106,38 @@ def test_contradiction_respects_certainty(tmp_path, certainty, expected):
     try:
         _search(con, searched_on=NOW)
         _claim(con, certainty=certainty, suffix=certainty)
-        _finding(con, claim_id=certainty, suffix=certainty)
+        _finding(con, claim_id=certainty, suffix=certainty, created_at="2026-11-01")
         assert due(con, project=PROJECT, now=NOW) == expected
     finally:
         con.close()
 
 
+def test_contradiction_on_the_search_day_does_not_refire(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _search(con, searched_on=NOW)
+        _claim(con, certainty="moderate", suffix="same-day")
+        _finding(
+            con,
+            claim_id="same-day",
+            suffix="same-day",
+            created_at="2026-10-31T10:00:00Z",
+        )
+        assert due(con, project=PROJECT, now=NOW) == ()
+    finally:
+        con.close()
+
+
+def test_contradiction_without_a_search_stays_nightly(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _claim(con, certainty="high", suffix="nosearch")
+        _finding(con, claim_id="nosearch", suffix="nosearch", created_at="2026-11-01")
+        assert due(con, project=PROJECT, now=NOW) == ("nightly",)
+    finally:
+        con.close()
+
+
 def test_exogenous_event_on_a_claimed_target(tmp_path):
     con = _ledger(tmp_path)
     try:
@@ -124,7 +149,7 @@ def test_exogenous_event_on_a_claimed_target(tmp_path):
                 "project": PROJECT,
                 "target_id": "Q.ACR",
                 "series_or_event": "cms guidance",
-                "event_on": NOW,
+                "event_on": "2026-11-01",
                 "citation": "cms",
                 "actor": "tester",
                 "content_hash": "exo-1",
@@ -137,6 +162,49 @@ def test_exogenous_event_on_a_claimed_target(tmp_path):
         con.close()
 
 
+def test_exogenous_on_the_search_day_does_not_refire(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _search(con, searched_on=NOW)
+        _claim(con, certainty="low", suffix="same-day")
+        assert append(
+            con,
+            "exogenous",
+            {
+                "project": PROJECT,
+                "target_id": "Q.ACR",
+                "series_or_event": "cms guidance",
+                "event_on": NOW,
+                "citation": "cms",
+                "actor": "tester",
+                "content_hash": "exo-same-day",
+            },
+        )
+        assert due(con, project=PROJECT, now=NOW) == ()
+    finally:
+        con.close()
+
+
+def test_exogenous_without_a_search_stays_nightly(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _claim(con, certainty="low", suffix="nosearch")
+        assert append(
+            con,
+            "exogenous",
+            {
+                "target_id": "Q.ACR",
+                "series_or_event": "cms guidance",
+                "event_on": "2026-11-01",
+                "actor": "tester",
+                "content_hash": "exo-nosearch",
+            },
+        )
+        assert due(con, project=PROJECT, now=NOW) == ("nightly",)
+    finally:
+        con.close()
+
+
 def test_cms_source_newer_than_the_search(tmp_path):
     con = _ledger(tmp_path)
     try:
@@ -171,12 +239,75 @@ def test_cms_source_newer_than_the_search(tmp_path):
         con.close()
 
 
+def test_cms_iso_timestamp_on_the_search_day_does_not_fire(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _search(con, searched_on=NOW)
+        assert append(
+            con,
+            "source",
+            {
+                "project": PROJECT,
+                "database": "cms",
+                "zotero_key": "ABCD2345",
+                "created_at": "2026-10-31T10:00:00Z",
+                "actor": "tester",
+                "content_hash": "src-iso-same",
+            },
+        )
+        assert due(con, project=PROJECT, now=NOW) == ()
+    finally:
+        con.close()
+
+
+def test_joined_cms_on_the_search_day_does_not_fire(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _search(con, searched_on=NOW)
+        assert append(
+            con,
+            "source",
+            {
+                "project": PROJECT,
+                "database": "citation,cms",
+                "zotero_key": "IJKM2345",
+                "created_at": "2026-10-31T10:00:00Z",
+                "actor": "tester",
+                "content_hash": "src-joined-same",
+            },
+        )
+        assert due(con, project=PROJECT, now=NOW) == ()
+    finally:
+        con.close()
+
+
+def test_cms_token_in_a_joined_database_fires_after_the_search_day(tmp_path):
+    con = _ledger(tmp_path)
+    try:
+        _search(con, searched_on=NOW)
+        assert append(
+            con,
+            "source",
+            {
+                "project": PROJECT,
+                "database": "citation,cms",
+                "zotero_key": "EFGH5678",
+                "created_at": "2026-11-01T00:00:00Z",
+                "actor": "tester",
+                "content_hash": "src-iso-later",
+            },
+        )
+        assert due(con, project=PROJECT, now=NOW) == ("cms",)
+    finally:
+        con.close()
+
+
 def test_reasons_stay_in_order(tmp_path):
     con = _ledger(tmp_path)
     try:
         _search(con, searched_on=YESTERDAY, suffix="old")
         _claim(con, certainty="high", suffix="high")
-        _finding(con, claim_id="high", suffix="high")
+        _finding(con, claim_id="high", suffix="high", created_at="2026-11-01")
         assert append(
             con,
             "exogenous",
diff --git a/tests/prisma/test_search_pubmed.py b/tests/prisma/test_search_pubmed.py
index 778fa0b..040bb92 100644
--- a/tests/prisma/test_search_pubmed.py
+++ b/tests/prisma/test_search_pubmed.py
@@ -17,6 +17,7 @@ from prisma.search.base import (
     record,
     seed_recall,
 )
+from prisma.search.due import due
 from prisma.search.protocols import PROTOCOLS
 from prisma.search.pubmed import parse_pubmed_xml
 from zot.db import TYPE_MAP, Db
@@ -93,6 +94,21 @@ def test_same_xml_on_two_days_appends_two_search_rows(tmp_path):
         con.close()
 
 
+def test_record_with_project_clears_due_and_keeps_projects_apart(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        result = parse_pubmed_xml(_XML, query="frailty", searched_on="2026-10-31")
+        assert record(con, result, actor="tester", project="lead-precursors") is True
+        assert record(con, result, actor="tester", project="lead-precursors") is False
+        assert due(con, project="lead-precursors", now="2026-10-31") == ()
+        assert record(con, result, actor="tester", project="lead-history") is True
+        count = con.execute("select count(*) from lead_evidence.search").fetchone()[0]
+        assert count == 2
+        assert due(con, project="lead-history", now="2026-10-31") == ()
+    finally:
+        con.close()
+
+
 def test_import_hits_is_idempotent_on_doi(tmp_path):
     path = str(tmp_path / "z.sqlite")
     con = create_db(path)
@@ -138,6 +154,74 @@ def test_import_hits_is_idempotent_on_doi(tmp_path):
         assert articles == 1
 
 
+def test_import_hits_tags_an_existing_doi_for_a_later_strand(tmp_path):
+    path = str(tmp_path / "z.sqlite")
+    con = create_db(path)
+    con.close()
+    hit = SearchHit(
+        title="Alpha frailty",
+        year=2020,
+        doi="10.1/alpha",
+        pmid="111",
+        first_author="Kim",
+        database="pubmed",
+    )
+    log = SearchLog(
+        database="pubmed",
+        interface="eutils",
+        searched_on="2026-10-31",
+        query="frailty",
+        filters="",
+        hit_count=1,
+        export_sha256="abc",
+    )
+    first = SearchResult(hits=(hit,), log=log)
+    second = SearchResult(
+        hits=(
+            SearchHit(
+                title="Alpha frailty",
+                year=2020,
+                doi="10.1/alpha",
+                pmid="111",
+                first_author="Kim",
+                database="openalex",
+            ),
+        ),
+        log=SearchLog(
+            database="openalex",
+            interface="openalex",
+            searched_on="2026-10-31",
+            query="frailty",
+            filters="",
+            hit_count=1,
+            export_sha256="def",
+        ),
+    )
+    with Db(path) as db:
+        assert import_hits(db, "lead-precursors", first) == 1
+        assert import_hits(db, "lead-limits", second) == 0
+        articles = db.con.execute(
+            "select count(*) from items where itemTypeID = ?",
+            (TYPE_MAP["journalArticle"],),
+        ).fetchone()[0]
+        assert articles == 1
+        names = {
+            row[0]
+            for row in db.con.execute(
+                """
+                SELECT t.name FROM tags t
+                JOIN itemTags it ON it.tagID = t.tagID
+                """
+            )
+        }
+        assert names >= {
+            "project:lead-precursors",
+            "project:lead-limits",
+            "found-in:pubmed",
+            "found-in:openalex",
+        }
+
+
 def test_protocols_cover_the_five_projects():
     assert set(PROTOCOLS) == {
         "lead-history",
-- 
2.49.1


From 9788f95ede2905e4b2a7f5fb3d594f8710baad4a Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 21:31:29 -0400
Subject: [PATCH 20/30] fix(perf): let the collector trace fixture replace an
 existing provider

OpenTelemetry ignores a second set_tracer_provider. The full suite sets one from the BCDA metrics test, so the collector's first span test saw no spans.
---
 tests/perf/test_collector.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/perf/test_collector.py b/tests/perf/test_collector.py
index ee7d0e8..6f84d45 100644
--- a/tests/perf/test_collector.py
+++ b/tests/perf/test_collector.py
@@ -32,6 +32,9 @@ def trace_setup(monkeypatch):
     exporter = _MemoryExporter()
     provider = TracerProvider()
     provider.add_span_processor(SimpleSpanProcessor(exporter))
+    # OpenTelemetry ignores a second set_tracer_provider until this flag
+    # is cleared. An earlier test may already have installed one.
+    trace._TRACER_PROVIDER_SET_ONCE._done = False
     trace.set_tracer_provider(provider)
 
     # Make perf.tracer() return real tracers from this provider
-- 
2.49.1


From 289649b3ffd7fd52f79860e5ef526c046a644a48 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:00:04 -0400
Subject: [PATCH 21/30] docs(lead): design the kernel, loop, and tool surface

---
 .../plans/2026-10-31-lead-kernel-and-loop.md  | 268 ++++++++++++++++++
 .../2026-10-31-lead-kernel-and-loop-design.md | 140 +++++++++
 2 files changed, 408 insertions(+)
 create mode 100644 docs/superpowers/plans/2026-10-31-lead-kernel-and-loop.md
 create mode 100644 docs/superpowers/specs/2026-10-31-lead-kernel-and-loop-design.md

diff --git a/docs/superpowers/plans/2026-10-31-lead-kernel-and-loop.md b/docs/superpowers/plans/2026-10-31-lead-kernel-and-loop.md
new file mode 100644
index 0000000..d76300b
--- /dev/null
+++ b/docs/superpowers/plans/2026-10-31-lead-kernel-and-loop.md
@@ -0,0 +1,268 @@
+# LEAD Kernel, Loop, and Tools Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Close the eight Part 1 review minors and build the computable Parts 2–4 of the LEAD chassis on public data, without inventing CMS coefficients.
+
+**Architecture:** Pure functions implement the published thresholds, the existing benchmark cap, and the promotion gates. DuckDB ledger rows record rules, releases, runs, and findings. The agent surface is a Python module that writes only those rows. Databricks and model training stay behind an injected transport and a recorded gap.
+
+**Tech Stack:** Python, Typer, DuckDB, pytest. No new service. No MCP SDK. No MLflow client.
+
+**Spec:** `docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md` and `docs/superpowers/specs/2026-10-31-lead-kernel-and-loop-design.md`
+
+## Global Constraints
+
+- Public data only. No PHI, CCLF, or client data. No live HTTP in unit tests.
+- Do not invent V28, concurrent, ESRD, Kim, Appendix B, DAH, TFU, or CAHPS coefficients.
+- A rule-tier claim's number must be one the kernel tests against a literal.
+- Ledger inserts stay append-only. The same content hash inserts nothing. Claim hashes ignore the actor.
+- `schema_sql("delta")` must not contain the substring `aco`.
+- An API or transport failure writes no screening decision and no run row.
+- `stack prisma search due` still exits 0.
+- Work on `feat/lead-evidence-chassis-part1` in the existing worktree. Do not remove `.claude/worktrees/`.
+- Do not install a crontab. Do not train a model. Do not spend Databricks.
+
+## Review Focus
+
+- An A&D score of exactly 3.0 meets HN.4 and does not meet HN.6. An ESRD score of exactly 0.35 meets HN.5 and does not meet HN.6.
+- A suspect with equal demographic weights raises. The care delta is not multiplied by the weight ratio.
+- `runs_start(..., where_ran="databricks")` raises on an exhausted budget and on a missing transport, and the transport is not called in either case.
+- `load_manifest` stores a row whose `path` does not exist and does not open `url`.
+- `agreed_excludes` raises when an id is present only in the second pass. Undated near-titles are not borderline. Identical undated titles with the same author still merge.
+
+---
+
+### Task 1: Part 1 review minors
+
+**Files:**
+- Modify: `src/prisma/search/base.py` (`bare_doi`, `_doi_item_id`)
+- Modify: `src/prisma/dedup.py` (`_doi_key`, borderline year compare)
+- Modify: `src/prisma/audit.py` (`agreed_excludes`, `audit_sample`)
+- Modify: `src/prisma/search/pubmed.py` (`parse_pubmed_xml`)
+- Modify: `src/prisma/search/openalex.py`, `crossref.py`, `semanticscholar.py`, `arxiv.py`
+- Modify: `src/econ/fred.py` (`SeriesRow.year`)
+- Modify: `src/cli/lead.py` (status printer)
+- Test: `tests/prisma/test_dedup.py`, `tests/prisma/test_audit.py`, `tests/prisma/test_search_pubmed.py`, `tests/prisma/test_search_others.py`, `tests/econ/test_fred_irs.py`, `tests/lead/test_ledger_store.py`
+
+**Interfaces:**
+- Consumes: `zot.fetch.select.normalise_doi(raw: str) -> str`. `status()` already returns `TargetStatus`.
+- Produces: `bare_doi` delegates to `normalise_doi` and keeps a trailing period. `pmid_of(value: object) -> str` in `prisma.search.base`. Status stdout line: `{id}\t{gaps}\t{last_run or ""}\t{open_contradictions}\t{rule}\t{aggregate}\t{literature}\t{synthetic}`.
+
+- [ ] **Step 1: Write the failing tests**
+
+DOI URL and bare DOI are one dedup group. `bare_doi("https://dx.doi.org/10.1/ab.")` stays `10.1/ab.`. Import of a bare DOI finds an item stored as `https://dx.doi.org/10.1/alpha` and returns 0. OpenAlex `ids.pmid` URL `https://pubmed.ncbi.nlm.nih.gov/42` yields pmid `42`. Semantic Scholar `externalIds.PubMed` `"999"` yields `999`. Crossref `"pmid": "12345"` yields `12345`. An arXiv `arxiv:pmid` of `77` yields `77`. Nine agreed-exclude ids at rate 0.10 draw one member of that list. An id only in the second pass raises `KeyError` from `agreed_excludes`. Blank PubMed XML `"  \n"` returns no hits and `sha256` of that payload. A FRED observation with `date: ""` has `year is None`. Undated near-titles `High needs frailty index` and `High-need frailty index` have an empty borderline list. Identical undated titles with author `Kim` are one group. Empty-ledger status stdout contains `Q.ACR\t1\t\t0\t0\t0\t0\t0`.
+
+- [ ] **Step 2: Run the new tests and confirm they fail**
+
+Run the new test names with `uv run pytest` from the worktree. Expected: assertion failures, not collection errors.
+
+- [ ] **Step 3: Implement the minors**
+
+`pmid_of` keeps digits after a PubMed URL or a `pmid:` prefix, else `""`. Parsers call it only when the source field exists. `agreed_excludes` uses the same key-set check as `disagreements`. `audit_sample` draws 1 when the floored count is 0, the list is nonempty, and the rate is positive. Blank PubMed XML returns a log with interface `eutils` and database `pubmed`. FRED year is `int | None`. Borderline comparison skips a missing year. Status prints the eight fields in the order above.
+
+- [ ] **Step 4: Re-run the touched tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`fix: close the part 1 review minors`
+
+### Task 2: High Needs criteria and suspect moves
+
+**Files:**
+- Create: `src/lead/high_needs.py`
+- Create: `src/lead/suspect.py`
+- Test: `tests/lead/test_high_needs.py`, `tests/lead/test_suspect.py`
+
+**Interfaces:**
+- Consumes: nothing from Task 1.
+- Produces: `QuarterFlags`, `criteria_met(flags) -> frozenset[str]`, `alignment_year(quarters) -> frozenset[str]`, `persists(prior, this_year) -> frozenset[str]`. `SuspectMove(coding_delta, care_delta, coding_after_reweight)`. `suspect_move(*, before_status, after_status, score_before, score_coding, score_care, weight_before, weight_after) -> SuspectMove`.
+
+- [ ] **Step 1: Write the failing tests**
+
+`QuarterFlags(quarter="2027Q1", ad_score=3.0)` meets `HN.4` and not `HN.6`. `ad_score=2.0` and `unplanned_admissions=2` meets `HN.6`. `ad_score=2.0` and one admission meets neither. `esrd_score=0.35` meets `HN.5` and not `HN.6`. `esrd_score=0.24` and two admissions meets `HN.6`. `snf_days=45` meets `HN.7`; 44 does not. `hn1_diagnosis=True` meets `HN.1`. `hn2_dme=True` meets `HN.2`. `kim_index=0.2, kim_cutoff=0.2` meets `HN.3`. A missing index or a missing cutoff does not. One quarter of four meeting `HN.7` sets `HN.7` for the year. `persists(frozenset({"HN.1"}), frozenset())` is `{HN.1}`.
+
+`suspect_move` with before `standard`, after `high_needs`, scores 1.0, 1.2, and 1.5, weights 1.0 and 0.5 returns `SuspectMove(0.2, 0.5, 0.1)`. The same status raises `ValueError`. Equal weights raise `ValueError`. `weight_before=0` raises `ValueError`.
+
+- [ ] **Step 2: Run the tests**
+
+Expected: import or attribute error because the modules do not exist.
+
+- [ ] **Step 3: Implement the two modules**
+
+Thresholds are literals `3.0`, `0.35`, `2.0`, `0.24`, `2`, and `45` in the module. Bands are half-open at the top. `suspect_move` subtracts `score_before` from each score and scales only `coding_delta`.
+
+- [ ] **Step 4: Re-run the tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`feat(lead): score High Needs criteria and separate suspect moves`
+
+### Task 3: Limits calculator and rule-tier claims
+
+**Files:**
+- Create: `src/lead/limits.py`
+- Create: `src/lead/rules.py`
+- Test: `tests/lead/test_limits.py`, `tests/lead/test_rules.py`
+
+**Interfaces:**
+- Consumes: `capped_risk_score`, `CAPPED`, `WITHIN_CAP` from `lead.benchmark`. `append` and `open_ledger`. `criteria_met` is not required. Targets `RA.CAP`, `HN.4`, `HN.5`, `HN.6`, `HN.7`.
+- Produces: `Headroom(scored, status, forgone)`. `earlier_qualification_headroom(prior, caught) -> Headroom`. `normalization_sensitivity(benchmark, factor) -> float`. `shrinkage_weight(aco_n, prior_n) -> float`. `publish_rules(con, *, actor) -> int`.
+
+- [ ] **Step 1: Write the failing tests**
+
+`earlier_qualification_headroom(1.0, 1.05)` is `Headroom(1.04, CAPPED, 0.01)`. `earlier_qualification_headroom(1.0, 1.035)` is `Headroom(1.035, WITHIN_CAP, 0.0)`. `normalization_sensitivity(1000.0, 1.25)` is `800.0`. Factor `0` raises `ValueError`. `shrinkage_weight(100, 300)` is `0.25`. `aco_n + prior_n == 0` raises `ValueError`.
+
+`publish_rules` on an empty ledger inserts claims whose `tiers` is `rule` and whose `effect_value` values include `0.04` on `RA.CAP`, `3.0` on `HN.4`, `0.35` on `HN.5`, `2.0` on `HN.6`, and `45.0` on `HN.7`. A second call returns 0. The HN.6 subject says the A&D band stops before 3.0 and the ESRD band stops before 0.35.
+
+- [ ] **Step 2: Run the tests**
+
+Expected: import error.
+
+- [ ] **Step 3: Implement**
+
+Headroom calls `capped_risk_score(prior, caught, "High Needs")`. Forgone is `max(caught - scored, 0)`. Rule claims use a content hash of kind, target, and effect value, not the actor. `publish_rules` counts inserts that returned true.
+
+- [ ] **Step 4: Re-run the tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`feat(lead): add the limits calculator and rule-tier claims`
+
+### Task 4: Quality gaps, manifest loader, geography spine
+
+**Files:**
+- Create: `src/lead/quality.py`
+- Create: `src/lead/manifest.py`
+- Create: `src/lead/spine.py`
+- Modify: `src/cli/lead.py` (`manifest load`)
+- Test: `tests/lead/test_quality.py`, `tests/lead/test_manifest.py`, `tests/lead/test_spine.py`
+
+**Interfaces:**
+- Consumes: `open_ledger`. `uamcc_summary` and `acr_summary` in `aco.express.cms_quality_measures`.
+- Produces: `known_gaps() -> tuple[tuple[str, str], ...]`, `reconciled() -> tuple[str, ...]`. `load_manifest(con, csv_path, *, actor) -> int`. `layer_of(source_id, lead_use) -> str`. `geography_key(*, fips="", zcta="") -> str`. `time_key(year, quarter) -> str`. CLI `stack lead manifest load --csv --db`.
+
+- [ ] **Step 1: Write the failing tests**
+
+`reconciled()` is `("Q.UAMCC", "Q.ACR")` and those summary functions import. `known_gaps()` target ids include `Q.DAH`, `Q.TFU`, `Q.CAHPS`, `HN.1`, `HN.2`, `HN.3`, `RA.PROSP`, `RA.CONC`, and `RA.ESRD`.
+
+A two-row CSV inserts 2, and the same file inserts 0 the second time. A row with `lead_use` `synthetic person file` has layer `synthetic`. `source_id` `tuva` is synthetic with a blank `lead_use`. A public row has layer `""`. A blank `sha256` raises `ValueError`. A `path` of `/no/such/lead-file` still inserts, and the stored `url` is the CSV value. The loader DDL string does not contain `aco`. `schema_sql("delta")` still does not contain `aco`.
+
+`geography_key(fips="01001")` is `01001`. A 4-digit FIPS raises. `geography_key(zcta="02138")` is `02138`. Both FIPS and ZCTA raise `ValueError`. `time_key(2027, 4)` is `2027Q4`. Quarter `5` raises.
+
+CLI load prints `inserted\t1` for a one-row file.
+
+- [ ] **Step 2: Run the tests**
+
+Expected: import error, then assertion failures once the modules exist. Watch the first run fail before implementing.
+
+- [ ] **Step 3: Implement**
+
+CSV columns are the enrichment manifest header. The table is `lead_evidence.public_release`, created by the loader. Hash is `source_id`, `release`, `sha256`. Do not open `url` and do not read `path`. Spine validators are the whole geography story.
+
+- [ ] **Step 4: Re-run the tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`feat(lead): load the public manifest and record measure gaps`
+
+### Task 5: Promotion gates, failure record, drift, lineage
+
+**Files:**
+- Create: `src/lead/gates.py`
+- Modify: `src/aco/lake/lineage.py`
+- Test: `tests/lead/test_gates.py`, `tests/lead/test_ledger_lineage.py`
+
+**Interfaces:**
+- Consumes: `append`, `open_ledger`, `by_id`. `LineageGraph`.
+- Produces: `GateInput`, `GateResult`, `promotion_gates(item) -> GateResult`, `record_outcome(con, *, actor, target_id, model, where_ran, finished_on, metrics, gate, relation, claim_id="") -> tuple[bool, bool]`, `events_in_window(*, window_start, alarm_on, events) -> tuple[str, ...]`. `LineageGraph.table_outputs(table_ref) -> list[str]`. Ledger edges on `build_lineage()`.
+
+- [ ] **Step 1: Write the failing tests**
+
+A gate input that beats both baselines, tests `train_through + 1`, has calibration error `0.01` against tolerance `0.05`, coverage `0.91` against target `0.90`, `subgroup_pass=True`, and a known claim id is promoted with empty reasons. Each single failure adds exactly one of `baseline`, `out_of_time`, `calibration`, `coverage`, `subgroup`, `claim`. An empty feature list is `claim`. A test year two years ahead is `out_of_time`.
+
+`record_outcome` with relation `discarded` and a failing gate inserts one run and one finding. The same call again returns `(False, False)`. Relation `promoted` with a passing gate also inserts both. Relation `other` raises `ValueError` and leaves the tables empty.
+
+An event on the window start and an event on `alarm_on` are returned. The day before and the day after are not.
+
+`build_lineage().table_deps("lead_evidence.claim")` includes `lead_evidence.extraction` and `lead_evidence.source`. `table_outputs("lead_evidence.claim")` includes `lead_evidence.feature`, `lead_evidence.run`, and `lead_evidence.finding`.
+
+- [ ] **Step 2: Run the tests**
+
+Expected: import error for `lead.gates`, and `AttributeError` for `table_outputs` if the lineage test runs first.
+
+- [ ] **Step 3: Implement**
+
+Reason order is the list above. `record_outcome` hashes the target, model, where, finished day, metrics, reasons, and relation, not the actor. Finding `relation` is the caller's relation. `gate_results` on the run is the comma-joined reasons. Lineage edges are added after the pipeline walk.
+
+- [ ] **Step 4: Re-run the tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`feat(lead): gate promotion and record discarded runs`
+
+### Task 6: Agent tool surface
+
+**Files:**
+- Create: `src/lead/tools.py`
+- Create: `src/lead/model.py`
+- Test: `tests/lead/test_tools.py`, `tests/lead/test_model.py`
+
+**Interfaces:**
+- Consumes: `append`, `open_ledger`, `by_id`, `due`, `earlier_qualification_headroom`, `normalization_sensitivity`, `shrinkage_weight`, `perf.tracer`.
+- Produces: `ledger_search`, `ledger_get_claim`, `targets_list`, `limits_compute`, `features_validate`, `features_propose`, `runs_start`, `runs_get`, `runs_compare`, `failures_check`, `findings_record`, `review_trigger_search`. `stamp(weights_hash, adapter_version) -> str`. `MODEL_GAPS`.
+
+- [ ] **Step 1: Write the failing tests**
+
+`ledger_search` filters by `target_id` and a casefolded subject substring. `ledger_get_claim` returns that claim and `None` for an unknown id. `targets_list()` includes `Q.ACR` and `HN.4`.
+
+`limits_compute("headroom", prior=1.0, caught=1.05).scored` is `1.04`. `limits_compute("normalization", benchmark=1000.0, factor=1.25)` is `800.0`. `limits_compute("shrinkage", aco_n=100, prior_n=300)` is `0.25`. An unknown kind raises `ValueError`.
+
+`features_validate` raises `KeyError` for target `NO.SUCH` and `ValueError` when a claim id is absent or the list is empty. `features_propose` inserts one feature and a second call with the same expression inserts nothing.
+
+`runs_start(..., where_ran="local")` inserts a run and does not call a transport that would raise. `where_ran="databricks"` with `transport=None` raises `RuntimeError`. `where_ran="databricks"` with `spent=1` and `budget=1` raises `RuntimeError` and does not call the transport. A transport and `spent=0`, `budget=1` is called once and the run is stored. `runs_get` returns that run. `runs_compare` returns both metric strings. A missing id raises `KeyError`.
+
+`failures_check` returns the run whose `gate_results` is nonempty and omits a promoted run. `findings_record` inserts one finding. `review_trigger_search` on an empty ledger returns `("nightly",)`.
+
+A recording tracer passed as `tracer_factory` receives the span name `ledger_search`. The default factory is `perf.tracer`.
+
+`stamp("abc", "v1")` is `abc+v1`. An empty hash or an empty version raises `ValueError`. `MODEL_GAPS` is nonempty.
+
+- [ ] **Step 2: Run the tests**
+
+Expected: import error.
+
+- [ ] **Step 3: Implement**
+
+Databricks checks the budget first, then the transport, then calls it, then appends. A raise writes no run. Feature and run hashes ignore the actor. `failures_check` reads `run.gate_results`. `review_trigger_search` calls `due` and does not search. Every public function uses the span helper.
+
+- [ ] **Step 4: Re-run the tests**
+
+Expected: PASS.
+
+- [ ] **Step 5: Commit**
+
+`feat(lead): add the agent tool surface`
+
+### Task 7: Whole-package verification
+
+- [ ] **Step 1: Run the touched packages, then the full suite**
+
+From the worktree: `uv run pytest tests/lead tests/prisma tests/econ tests/aco -q`, then `uv run pytest -q`. Expected: the suite passes. A failure that this work did not cause is still reported by name.
+
+- [ ] **Step 2: Commit any fix the suite required**
+
+Message names the failing test.
+
+- [ ] **Step 3: Final review of this plan's range**
+
+Review `e3def65..HEAD` only for the commits after Part 1's `9788f95`. Do not re-review Part 1. Critical and Important findings get one TDD fix pass. Minors are ledgered.
diff --git a/docs/superpowers/specs/2026-10-31-lead-kernel-and-loop-design.md b/docs/superpowers/specs/2026-10-31-lead-kernel-and-loop-design.md
new file mode 100644
index 0000000..93045a7
--- /dev/null
+++ b/docs/superpowers/specs/2026-10-31-lead-kernel-and-loop-design.md
@@ -0,0 +1,140 @@
+# LEAD kernel, modeling loop, and agent tools
+
+Status: builder design for Parts 2–4 of the approved chassis spec
+`docs/superpowers/specs/2026-10-31-lead-evidence-chassis-design.md`.
+The user told the executor to do the remaining work without another
+approval pause. This file records the calls the outline left open.
+It does not replace the approved spec.
+
+Public data only. No PHI, CCLF, or client data. No live HTTP and no
+Databricks call in unit tests. Unpublished CMS coefficient tables are
+not invented. A number enters the `rule` tier only when a function in
+this repo reproduces it.
+
+## What is computed
+
+High Needs criteria take caller-supplied quarter flags. The thresholds
+are the ones in the approved spec:
+
+- HN.1 and HN.2 are booleans. Appendix B code lists are not in this repo.
+- HN.3 is `kim_index >= kim_cutoff`, both supplied by the caller. Kim
+  weights and the cutoff are not in this repo.
+- HN.4 is an A&D score `>= 3.0`. The score is an input. V28 software is
+  not parsed.
+- HN.5 is an ESRD score `>= 0.35`. ESRD model internals are not parsed.
+- HN.6 is an A&D score `2.0 <= score < 3.0`, or an ESRD score
+  `0.24 <= score < 0.35`, plus at least 2 unplanned admissions. The
+  score 3.0 is HN.4 only. The score 0.35 is HN.5 only.
+- HN.7 is at least 45 covered SNF days.
+
+Any quarter that meets a criterion sets it for the whole alignment year.
+A criterion met in a prior year stays met. The 12-month counts are
+caller-supplied. This module does not read claims.
+
+A suspect is a status change. Coding and care stay separate numbers.
+Demographic reweighting multiplies only the coding delta:
+`coding_delta * (weight_after / weight_before)`. The same status raises.
+Equal weights raise. There is no single collapsed delta.
+
+The limits calculator:
+
+- Headroom calls `lead.benchmark.capped_risk_score` with category
+  `High Needs`. The published check is prior 1.0 and caught 1.05, which
+  returns 1.04 and the capped status. Forgone growth is the positive
+  difference between the uncapped score and the capped score.
+- Normalization is `benchmark / factor` for a caller-supplied factor
+  greater than zero. The CMS normalization factor is not invented.
+- Shrinkage is `aco_n / (aco_n + prior_n)`. That is this chassis's
+  sensitivity formula, not a CMS coefficient, so it is not a rule-tier
+  claim.
+
+UAMCC and ACR stay in `src/aco/express/cms_quality_measures.py`. DAH,
+TFU, and CAHPS are recorded gaps. CAHPS results would be an input.
+
+Rule-tier claims are written for the published identities the kernel
+checks: the 4 percent High Needs cap (`RA.CAP`), and the HN.4, HN.5,
+HN.6, and HN.7 thresholds. Re-inserting the same claim hash inserts
+nothing. The actor is not part of the hash.
+
+## What is loaded
+
+`lead.manifest.load_manifest` reads a caller-supplied CSV with the
+enrichment manifest columns. It inserts into `lead_evidence.public_release`
+in the local DuckDB file. The table DDL lives with the loader, not in
+`lead.ledger.schema.schema_sql`, so the Delta DDL stays free of the
+substring `aco`. A row is identified by `source_id`, `release`, and
+`sha256`. The same triple inserts nothing. A blank key raises. The
+loader does not download, and a missing file at `path` is still stored.
+
+Geography accepts one 5-digit county FIPS or one 5-digit ZCTA, and a
+year plus a quarter in 1..4. Supplying both a FIPS and a ZCTA raises.
+There is no national crosswalk.
+
+A row is tagged `synthetic` when `lead_use` contains that word or
+`source_id` is one of `bcda_sandbox`, `tuva`, `cms_synthetic`,
+`de_synpuf`, `synthea`.
+
+## Modeling loop
+
+Promotion is a pure function. A candidate is promoted only when all of
+these hold:
+
+- it beats AutoML and it beats a naive or persistence baseline
+- `test_year == train_through + 1`
+- calibration error is at most the tolerance
+- conformal coverage is at least the target
+- the caller reports that dual status, SVI quartile, and rurality checks
+  passed
+- every feature lists a claim id that already exists, and there is at
+  least one feature
+
+A failed or discarded run appends one `run` row and one `finding` row.
+The same hashes insert nothing. Drift annotation returns the strand-5
+event citations whose `event_on` day falls inside the window, compared
+on the `YYYY-MM-DD` prefix, including both ends.
+
+The ledger tables are added to the lakehouse lineage graph so a claim's
+inputs and a claim's outputs can both be listed. Pipeline lineage is
+unchanged.
+
+## Agent surface
+
+`lead.tools` is the tool surface. The MCP SDK is not a project
+dependency, so these are Python functions plus the existing Typer
+commands. Each call opens a `lead.tools` span through `perf.tracer`.
+The agent writes ledger rows only.
+
+`runs.start` with `where_ran="local"` appends a run and does not call a
+transport. `where_ran="databricks"` raises when `spent >= budget`, and
+raises when no transport was injected. Both raises happen before a
+transport call. A passed transport is called only after those checks.
+Unit tests inject the transport. Nothing in this package opens a
+Databricks connection or trains a model.
+
+LoRA training, model serving, and the strand-3 model pick are recorded
+gaps. `lead.model.stamp` only concatenates a weights hash and an adapter
+version. An empty half raises.
+
+`stack lead ledger status` prints, per target,
+`id, gaps, last_run, open_contradictions, rule, aggregate, literature,
+synthetic`, tab-separated. An empty ledger prints a blank last run and
+a gap of 1. `stack lead manifest load` prints how many rows were new.
+
+## Deferred Part 1 minors, now in scope
+
+- DOI keys use `zot.fetch.select.normalise_doi`. `bare_doi` does not
+  strip a trailing period. Dedup keys do. An existing Zotero DOI stored
+  as a URL matches a bare DOI.
+- PMID is read from OpenAlex `ids.pmid`, Semantic Scholar
+  `externalIds.PubMed` or `PMID`, a Crossref `pmid` field, and an arXiv
+  `pmid` element. The value is digits only, after a `pmid:` prefix or a
+  PubMed URL. Absent fields stay empty.
+- The 10 percent audit draws one id when the floored count is zero and
+  the rate and the list are both nonempty. Rate 0 stays empty.
+- `agreed_excludes` raises `KeyError` when the two passes name different
+  ids.
+- Blank PubMed XML returns zero hits and the sha256 of that payload.
+- A FRED date whose first four characters are not digits stores year
+  `None`.
+- Near-title borderline pairs require both years to be present and equal.
+  Identical undated titles with the same author still merge.
-- 
2.49.1


From 080df238b3af2029e8b4204b11574ddee9f859a0 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:00:04 -0400
Subject: [PATCH 22/30] fix: close the part 1 review minors

---
 src/cli/lead.py                      |  8 +++-
 src/econ/fred.py                     |  4 +-
 src/prisma/audit.py                  | 21 ++++++++--
 src/prisma/dedup.py                  |  5 ++-
 src/prisma/search/arxiv.py           |  5 ++-
 src/prisma/search/base.py            | 57 ++++++++++++++++++++--------
 src/prisma/search/crossref.py        | 11 +++++-
 src/prisma/search/openalex.py        |  4 +-
 src/prisma/search/pubmed.py          | 18 ++++++++-
 src/prisma/search/semanticscholar.py | 11 +++++-
 tests/econ/test_fred_irs.py          |  6 +++
 tests/lead/test_ledger_store.py      |  7 ++++
 tests/prisma/test_audit.py           | 12 ++++++
 tests/prisma/test_dedup.py           | 43 +++++++++++++++++++++
 tests/prisma/test_search_others.py   | 55 +++++++++++++++++++++++++++
 tests/prisma/test_search_pubmed.py   | 48 +++++++++++++++++++++++
 16 files changed, 284 insertions(+), 31 deletions(-)

diff --git a/src/cli/lead.py b/src/cli/lead.py
index 7312698..e812f82 100644
--- a/src/cli/lead.py
+++ b/src/cli/lead.py
@@ -28,7 +28,13 @@ def ledger_status_cmd(
     con = open_ledger(db)
     try:
         for row in ledger_status(con):
-            typer.echo(row.target_id)
+            counts = row.claims_by_tier
+            typer.echo(
+                f"{row.target_id}\t{row.gaps}\t{row.last_run or ''}\t"
+                f"{row.open_contradictions}\t{counts['rule']}\t"
+                f"{counts['aggregate']}\t{counts['literature']}\t"
+                f"{counts['synthetic']}"
+            )
     finally:
         con.close()
 
diff --git a/src/econ/fred.py b/src/econ/fred.py
index 62ba971..7774836 100644
--- a/src/econ/fred.py
+++ b/src/econ/fred.py
@@ -9,7 +9,7 @@ from dataclasses import dataclass
 @dataclass(frozen=True)
 class SeriesRow:
     series_id: str
-    year: int
+    year: int | None
     period: str
     value: float | None
 
@@ -33,7 +33,7 @@ def parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]:
         if not isinstance(item, Mapping):
             continue
         date = str(item.get("date") or "")
-        year = int(date[:4]) if len(date) >= 4 and date[:4].isdigit() else 0
+        year = int(date[:4]) if len(date) >= 4 and date[:4].isdigit() else None
         rows.append(
             SeriesRow(
                 series_id=str(item.get("series_id") or ""),
diff --git a/src/prisma/audit.py b/src/prisma/audit.py
index 920fcff..b94bb0f 100644
--- a/src/prisma/audit.py
+++ b/src/prisma/audit.py
@@ -14,7 +14,13 @@ def disagreements(a: Mapping[str, str], b: Mapping[str, str]) -> tuple[str, ...]
 
 
 def agreed_excludes(a: Mapping[str, str], b: Mapping[str, str]) -> tuple[str, ...]:
-    """Sorted ids both passes labeled exclude."""
+    """Sorted ids both passes labeled exclude.
+
+    Both passes must name the same ids. An id in only one pass raises
+    ``KeyError``, the same rule as :func:`disagreements`.
+    """
+    if set(a) != set(b):
+        raise KeyError(min(set(a) ^ set(b)))
     return tuple(
         sorted(
             item_id
@@ -32,6 +38,15 @@ def advances(decision: str) -> bool:
 def audit_sample(
     ids: Sequence[str], *, rate: float = 0.10, seed: int
 ) -> tuple[str, ...]:
-    """Sort the ids, then draw ``int(len * rate)`` of them with ``seed``."""
+    """Sort the ids, then draw ``int(len * rate)`` of them with ``seed``.
+
+    A positive rate on a nonempty list draws at least one id, so a list
+    shorter than ten still gets a 10 percent audit. Rate 0 draws nothing.
+    """
     ordered = sorted(ids)
-    return tuple(random.Random(seed).sample(ordered, int(len(ordered) * rate)))
+    count = int(len(ordered) * rate)
+    if ordered and rate > 0 and count == 0:
+        count = 1
+    if count == 0:
+        return ()
+    return tuple(random.Random(seed).sample(ordered, count))
diff --git a/src/prisma/dedup.py b/src/prisma/dedup.py
index 678beba..1cb77d2 100644
--- a/src/prisma/dedup.py
+++ b/src/prisma/dedup.py
@@ -13,6 +13,7 @@ from dataclasses import dataclass
 from difflib import SequenceMatcher
 
 from zot.db import TYPE_MAP, Db
+from zot.fetch.select import normalise_doi
 
 _YEAR = re.compile(r"(\d{4})")
 _SIMILAR = 0.85
@@ -46,7 +47,7 @@ def normalize(title: str) -> str:
 
 
 def _doi_key(doi: str) -> str:
-    return doi.strip().casefold().rstrip(".")
+    return normalise_doi(doi).casefold().rstrip(".")
 
 
 def _title_key(rec: Record) -> tuple[str, int | None, str] | None:
@@ -108,7 +109,7 @@ def cluster(records: Sequence[Record]) -> DedupResult:
         if not left_title:
             continue
         for right in singles[i + 1 :]:
-            if left.year != right.year:
+            if left.year is None or right.year is None or left.year != right.year:
                 continue
             right_title = normalize(right.title)
             if not right_title:
diff --git a/src/prisma/search/arxiv.py b/src/prisma/search/arxiv.py
index 7b0bb17..0cd2802 100644
--- a/src/prisma/search/arxiv.py
+++ b/src/prisma/search/arxiv.py
@@ -9,6 +9,7 @@ from prisma.search.base import (
     SearchResult,
     bare_doi,
     finish,
+    pmid_of,
     surname,
     year_of,
 )
@@ -44,7 +45,9 @@ def parse_arxiv(payload: str, *, query: str, searched_on: str) -> SearchResult:
                 title=_text(_child(entry, "title")),
                 year=year_of(_text(_child(entry, "published"))),
                 doi=bare_doi(doi),
-                pmid="",
+                pmid=pmid_of(
+                    _text(_child(entry, "pmid", _ARXIV)) or _text(_child(entry, "pmid"))
+                ),
                 first_author=surname(name),
                 database="arxiv",
             )
diff --git a/src/prisma/search/base.py b/src/prisma/search/base.py
index f5db393..d1b8d6e 100644
--- a/src/prisma/search/base.py
+++ b/src/prisma/search/base.py
@@ -3,10 +3,15 @@
 from __future__ import annotations
 
 import hashlib
+import re
 from dataclasses import dataclass
 
 from lead.ledger.store import append
 from zot.db import FIELD_MAP, TYPE_MAP, Db
+from zot.fetch.select import normalise_doi
+
+_PMID_URL = re.compile(r"pubmed\.ncbi\.nlm\.nih\.gov/(\d+)", re.IGNORECASE)
+_PMID_PREFIX = re.compile(r"^pmid:\s*", re.IGNORECASE)
 
 
 @dataclass(frozen=True)
@@ -73,13 +78,29 @@ def surname(name: str) -> str:
 
 
 def bare_doi(value: str) -> str:
-    """Drop a ``doi.org`` or ``doi:`` prefix."""
-    text = value.strip()
-    lowered = text.casefold()
-    for prefix in ("https://doi.org/", "http://doi.org/", "doi:"):
-        if lowered.startswith(prefix):
-            return text[len(prefix) :]
-    return text
+    """Drop a ``doi.org``, ``dx.doi.org``, or ``doi:`` prefix.
+
+    A trailing period stays. Dedup strips that period itself.
+    """
+    return normalise_doi(value)
+
+
+def pmid_of(value: object) -> str:
+    """Digits of a PMID, or empty when the field is not one.
+
+    A ``pmid:`` prefix and a PubMed URL are stripped. Any other text,
+    including a DOI, is empty.
+    """
+    if value is None:
+        return ""
+    text = str(value).strip()
+    if not text:
+        return ""
+    url = _PMID_URL.search(text)
+    if url:
+        return url.group(1)
+    text = _PMID_PREFIX.sub("", text).strip()
+    return text if text.isdigit() else ""
 
 
 def year_of(value: object) -> int | None:
@@ -165,16 +186,20 @@ def import_hits(zdb: Db, project: str, result: SearchResult) -> int:
 
 
 def _doi_item_id(zdb: Db, doi: str) -> int | None:
-    row = zdb.con.execute(
+    """Match a DOI already stored as a URL, a ``doi:`` prefix, or bare text."""
+    wanted = normalise_doi(doi).casefold().rstrip(".")
+    if not wanted:
+        return None
+    rows = zdb.con.execute(
         """
-        SELECT id.itemID
+        SELECT id.itemID, idv.value
         FROM itemData id
         JOIN itemDataValues idv ON id.valueID = idv.valueID
-        WHERE id.fieldID = ? AND lower(idv.value) = lower(?)
-        LIMIT 1
+        WHERE id.fieldID = ?
         """,
-        (FIELD_MAP["DOI"], doi.strip()),
-    ).fetchone()
-    if row is None:
-        return None
-    return int(row[0])
+        (FIELD_MAP["DOI"],),
+    ).fetchall()
+    for item_id, value in rows:
+        if normalise_doi(str(value)).casefold().rstrip(".") == wanted:
+            return int(item_id)
+    return None
diff --git a/src/prisma/search/crossref.py b/src/prisma/search/crossref.py
index e0a4228..37578d8 100644
--- a/src/prisma/search/crossref.py
+++ b/src/prisma/search/crossref.py
@@ -4,7 +4,14 @@ from __future__ import annotations
 
 import json
 
-from prisma.search.base import SearchHit, SearchResult, bare_doi, finish, year_of
+from prisma.search.base import (
+    SearchHit,
+    SearchResult,
+    bare_doi,
+    finish,
+    pmid_of,
+    year_of,
+)
 
 
 def _title(row: dict) -> str:
@@ -35,7 +42,7 @@ def parse_crossref(payload: str, *, query: str, searched_on: str) -> SearchResul
                 title=_title(row),
                 year=_year(row),
                 doi=bare_doi(row.get("DOI") or ""),
-                pmid="",
+                pmid=pmid_of(row["pmid"] if "pmid" in row else row.get("PMID")),
                 first_author=family,
                 database="crossref",
             )
diff --git a/src/prisma/search/openalex.py b/src/prisma/search/openalex.py
index 65c7d3d..54ae436 100644
--- a/src/prisma/search/openalex.py
+++ b/src/prisma/search/openalex.py
@@ -9,6 +9,7 @@ from prisma.search.base import (
     SearchResult,
     bare_doi,
     finish,
+    pmid_of,
     surname,
     year_of,
 )
@@ -23,12 +24,13 @@ def parse_openalex(payload: str, *, query: str, searched_on: str) -> SearchResul
         name = ""
         if authorships:
             name = ((authorships[0].get("author") or {}).get("display_name")) or ""
+        ids = row.get("ids") if isinstance(row.get("ids"), dict) else {}
         hits.append(
             SearchHit(
                 title=row.get("title") or "",
                 year=year_of(row.get("publication_year")),
                 doi=bare_doi(row.get("doi") or ""),
-                pmid="",
+                pmid=pmid_of(ids.get("pmid")),
                 first_author=surname(name),
                 database="openalex",
             )
diff --git a/src/prisma/search/pubmed.py b/src/prisma/search/pubmed.py
index 3edd7ef..17c03c2 100644
--- a/src/prisma/search/pubmed.py
+++ b/src/prisma/search/pubmed.py
@@ -29,7 +29,23 @@ def _doi(article: ET.Element) -> str:
 
 
 def parse_pubmed_xml(xml: str, *, query: str, searched_on: str) -> SearchResult:
-    """Read one PubmedArticleSet. Missing fields become empty or None."""
+    """Read one PubmedArticleSet. Missing fields become empty or None.
+
+    A blank payload has no hits. The log still records that payload's sha256.
+    """
+    if not xml.strip():
+        return SearchResult(
+            hits=(),
+            log=SearchLog(
+                database="pubmed",
+                interface="eutils",
+                searched_on=searched_on,
+                query=query,
+                filters="",
+                hit_count=0,
+                export_sha256=hashlib.sha256(xml.encode()).hexdigest(),
+            ),
+        )
     root = ET.fromstring(xml)
     hits: list[SearchHit] = []
     for article in root.iter("PubmedArticle"):
diff --git a/src/prisma/search/semanticscholar.py b/src/prisma/search/semanticscholar.py
index ee70bb0..dbb99ae 100644
--- a/src/prisma/search/semanticscholar.py
+++ b/src/prisma/search/semanticscholar.py
@@ -9,6 +9,7 @@ from prisma.search.base import (
     SearchResult,
     bare_doi,
     finish,
+    pmid_of,
     surname,
     year_of,
 )
@@ -23,13 +24,19 @@ def parse_semanticscholar(
     for row in rows:
         authors = row.get("authors") or []
         name = (authors[0].get("name") or "") if authors else ""
-        doi = ((row.get("externalIds") or {}).get("DOI")) or ""
+        external = row.get("externalIds") or {}
+        doi = (external.get("DOI") if isinstance(external, dict) else "") or ""
+        raw_pmid = None
+        if isinstance(external, dict):
+            raw_pmid = external.get("PubMed")
+            if raw_pmid in (None, ""):
+                raw_pmid = external.get("PMID")
         hits.append(
             SearchHit(
                 title=row.get("title") or "",
                 year=year_of(row.get("year")),
                 doi=bare_doi(doi),
-                pmid="",
+                pmid=pmid_of(raw_pmid),
                 first_author=surname(name),
                 database="semanticscholar",
             )
diff --git a/tests/econ/test_fred_irs.py b/tests/econ/test_fred_irs.py
index ddfee62..f12dea8 100644
--- a/tests/econ/test_fred_irs.py
+++ b/tests/econ/test_fred_irs.py
@@ -16,6 +16,12 @@ def test_fred_dot_value_is_missing():
     assert rows[0].period == "2024-01-01"
 
 
+def test_fred_missing_date_has_no_year():
+    rows = parse_observations({"observations": [{"date": "", "value": "1"}]})
+    assert rows[0].year is None
+    assert rows[0].value == 1.0
+
+
 def test_county_income_keeps_the_fips_text():
     rows = parse_county_income("fips,year,agi\n01001,2022,40000\n")
     assert rows == (CountyIncome(fips="01001", year=2022, agi=40000),)
diff --git a/tests/lead/test_ledger_store.py b/tests/lead/test_ledger_store.py
index 3878333..8439594 100644
--- a/tests/lead/test_ledger_store.py
+++ b/tests/lead/test_ledger_store.py
@@ -219,3 +219,10 @@ def test_status_command_prints_every_target(tmp_path):
     result = runner.invoke(app, ["lead", "ledger", "status", "--db", str(path)])
     assert result.exit_code == 0
     assert "Q.ACR" in result.stdout
+
+
+def test_status_command_prints_gaps_runs_contradictions_and_tiers(tmp_path):
+    path = tmp_path / "ledger.duckdb"
+    result = runner.invoke(app, ["lead", "ledger", "status", "--db", str(path)])
+    assert result.exit_code == 0
+    assert "Q.ACR\t1\t\t0\t0\t0\t0\t0" in result.stdout
diff --git a/tests/prisma/test_audit.py b/tests/prisma/test_audit.py
index bbf3e89..cab76fa 100644
--- a/tests/prisma/test_audit.py
+++ b/tests/prisma/test_audit.py
@@ -38,6 +38,18 @@ def test_agreed_excludes_are_ids_both_passes_exclude():
     assert agreed_excludes(a, b) == ("i2",)
 
 
+def test_agreed_excludes_require_the_same_ids():
+    with pytest.raises(KeyError):
+        agreed_excludes({"i1": "exclude"}, {"i1": "exclude", "i2": "exclude"})
+
+
+def test_audit_sample_draws_one_when_ten_percent_floors_to_zero():
+    ids = [f"i{n}" for n in range(9)]
+    drawn = audit_sample(ids, rate=0.10, seed=1)
+    assert len(drawn) == 1
+    assert drawn[0] in ids
+
+
 def test_uncertain_and_include_advance_exclude_does_not():
     assert advances("uncertain") is True
     assert advances("include") is True
diff --git a/tests/prisma/test_dedup.py b/tests/prisma/test_dedup.py
index f9bc886..2057a57 100644
--- a/tests/prisma/test_dedup.py
+++ b/tests/prisma/test_dedup.py
@@ -79,6 +79,49 @@ def test_near_titles_stay_borderline_and_unmerged():
     assert result.borderline == ((first, second),)
 
 
+def test_doi_url_and_bare_doi_are_one_group():
+    first = _rec(key="AAAAAAAA", doi="https://dx.doi.org/10.1000/ABC", title="One")
+    second = _rec(key="BBBBBBBB", doi="10.1000/abc", title="Two")
+    result = cluster([first, second])
+    assert len(result.groups) == 1
+    assert {rec.key for rec in result.groups[0]} == {"AAAAAAAA", "BBBBBBBB"}
+
+
+def test_undated_near_titles_are_not_borderline():
+    first = _rec(
+        key="AAAAAAAA",
+        title="High needs frailty index",
+        year=None,
+        first_author="Kim",
+    )
+    second = _rec(
+        key="BBBBBBBB",
+        title="High-need frailty index",
+        year=None,
+        first_author="Lee",
+    )
+    result = cluster([first, second])
+    assert len(result.groups) == 2
+    assert result.borderline == ()
+
+
+def test_identical_undated_titles_with_the_same_author_merge():
+    first = _rec(
+        key="AAAAAAAA",
+        title="High needs frailty index",
+        year=None,
+        first_author="Kim",
+    )
+    second = _rec(
+        key="BBBBBBBB",
+        title="High needs frailty index",
+        year=None,
+        first_author="Kim",
+    )
+    result = cluster([first, second])
+    assert len(result.groups) == 1
+
+
 def test_dissimilar_titles_are_neither_merged_nor_borderline():
     first = _rec(key="AAAAAAAA", title="zzzz unique alpha", first_author="Kim")
     second = _rec(key="BBBBBBBB", title="qqqq other beta", first_author="Lee")
diff --git a/tests/prisma/test_search_others.py b/tests/prisma/test_search_others.py
index c392317..3436204 100644
--- a/tests/prisma/test_search_others.py
+++ b/tests/prisma/test_search_others.py
@@ -5,6 +5,7 @@ from __future__ import annotations
 import pytest
 
 from prisma.search.arxiv import parse_arxiv
+from prisma.search.base import bare_doi
 from prisma.search.crossref import parse_crossref
 from prisma.search.europepmc import parse_europepmc
 from prisma.search.openalex import parse_openalex
@@ -68,6 +69,60 @@ _SPARSE = [
 ]
 
 
+def test_bare_doi_strips_a_dx_doi_org_prefix_and_keeps_a_trailing_period():
+    assert bare_doi("https://dx.doi.org/10.1/ab") == "10.1/ab"
+    assert bare_doi("https://dx.doi.org/10.1/ab.") == "10.1/ab."
+
+
+def test_openalex_reads_a_pubmed_url_pmid():
+    payload = (
+        '{"results":[{"title":"Report title","publication_year":2020,'
+        '"doi":"https://doi.org/10.1/o","ids":{"pmid":'
+        '"https://pubmed.ncbi.nlm.nih.gov/42"},'
+        '"authorships":[{"author":{"display_name":"Ada Kim"}}]}]}'
+    )
+    hit = parse_openalex(payload, query="frailty", searched_on="2026-10-31").hits[0]
+    assert hit.pmid == "42"
+
+
+def test_semanticscholar_reads_external_pubmed_id():
+    payload = (
+        '{"data":[{"title":"Report title","year":2020,'
+        '"externalIds":{"DOI":"10.1/s","PubMed":"999"},'
+        '"authors":[{"name":"Ada Kim"}]}]}'
+    )
+    hit = parse_semanticscholar(
+        payload, query="frailty", searched_on="2026-10-31"
+    ).hits[0]
+    assert hit.pmid == "999"
+
+
+def test_crossref_reads_a_pmid_field():
+    payload = (
+        '{"message":{"items":[{"title":["Report title"],"DOI":"10.1/c",'
+        '"pmid":"12345","issued":{"date-parts":[[2020,1,2]]},'
+        '"author":[{"family":"Kim","given":"Ada"}]}]}}'
+    )
+    hit = parse_crossref(payload, query="frailty", searched_on="2026-10-31").hits[0]
+    assert hit.pmid == "12345"
+
+
+def test_arxiv_reads_a_pmid_element():
+    payload = """\
+
+  
+    Report title
+    2020-01-02T00:00:00Z
+    Ada Kim
+    10.1/a
+    77
+  
+
+"""
+    hit = parse_arxiv(payload, query="frailty", searched_on="2026-10-31").hits[0]
+    assert hit.pmid == "77"
+
+
 @pytest.mark.parametrize(
     ("parser", "payload", "database", "doi", "year", "author"), _CASES
 )
diff --git a/tests/prisma/test_search_pubmed.py b/tests/prisma/test_search_pubmed.py
index 040bb92..729bdef 100644
--- a/tests/prisma/test_search_pubmed.py
+++ b/tests/prisma/test_search_pubmed.py
@@ -109,6 +109,54 @@ def test_record_with_project_clears_due_and_keeps_projects_apart(tmp_path):
         con.close()
 
 
+def test_blank_pubmed_xml_has_no_hits():
+    payload = "  \n"
+    result = parse_pubmed_xml(payload, query="frailty", searched_on="2026-10-31")
+    assert result.hits == ()
+    assert result.log.hit_count == 0
+    assert result.log.database == "pubmed"
+    assert result.log.interface == "eutils"
+    assert result.log.export_sha256 == hashlib.sha256(payload.encode()).hexdigest()
+
+
+def test_import_hits_matches_a_doi_stored_as_a_dx_url(tmp_path):
+    path = str(tmp_path / "z.sqlite")
+    con = create_db(path)
+    con.close()
+    hit = SearchHit(
+        title="Alpha frailty",
+        year=2020,
+        doi="10.1/alpha",
+        pmid="111",
+        first_author="Kim",
+        database="openalex",
+    )
+    result = SearchResult(
+        hits=(hit,),
+        log=SearchLog(
+            database="openalex",
+            interface="openalex",
+            searched_on="2026-10-31",
+            query="frailty",
+            filters="",
+            hit_count=1,
+            export_sha256="abc",
+        ),
+    )
+    with Db(path) as db:
+        item_id = db.create_item(TYPE_MAP["journalArticle"])
+        db.set_fields(
+            item_id,
+            {"title": "Alpha frailty", "DOI": "https://dx.doi.org/10.1/alpha"},
+        )
+        assert import_hits(db, "lead-limits", result) == 0
+        articles = db.con.execute(
+            "select count(*) from items where itemTypeID = ?",
+            (TYPE_MAP["journalArticle"],),
+        ).fetchone()[0]
+        assert articles == 1
+
+
 def test_import_hits_is_idempotent_on_doi(tmp_path):
     path = str(tmp_path / "z.sqlite")
     con = create_db(path)
-- 
2.49.1


From 9bddb3ddf02bd04ea5d771238461c71c1766c49b Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:01:50 -0400
Subject: [PATCH 23/30] feat(lead): score High Needs criteria and separate
 suspect moves

---
 src/lead/high_needs.py        | 80 +++++++++++++++++++++++++++++++++++
 src/lead/suspect.py           | 50 ++++++++++++++++++++++
 tests/lead/test_high_needs.py | 76 +++++++++++++++++++++++++++++++++
 tests/lead/test_suspect.py    | 62 +++++++++++++++++++++++++++
 4 files changed, 268 insertions(+)
 create mode 100644 src/lead/high_needs.py
 create mode 100644 src/lead/suspect.py
 create mode 100644 tests/lead/test_high_needs.py
 create mode 100644 tests/lead/test_suspect.py

diff --git a/src/lead/high_needs.py b/src/lead/high_needs.py
new file mode 100644
index 0000000..a2c6e35
--- /dev/null
+++ b/src/lead/high_needs.py
@@ -0,0 +1,80 @@
+"""High Needs criteria for one quarter and for an alignment year.
+
+The scores, the Kim index, the diagnosis flag, and the DME flag are
+caller-supplied. Appendix B code lists, Kim weights, and the V28 software
+are not in this repo, so this module does not invent them. The thresholds
+are the ones named in the LEAD methodology for PY 2027.
+
+A score of 3.0 is HN.4 only. A score of 0.35 is HN.5 only. The HN.6 bands
+stop below those thresholds. Once any quarter meets a criterion, it applies
+for the whole alignment year, and a criterion met in a prior year stays met.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+AD_HIGH = 3.0
+ESRD_HIGH = 0.35
+AD_BAND_LOW = 2.0
+ESRD_BAND_LOW = 0.24
+UNPLANNED_ADMISSIONS = 2
+SNF_DAYS = 45
+
+
+@dataclass(frozen=True)
+class QuarterFlags:
+    """One quarter's lookback, already counted by the caller."""
+
+    quarter: str
+    hn1_diagnosis: bool = False
+    hn2_dme: bool = False
+    kim_index: float | None = None
+    kim_cutoff: float | None = None
+    ad_score: float | None = None
+    esrd_score: float | None = None
+    unplanned_admissions: int = 0
+    snf_days: int = 0
+
+
+def criteria_met(flags: QuarterFlags) -> frozenset[str]:
+    """Target ids this quarter meets. An empty set meets none."""
+    met: set[str] = set()
+    if flags.hn1_diagnosis:
+        met.add("HN.1")
+    if flags.hn2_dme:
+        met.add("HN.2")
+    if (
+        flags.kim_index is not None
+        and flags.kim_cutoff is not None
+        and flags.kim_index >= flags.kim_cutoff
+    ):
+        met.add("HN.3")
+    if flags.ad_score is not None and flags.ad_score >= AD_HIGH:
+        met.add("HN.4")
+    if flags.esrd_score is not None and flags.esrd_score >= ESRD_HIGH:
+        met.add("HN.5")
+    admissions = flags.unplanned_admissions >= UNPLANNED_ADMISSIONS
+    ad_band = flags.ad_score is not None and AD_BAND_LOW <= flags.ad_score < AD_HIGH
+    esrd_band = (
+        flags.esrd_score is not None and ESRD_BAND_LOW <= flags.esrd_score < ESRD_HIGH
+    )
+    if admissions and (ad_band or esrd_band):
+        met.add("HN.6")
+    if flags.snf_days >= SNF_DAYS:
+        met.add("HN.7")
+    return frozenset(met)
+
+
+def alignment_year(quarters: Sequence[QuarterFlags]) -> frozenset[str]:
+    """Union of the criteria met in any quarter of the alignment year."""
+    met: set[str] = set()
+    for quarter in quarters:
+        met |= criteria_met(quarter)
+    return frozenset(met)
+
+
+def persists(prior: frozenset[str], this_year: frozenset[str]) -> frozenset[str]:
+    """High Needs status does not go away once a criterion has been met."""
+    return prior | this_year
diff --git a/src/lead/suspect.py b/src/lead/suspect.py
new file mode 100644
index 0000000..5aa7f8d
--- /dev/null
+++ b/src/lead/suspect.py
@@ -0,0 +1,50 @@
+"""A suspect is a status change, by coding or by care.
+
+Coding makes a status the person already has visible to the score. Care
+changes the underlying state. Demographic reweighting multiplies only the
+coding delta. The two are never collapsed into one number.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class SuspectMove:
+    """How far each mechanism moves the score."""
+
+    coding_delta: float
+    care_delta: float
+    coding_after_reweight: float
+
+
+def suspect_move(
+    *,
+    before_status: str,
+    after_status: str,
+    score_before: float,
+    score_coding: float,
+    score_care: float,
+    weight_before: float,
+    weight_after: float,
+) -> SuspectMove:
+    """Return the coding delta, the care delta, and the reweighted coding delta.
+
+    Raises:
+        ValueError: the status does not change, the weights are equal, or
+            the starting weight is zero.
+    """
+    if before_status == after_status:
+        raise ValueError("a suspect is a status change")
+    if weight_before == weight_after:
+        raise ValueError("reweight needs a different weight")
+    if weight_before == 0:
+        raise ValueError("weight_before must not be zero")
+    coding_delta = score_coding - score_before
+    care_delta = score_care - score_before
+    return SuspectMove(
+        coding_delta=coding_delta,
+        care_delta=care_delta,
+        coding_after_reweight=coding_delta * (weight_after / weight_before),
+    )
diff --git a/tests/lead/test_high_needs.py b/tests/lead/test_high_needs.py
new file mode 100644
index 0000000..e600a3c
--- /dev/null
+++ b/tests/lead/test_high_needs.py
@@ -0,0 +1,76 @@
+"""High Needs criteria from caller-supplied quarter flags."""
+
+from __future__ import annotations
+
+from lead.high_needs import QuarterFlags, alignment_year, criteria_met, persists
+
+
+def test_ad_score_at_3_is_hn4_and_not_hn6():
+    met = criteria_met(QuarterFlags(quarter="2027Q1", ad_score=3.0))
+    assert met == frozenset({"HN.4"})
+
+
+def test_ad_band_with_two_admissions_is_hn6():
+    met = criteria_met(
+        QuarterFlags(quarter="2027Q1", ad_score=2.0, unplanned_admissions=2)
+    )
+    assert met == frozenset({"HN.6"})
+
+
+def test_ad_band_with_one_admission_meets_nothing():
+    met = criteria_met(
+        QuarterFlags(quarter="2027Q1", ad_score=2.0, unplanned_admissions=1)
+    )
+    assert met == frozenset()
+
+
+def test_ad_below_the_band_meets_nothing():
+    met = criteria_met(
+        QuarterFlags(quarter="2027Q1", ad_score=1.999, unplanned_admissions=2)
+    )
+    assert met == frozenset()
+
+
+def test_esrd_score_at_0_35_is_hn5_and_not_hn6():
+    met = criteria_met(QuarterFlags(quarter="2027Q1", esrd_score=0.35))
+    assert "HN.5" in met
+    assert "HN.6" not in met
+
+
+def test_esrd_band_with_two_admissions_is_hn6():
+    met = criteria_met(
+        QuarterFlags(quarter="2027Q1", esrd_score=0.24, unplanned_admissions=2)
+    )
+    assert met == frozenset({"HN.6"})
+
+
+def test_forty_five_snf_days_meet_hn7():
+    assert "HN.7" in criteria_met(QuarterFlags(quarter="2027Q1", snf_days=45))
+    assert "HN.7" not in criteria_met(QuarterFlags(quarter="2027Q1", snf_days=44))
+
+
+def test_diagnosis_and_dme_are_separate_criteria():
+    assert criteria_met(
+        QuarterFlags(quarter="2027Q1", hn1_diagnosis=True)
+    ) == frozenset({"HN.1"})
+    assert criteria_met(QuarterFlags(quarter="2027Q1", hn2_dme=True)) == frozenset(
+        {"HN.2"}
+    )
+
+
+def test_kim_needs_both_the_index_and_the_cutoff():
+    assert "HN.3" in criteria_met(
+        QuarterFlags(quarter="2027Q1", kim_index=0.2, kim_cutoff=0.2)
+    )
+    assert "HN.3" not in criteria_met(QuarterFlags(quarter="2027Q1", kim_index=0.2))
+    assert "HN.3" not in criteria_met(QuarterFlags(quarter="2027Q1", kim_cutoff=0.2))
+
+
+def test_one_quarter_sets_the_alignment_year():
+    quiet = QuarterFlags(quarter="2027Q1")
+    met = QuarterFlags(quarter="2027Q2", snf_days=45)
+    assert alignment_year((quiet, met, quiet, quiet)) == frozenset({"HN.7"})
+
+
+def test_a_met_criterion_never_goes_away():
+    assert persists(frozenset({"HN.1"}), frozenset()) == frozenset({"HN.1"})
diff --git a/tests/lead/test_suspect.py b/tests/lead/test_suspect.py
new file mode 100644
index 0000000..10c43cc
--- /dev/null
+++ b/tests/lead/test_suspect.py
@@ -0,0 +1,62 @@
+"""A suspect is a status change. Coding and care stay separate numbers."""
+
+from __future__ import annotations
+
+import pytest
+
+from lead.suspect import SuspectMove, suspect_move
+
+
+def test_coding_and_care_stay_apart_and_reweight_shrinks_only_coding():
+    move = suspect_move(
+        before_status="standard",
+        after_status="high_needs",
+        score_before=1.0,
+        score_coding=1.2,
+        score_care=1.5,
+        weight_before=1.0,
+        weight_after=0.5,
+    )
+    assert isinstance(move, SuspectMove)
+    assert move.coding_delta == pytest.approx(0.2)
+    assert move.care_delta == pytest.approx(0.5)
+    assert move.coding_after_reweight == pytest.approx(0.1)
+
+
+def test_the_same_status_is_not_a_suspect():
+    with pytest.raises(ValueError):
+        suspect_move(
+            before_status="high_needs",
+            after_status="high_needs",
+            score_before=1.0,
+            score_coding=1.2,
+            score_care=1.5,
+            weight_before=1.0,
+            weight_after=0.5,
+        )
+
+
+def test_equal_weights_raise():
+    with pytest.raises(ValueError):
+        suspect_move(
+            before_status="standard",
+            after_status="high_needs",
+            score_before=1.0,
+            score_coding=1.2,
+            score_care=1.5,
+            weight_before=1.0,
+            weight_after=1.0,
+        )
+
+
+def test_a_zero_starting_weight_raises():
+    with pytest.raises(ValueError):
+        suspect_move(
+            before_status="standard",
+            after_status="high_needs",
+            score_before=1.0,
+            score_coding=1.2,
+            score_care=1.5,
+            weight_before=0.0,
+            weight_after=0.5,
+        )
-- 
2.49.1


From c1a8db91b425a42ba813dbfe25a61a6944922680 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:03:04 -0400
Subject: [PATCH 24/30] feat(lead): add the limits calculator and rule-tier
 claims

---
 src/lead/limits.py        | 49 ++++++++++++++++++++++++++
 src/lead/rules.py         | 73 +++++++++++++++++++++++++++++++++++++++
 tests/lead/test_limits.py | 43 +++++++++++++++++++++++
 tests/lead/test_rules.py  | 33 ++++++++++++++++++
 4 files changed, 198 insertions(+)
 create mode 100644 src/lead/limits.py
 create mode 100644 src/lead/rules.py
 create mode 100644 tests/lead/test_limits.py
 create mode 100644 tests/lead/test_rules.py

diff --git a/src/lead/limits.py b/src/lead/limits.py
new file mode 100644
index 0000000..23f18a4
--- /dev/null
+++ b/src/lead/limits.py
@@ -0,0 +1,49 @@
+"""Strand-2 limit expressions on caller-supplied public inputs.
+
+Headroom uses the published High Needs risk-score growth cap. Normalization
+divides by a factor the caller supplies. Shrinkage is this chassis's ACO-size
+weight, not a CMS coefficient, so it is not a rule-tier claim.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from lead.benchmark import capped_risk_score
+
+
+@dataclass(frozen=True)
+class Headroom:
+    """Capped score, the cap status, and growth the cap removes."""
+
+    scored: float
+    status: str
+    forgone: float
+
+
+def earlier_qualification_headroom(prior: float, caught: float) -> Headroom:
+    """Apply the High Needs cap to a score caught earlier in the year."""
+    scored, status = capped_risk_score(prior, caught, "High Needs")
+    return Headroom(scored=scored, status=status, forgone=max(caught - scored, 0.0))
+
+
+def normalization_sensitivity(benchmark: float, factor: float) -> float:
+    """Benchmark divided by a caller-supplied normalization factor.
+
+    Raises:
+        ValueError: ``factor`` is not positive.
+    """
+    if factor <= 0:
+        raise ValueError("factor must be positive")
+    return benchmark / factor
+
+
+def shrinkage_weight(aco_n: int, prior_n: int) -> float:
+    """ACO size over ACO size plus the prior count.
+
+    Raises:
+        ValueError: the two counts are not a positive total.
+    """
+    if aco_n < 0 or prior_n < 0 or aco_n + prior_n == 0:
+        raise ValueError("aco size and prior must be a positive weight")
+    return aco_n / (aco_n + prior_n)
diff --git a/src/lead/rules.py b/src/lead/rules.py
new file mode 100644
index 0000000..d2e0332
--- /dev/null
+++ b/src/lead/rules.py
@@ -0,0 +1,73 @@
+"""Rule-tier claims for published numbers this package reproduces.
+
+The actor is not part of the content hash, so a second publish inserts
+nothing. Numbers that are not reproduced by a function in this repo are
+not written here.
+"""
+
+from __future__ import annotations
+
+import hashlib
+
+from lead.ledger.store import append
+
+_CLAIMS = (
+    (
+        "RA.CAP",
+        0.04,
+        "High Needs PY risk-score growth is capped at 4 percent of BY 3.",
+    ),
+    (
+        "HN.4",
+        3.0,
+        "An A&D risk score >= 3.0 meets HN.4. The score 3.0 is not in the HN.6 band.",
+    ),
+    (
+        "HN.5",
+        0.35,
+        "An ESRD risk score >= 0.35 meets HN.5. The score 0.35 is not in the HN.6 band.",
+    ),
+    (
+        "HN.6",
+        2.0,
+        "HN.6 needs at least 2 unplanned admissions, plus an A&D score from 2.0 "
+        "up to but not including 3.0, or an ESRD score from 0.24 up to but not "
+        "including 0.35.",
+    ),
+    (
+        "HN.7",
+        45.0,
+        "HN.7 is at least 45 covered SNF days in 12 months.",
+    ),
+)
+
+
+def _hash(target_id: str, effect: float) -> str:
+    text = f"rule\n{target_id}\n{effect}\n"
+    return hashlib.sha256(text.encode()).hexdigest()
+
+
+def publish_rules(con, *, actor: str) -> int:
+    """Append one rule claim per published identity. Return how many were new."""
+    inserted = 0
+    for target_id, effect, subject in _CLAIMS:
+        content = _hash(target_id, effect)
+        wrote = append(
+            con,
+            "claim",
+            {
+                "id": content,
+                "kind": "limit",
+                "subject": subject,
+                "target_id": target_id,
+                "effect_measure": "threshold",
+                "effect_value": effect,
+                "certainty": "high",
+                "tiers": "rule",
+                "content_hash": content,
+                "actor": actor,
+            },
+        )
+        if wrote:
+            inserted += 1
+    return inserted
diff --git a/tests/lead/test_limits.py b/tests/lead/test_limits.py
new file mode 100644
index 0000000..1721ec1
--- /dev/null
+++ b/tests/lead/test_limits.py
@@ -0,0 +1,43 @@
+"""Limits on public inputs: the High Needs cap, normalization, shrinkage."""
+
+from __future__ import annotations
+
+import pytest
+
+from lead.benchmark import CAPPED, WITHIN_CAP
+from lead.limits import (
+    Headroom,
+    earlier_qualification_headroom,
+    normalization_sensitivity,
+    shrinkage_weight,
+)
+
+
+def test_catching_high_needs_above_the_cap_forgoes_the_excess():
+    room = earlier_qualification_headroom(1.0, 1.05)
+    assert room == Headroom(room.scored, CAPPED, room.forgone)
+    assert room.scored == 1.04
+    assert room.forgone == pytest.approx(0.01)
+
+
+def test_growth_inside_the_cap_forgoes_nothing():
+    room = earlier_qualification_headroom(1.0, 1.035)
+    assert room == Headroom(1.035, WITHIN_CAP, 0.0)
+
+
+def test_normalization_divides_by_the_caller_factor():
+    assert normalization_sensitivity(1000.0, 1.25) == 800.0
+
+
+def test_a_nonpositive_normalization_factor_raises():
+    with pytest.raises(ValueError):
+        normalization_sensitivity(1000.0, 0)
+
+
+def test_shrinkage_is_the_aco_share():
+    assert shrinkage_weight(100, 300) == 0.25
+
+
+def test_shrinkage_with_no_observations_raises():
+    with pytest.raises(ValueError):
+        shrinkage_weight(0, 0)
diff --git a/tests/lead/test_rules.py b/tests/lead/test_rules.py
new file mode 100644
index 0000000..edb1f36
--- /dev/null
+++ b/tests/lead/test_rules.py
@@ -0,0 +1,33 @@
+"""Rule-tier claims for numbers the kernel reproduces."""
+
+from __future__ import annotations
+
+from lead.ledger.store import open_ledger
+from lead.rules import publish_rules
+
+
+def test_publish_rules_inserts_each_published_threshold_once(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        inserted = publish_rules(con, actor="tester")
+        rows = con.execute(
+            """
+            SELECT target_id, effect_value, tiers, subject
+            FROM lead_evidence.claim
+            ORDER BY target_id
+            """
+        ).fetchall()
+        by_target = {row[0]: row for row in rows}
+        assert inserted == len(rows) == 5
+        assert by_target["RA.CAP"][1] == 0.04
+        assert by_target["HN.4"][1] == 3.0
+        assert by_target["HN.5"][1] == 0.35
+        assert by_target["HN.6"][1] == 2.0
+        assert by_target["HN.7"][1] == 45.0
+        assert all(row[2] == "rule" for row in rows)
+        subject = by_target["HN.6"][3]
+        assert "3.0" in subject
+        assert "0.35" in subject
+        assert publish_rules(con, actor="other") == 0
+    finally:
+        con.close()
-- 
2.49.1


From 31a706beb5d3ae1228fd6bac68a3bc4dc5d47455 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:04:29 -0400
Subject: [PATCH 25/30] feat(lead): load the public manifest and record measure
 gaps

---
 src/cli/lead.py             |  18 +++++
 src/lead/manifest.py        | 157 ++++++++++++++++++++++++++++++++++++
 src/lead/quality.py         |  28 +++++++
 src/lead/spine.py           |  46 +++++++++++
 tests/lead/test_manifest.py | 101 +++++++++++++++++++++++
 tests/lead/test_quality.py  |  27 +++++++
 tests/lead/test_spine.py    |  34 ++++++++
 7 files changed, 411 insertions(+)
 create mode 100644 src/lead/manifest.py
 create mode 100644 src/lead/quality.py
 create mode 100644 src/lead/spine.py
 create mode 100644 tests/lead/test_manifest.py
 create mode 100644 tests/lead/test_quality.py
 create mode 100644 tests/lead/test_spine.py

diff --git a/src/cli/lead.py b/src/cli/lead.py
index e812f82..af26045 100644
--- a/src/cli/lead.py
+++ b/src/cli/lead.py
@@ -6,11 +6,14 @@ import typer
 
 from lead.ledger.store import open_ledger
 from lead.ledger.store import status as ledger_status
+from lead.manifest import load_manifest
 from lead.targets import TARGETS
 
 app = typer.Typer(no_args_is_help=True)
 ledger_app = typer.Typer(no_args_is_help=True)
+manifest_app = typer.Typer(no_args_is_help=True)
 app.add_typer(ledger_app, name="ledger", help="Evidence ledger.")
+app.add_typer(manifest_app, name="manifest", help="Public-release manifest.")
 
 
 @app.command(name="targets")
@@ -67,3 +70,18 @@ def ledger_export_cmd(
         con.close()
     for name in ("source", "decision", "extraction"):
         typer.echo(f"{name}\t{counts[name]}")
+
+
+@manifest_app.command("load")
+def manifest_load(
+    csv: str = typer.Option(..., "--csv", help="Path to the manifest CSV."),
+    db: str = typer.Option(..., "--db", help="Path to the DuckDB ledger."),
+    actor: str = typer.Option("stack", "--actor", help="Who loaded the rows."),
+) -> None:
+    """Record manifest rows. This does not download the files."""
+    con = open_ledger(db)
+    try:
+        inserted = load_manifest(con, csv, actor=actor)
+    finally:
+        con.close()
+    typer.echo(f"inserted\t{inserted}")
diff --git a/src/lead/manifest.py b/src/lead/manifest.py
new file mode 100644
index 0000000..d4bd03d
--- /dev/null
+++ b/src/lead/manifest.py
@@ -0,0 +1,157 @@
+"""Load a public-release manifest into the local DuckDB ledger.
+
+The caller supplies the CSV path. This module does not download the files
+and does not open their URLs. A missing path is still recorded. The same
+``source_id``, ``release``, and ``sha256`` insert nothing.
+
+The table DDL lives here, not in ``lead.ledger.schema``, so the Delta
+schema string stays free of a catalog token.
+"""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+from pathlib import Path
+
+SYNTHETIC_SOURCE_IDS = frozenset(
+    {"bcda_sandbox", "tuva", "cms_synthetic", "de_synpuf", "synthea"}
+)
+
+REQUIRED = (
+    "source_id",
+    "title",
+    "publisher",
+    "kind",
+    "url",
+    "api_endpoint",
+    "landing_page",
+    "release",
+    "fetched_utc",
+    "status",
+    "path",
+    "sha256",
+    "rows",
+    "size_bytes",
+    "license_terms",
+    "lead_use",
+)
+
+RELEASE_DDL = """
+CREATE SCHEMA IF NOT EXISTS lead_evidence;
+CREATE TABLE IF NOT EXISTS lead_evidence.public_release (
+    source_id VARCHAR,
+    title VARCHAR,
+    publisher VARCHAR,
+    kind VARCHAR,
+    url VARCHAR,
+    api_endpoint VARCHAR,
+    landing_page VARCHAR,
+    release VARCHAR,
+    fetched_utc VARCHAR,
+    status VARCHAR,
+    path VARCHAR,
+    sha256 VARCHAR,
+    row_count VARCHAR,
+    size_bytes VARCHAR,
+    license_terms VARCHAR,
+    lead_use VARCHAR,
+    layer VARCHAR,
+    actor VARCHAR,
+    content_hash VARCHAR
+)
+"""
+
+
+def layer_of(source_id: str, lead_use: str) -> str:
+    """``synthetic`` for a synthetic source or a lead_use that says so."""
+    if (
+        "synthetic" in lead_use.casefold()
+        or source_id.casefold() in SYNTHETIC_SOURCE_IDS
+    ):
+        return "synthetic"
+    return ""
+
+
+def _ensure(con) -> None:
+    for statement in RELEASE_DDL.split(";"):
+        text = statement.strip()
+        if text:
+            con.execute(text)
+
+
+def _identity(row: dict[str, str]) -> tuple[str, str, str]:
+    source_id = (row.get("source_id") or "").strip()
+    release = (row.get("release") or "").strip()
+    digest = (row.get("sha256") or "").strip()
+    if not source_id:
+        raise ValueError("source_id is required")
+    if not release:
+        raise ValueError("release is required")
+    if not digest:
+        raise ValueError("sha256 is required")
+    return source_id, release, digest
+
+
+def load_manifest(con, csv_path: str, *, actor: str) -> int:
+    """Insert new release rows. Return how many were new.
+
+    Raises:
+        ValueError: a required column or a required key is missing.
+    """
+    _ensure(con)
+    with Path(csv_path).open(newline="") as handle:
+        reader = csv.DictReader(handle)
+        missing = [name for name in REQUIRED if name not in (reader.fieldnames or [])]
+        if missing:
+            raise ValueError(f"missing columns: {missing}")
+        raw = list(reader)
+    identities = [_identity(row) for row in raw]
+    inserted = 0
+    for row, (source_id, release, digest) in zip(raw, identities, strict=True):
+        content = hashlib.sha256(
+            f"{source_id}\n{release}\n{digest}\n".encode()
+        ).hexdigest()
+        existing = con.execute(
+            """
+            SELECT 1 FROM lead_evidence.public_release
+            WHERE source_id = ? AND release = ? AND sha256 = ?
+            """,
+            [source_id, release, digest],
+        ).fetchone()
+        if existing:
+            continue
+        layer = layer_of(source_id, row.get("lead_use") or "")
+        con.execute(
+            """
+            INSERT INTO lead_evidence.public_release (
+                source_id, title, publisher, kind, url, api_endpoint,
+                landing_page, release, fetched_utc, status, path, sha256,
+                row_count, size_bytes, license_terms, lead_use, layer,
+                actor, content_hash
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            [
+                source_id,
+                row.get("title") or "",
+                row.get("publisher") or "",
+                row.get("kind") or "",
+                row.get("url") or "",
+                row.get("api_endpoint") or "",
+                row.get("landing_page") or "",
+                release,
+                row.get("fetched_utc") or "",
+                row.get("status") or "",
+                row.get("path") or "",
+                digest,
+                row.get("rows") or "",
+                row.get("size_bytes") or "",
+                row.get("license_terms") or "",
+                row.get("lead_use") or "",
+                layer,
+                actor,
+                content,
+            ],
+        )
+        inserted += 1
+    return inserted
diff --git a/src/lead/quality.py b/src/lead/quality.py
new file mode 100644
index 0000000..47c3e44
--- /dev/null
+++ b/src/lead/quality.py
@@ -0,0 +1,28 @@
+"""Which quality targets this repo can already compute.
+
+UAMCC and ACR stay in ``aco.express.cms_quality_measures``. The other
+measures are gaps: their formulas are not in this repo, and this module
+does not invent them.
+"""
+
+from __future__ import annotations
+
+
+def reconciled() -> tuple[str, ...]:
+    """Targets whose formulas already live in the ACO quality measures."""
+    return ("Q.UAMCC", "Q.ACR")
+
+
+def known_gaps() -> tuple[tuple[str, str], ...]:
+    """Target id and why this package does not compute it."""
+    return (
+        ("Q.DAH", "CMS DAH formula is not in this repo"),
+        ("Q.TFU", "CMS TFU formula is not in this repo"),
+        ("Q.CAHPS", "CAHPS results are an input, not a formula"),
+        ("HN.1", "Appendix B diagnosis list is not in this repo"),
+        ("HN.2", "Appendix B DME list is not in this repo"),
+        ("HN.3", "Kim index weights and cutoff are not in this repo"),
+        ("RA.PROSP", "V28 prospective software is not parsed here"),
+        ("RA.CONC", "Concurrent model coefficients are not in this repo"),
+        ("RA.ESRD", "ESRD model internals are not in this repo"),
+    )
diff --git a/src/lead/spine.py b/src/lead/spine.py
new file mode 100644
index 0000000..7246e3d
--- /dev/null
+++ b/src/lead/spine.py
@@ -0,0 +1,46 @@
+"""Geography and time keys for the public-data spine.
+
+A county FIPS and a ZCTA are each a key. This module does not carry a
+national crosswalk, so the two cannot be joined here.
+"""
+
+from __future__ import annotations
+
+import re
+
+_FIVE = re.compile(r"\d{5}")
+
+
+def _five(value: str) -> str:
+    if _FIVE.fullmatch(value) is None:
+        raise ValueError(value)
+    return value
+
+
+def geography_key(*, fips: str = "", zcta: str = "") -> str:
+    """One validated 5-digit FIPS or ZCTA.
+
+    Raises:
+        ValueError: both are set, neither is set, or the value is not
+            five digits.
+    """
+    if fips and zcta:
+        raise ValueError("no national crosswalk")
+    if fips:
+        return _five(fips)
+    if zcta:
+        return _five(zcta)
+    raise ValueError("geography is required")
+
+
+def time_key(year: int, quarter: int) -> str:
+    """``YYYYQ`` for a year in 1900..2100 and a quarter in 1..4.
+
+    Raises:
+        ValueError: the year or the quarter is outside that range.
+    """
+    if year < 1900 or year > 2100:
+        raise ValueError(year)
+    if quarter not in {1, 2, 3, 4}:
+        raise ValueError(quarter)
+    return f"{year}Q{quarter}"
diff --git a/tests/lead/test_manifest.py b/tests/lead/test_manifest.py
new file mode 100644
index 0000000..900482d
--- /dev/null
+++ b/tests/lead/test_manifest.py
@@ -0,0 +1,101 @@
+"""The enrichment manifest loads into local DuckDB. It does not download."""
+
+from __future__ import annotations
+
+import pytest
+from typer.testing import CliRunner
+
+from cli import app
+from lead.ledger.schema import schema_sql
+from lead.ledger.store import open_ledger
+from lead.manifest import RELEASE_DDL, load_manifest
+
+runner = CliRunner()
+
+_HEADER = (
+    "source_id,title,publisher,kind,url,api_endpoint,landing_page,release,"
+    "fetched_utc,status,path,sha256,rows,size_bytes,license_terms,lead_use"
+)
+
+
+def _csv(tmp_path, body: str):
+    path = tmp_path / "manifest.csv"
+    path.write_text(_HEADER + "\n" + body)
+    return path
+
+
+def test_load_is_idempotent_and_tags_synthetic_rows(tmp_path):
+    csv_path = _csv(
+        tmp_path,
+        "ahrq,Title,Pub,table,https://example.test/a,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,abc,1,2,public,benchmark\n"
+        "tuva,Title,Pub,table,https://example.test/b,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,def,1,2,public,\n"
+        "cms,Title,Pub,table,https://example.test/c,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,ghi,1,2,public,synthetic person file\n",
+    )
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        assert load_manifest(con, str(csv_path), actor="tester") == 3
+        assert load_manifest(con, str(csv_path), actor="tester") == 0
+        rows = con.execute(
+            """
+            SELECT source_id, url, path, layer
+            FROM lead_evidence.public_release
+            ORDER BY source_id
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    by_source = {row[0]: row for row in rows}
+    assert by_source["ahrq"][1] == "https://example.test/a"
+    assert by_source["ahrq"][2] == "/no/such/lead-file"
+    assert by_source["ahrq"][3] == ""
+    assert by_source["tuva"][3] == "synthetic"
+    assert by_source["cms"][3] == "synthetic"
+
+
+def test_a_blank_sha256_raises_and_inserts_nothing(tmp_path):
+    csv_path = _csv(
+        tmp_path,
+        "ahrq,Title,Pub,table,https://example.test/a,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,,1,2,public,benchmark\n",
+    )
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        with pytest.raises(ValueError):
+            load_manifest(con, str(csv_path), actor="tester")
+        count = con.execute(
+            "SELECT count(*) FROM lead_evidence.public_release"
+        ).fetchone()[0]
+        assert count == 0
+    finally:
+        con.close()
+
+
+def test_release_ddl_and_delta_schema_omit_the_catalog_token():
+    assert "aco" not in RELEASE_DDL
+    assert "aco" not in schema_sql("delta")
+
+
+def test_manifest_load_command_prints_the_insert_count(tmp_path):
+    csv_path = _csv(
+        tmp_path,
+        "ahrq,Title,Pub,table,https://example.test/a,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,abc,1,2,public,benchmark\n",
+    )
+    db = tmp_path / "ledger.duckdb"
+    result = runner.invoke(
+        app,
+        [
+            "lead",
+            "manifest",
+            "load",
+            "--csv",
+            str(csv_path),
+            "--db",
+            str(db),
+        ],
+    )
+    assert result.exit_code == 0
+    assert "inserted\t1" in result.stdout
diff --git a/tests/lead/test_quality.py b/tests/lead/test_quality.py
new file mode 100644
index 0000000..055ffe7
--- /dev/null
+++ b/tests/lead/test_quality.py
@@ -0,0 +1,27 @@
+"""UAMCC and ACR stay where they are. The missing measures are gaps."""
+
+from __future__ import annotations
+
+from aco.express.cms_quality_measures import acr_summary, uamcc_summary
+from lead.quality import known_gaps, reconciled
+
+
+def test_uamcc_and_acr_are_reconciled_to_the_existing_measures():
+    assert reconciled() == ("Q.UAMCC", "Q.ACR")
+    assert callable(uamcc_summary)
+    assert callable(acr_summary)
+
+
+def test_unpublished_formulas_are_named_gaps():
+    targets = {target for target, _reason in known_gaps()}
+    assert {
+        "Q.DAH",
+        "Q.TFU",
+        "Q.CAHPS",
+        "HN.1",
+        "HN.2",
+        "HN.3",
+        "RA.PROSP",
+        "RA.CONC",
+        "RA.ESRD",
+    } <= targets
diff --git a/tests/lead/test_spine.py b/tests/lead/test_spine.py
new file mode 100644
index 0000000..96a432a
--- /dev/null
+++ b/tests/lead/test_spine.py
@@ -0,0 +1,34 @@
+"""Geography and time keys. No national crosswalk."""
+
+from __future__ import annotations
+
+import pytest
+
+from lead.spine import geography_key, time_key
+
+
+def test_a_five_digit_fips_is_the_geography_key():
+    assert geography_key(fips="01001") == "01001"
+
+
+def test_a_short_fips_raises():
+    with pytest.raises(ValueError):
+        geography_key(fips="1001")
+
+
+def test_a_five_digit_zcta_is_the_geography_key():
+    assert geography_key(zcta="02138") == "02138"
+
+
+def test_fips_and_zcta_together_raise():
+    with pytest.raises(ValueError):
+        geography_key(fips="01001", zcta="02138")
+
+
+def test_time_key_is_year_and_quarter():
+    assert time_key(2027, 4) == "2027Q4"
+
+
+def test_a_quarter_outside_1_to_4_raises():
+    with pytest.raises(ValueError):
+        time_key(2027, 5)
-- 
2.49.1


From 97816d60afc065ea5dd13dd7b8363759da359cc7 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:06:25 -0400
Subject: [PATCH 26/30] feat(lead): gate promotion and record discarded runs

---
 src/aco/lake/lineage.py           |  16 ++++
 src/lead/gates.py                 | 140 ++++++++++++++++++++++++++++
 tests/lead/test_gates.py          | 148 ++++++++++++++++++++++++++++++
 tests/lead/test_ledger_lineage.py |  16 ++++
 4 files changed, 320 insertions(+)
 create mode 100644 src/lead/gates.py
 create mode 100644 tests/lead/test_gates.py
 create mode 100644 tests/lead/test_ledger_lineage.py

diff --git a/src/aco/lake/lineage.py b/src/aco/lake/lineage.py
index e10eb9a..33f5631 100644
--- a/src/aco/lake/lineage.py
+++ b/src/aco/lake/lineage.py
@@ -33,6 +33,12 @@ class LineageGraph:
         """Return direct input tables for a given output table."""
         return self.table_edges.get(table_ref, [])
 
+    def table_outputs(self, table_ref: str) -> list[str]:
+        """Return tables that list ``table_ref`` as an input."""
+        return sorted(
+            output for output, inputs in self.table_edges.items() if table_ref in inputs
+        )
+
     def all_tables(self) -> list[str]:
         """Return all tables in the lineage graph."""
         tables = set(self.table_edges.keys())
@@ -98,4 +104,14 @@ def build_lineage() -> LineageGraph:
                 if col_sources:
                     graph.column_sources[output_name] = col_sources
 
+    for output, inputs in _LEDGER_EDGES.items():
+        graph.table_edges[output] = list(inputs)
     return graph
+
+
+_LEDGER_EDGES = {
+    "lead_evidence.claim": ["lead_evidence.extraction", "lead_evidence.source"],
+    "lead_evidence.feature": ["lead_evidence.claim"],
+    "lead_evidence.run": ["lead_evidence.feature", "lead_evidence.claim"],
+    "lead_evidence.finding": ["lead_evidence.claim", "lead_evidence.run"],
+}
diff --git a/src/lead/gates.py b/src/lead/gates.py
new file mode 100644
index 0000000..ebc307f
--- /dev/null
+++ b/src/lead/gates.py
@@ -0,0 +1,140 @@
+"""Promotion gates and the record of runs that did not pass them.
+
+A candidate is promoted only when every gate holds. A failed or discarded
+run still appends one run row and one finding row. The actor is not part
+of either hash.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+
+from lead.ledger.store import append
+from lead.targets import by_id
+
+_RELATIONS = frozenset({"failed", "discarded", "promoted"})
+
+
+@dataclass(frozen=True)
+class GateInput:
+    """What one candidate showed, out of time, against named claims."""
+
+    beats_automl: bool
+    beats_naive: bool
+    train_through: int
+    test_year: int
+    calibration_error: float
+    calibration_tolerance: float
+    conformal_coverage: float
+    coverage_target: float
+    subgroup_pass: bool
+    feature_claim_ids: tuple[str, ...]
+    known_claim_ids: frozenset[str]
+
+
+@dataclass(frozen=True)
+class GateResult:
+    """Empty reasons means the candidate is promoted."""
+
+    promoted: bool
+    reasons: tuple[str, ...]
+
+
+def promotion_gates(item: GateInput) -> GateResult:
+    """Check the Part 3 gates. Reasons stay in a fixed order."""
+    reasons: list[str] = []
+    if not item.beats_automl or not item.beats_naive:
+        reasons.append("baseline")
+    if item.test_year != item.train_through + 1:
+        reasons.append("out_of_time")
+    if item.calibration_error > item.calibration_tolerance:
+        reasons.append("calibration")
+    if item.conformal_coverage < item.coverage_target:
+        reasons.append("coverage")
+    if not item.subgroup_pass:
+        reasons.append("subgroup")
+    if not item.feature_claim_ids or any(
+        claim_id not in item.known_claim_ids for claim_id in item.feature_claim_ids
+    ):
+        reasons.append("claim")
+    return GateResult(promoted=not reasons, reasons=tuple(reasons))
+
+
+def _hash(*parts: str) -> str:
+    return hashlib.sha256("\n".join(parts).encode()).hexdigest()
+
+
+def record_outcome(
+    con,
+    *,
+    actor: str,
+    target_id: str,
+    model: str,
+    where_ran: str,
+    finished_on: str,
+    metrics: str,
+    gate: GateResult,
+    relation: str,
+    claim_id: str = "",
+) -> tuple[bool, bool]:
+    """Append a run and a finding. Return whether each insert was new.
+
+    Raises:
+        ValueError: ``relation`` is not failed, discarded, or promoted.
+        KeyError: ``target_id`` is not in the registry.
+    """
+    if relation not in _RELATIONS:
+        raise ValueError(relation)
+    by_id(target_id)
+    reasons = ",".join(gate.reasons)
+    run_hash = _hash(
+        target_id, model, where_ran, finished_on, metrics, reasons, relation
+    )
+    finding_hash = _hash(run_hash, relation, claim_id)
+    run_wrote = append(
+        con,
+        "run",
+        {
+            "id": run_hash,
+            "target_id": target_id,
+            "where_ran": where_ran,
+            "finished_on": finished_on,
+            "model": model,
+            "metrics": metrics,
+            "gate_results": reasons,
+            "content_hash": run_hash,
+            "actor": actor,
+        },
+    )
+    finding: dict[str, object] = {
+        "id": finding_hash,
+        "relation": relation,
+        "numbers": reasons,
+        "content_hash": finding_hash,
+        "actor": actor,
+    }
+    if claim_id:
+        finding["claim_id"] = claim_id
+    finding_wrote = append(con, "finding", finding)
+    return run_wrote, finding_wrote
+
+
+def events_in_window(
+    *,
+    window_start: str,
+    alarm_on: str,
+    events: Sequence[Mapping[str, str]],
+) -> tuple[str, ...]:
+    """Citations whose day falls inside the window, including both ends."""
+    start = window_start[:10]
+    end = alarm_on[:10]
+    found: list[str] = []
+    for event in events:
+        day = str(event.get("event_on") or "")[:10]
+        if start <= day <= end:
+            found.append(
+                str(event.get("citation") or event.get("series_or_event") or "")
+            )
+    return tuple(found)
diff --git a/tests/lead/test_gates.py b/tests/lead/test_gates.py
new file mode 100644
index 0000000..261a97c
--- /dev/null
+++ b/tests/lead/test_gates.py
@@ -0,0 +1,148 @@
+"""Promotion gates, the failure record, and drift windows."""
+
+from __future__ import annotations
+
+import pytest
+
+from lead.gates import (
+    GateInput,
+    GateResult,
+    events_in_window,
+    promotion_gates,
+    record_outcome,
+)
+from lead.ledger.store import open_ledger
+
+
+def _passing(**overrides: object) -> GateInput:
+    base: dict[str, object] = {
+        "beats_automl": True,
+        "beats_naive": True,
+        "train_through": 2026,
+        "test_year": 2027,
+        "calibration_error": 0.01,
+        "calibration_tolerance": 0.05,
+        "conformal_coverage": 0.91,
+        "coverage_target": 0.90,
+        "subgroup_pass": True,
+        "feature_claim_ids": ("claim-1",),
+        "known_claim_ids": frozenset({"claim-1"}),
+    }
+    base.update(overrides)
+    return GateInput(**base)  # type: ignore[arg-type]
+
+
+def test_a_candidate_that_clears_every_gate_is_promoted():
+    result = promotion_gates(_passing())
+    assert result == GateResult(promoted=True, reasons=())
+
+
+@pytest.mark.parametrize(
+    ("overrides", "reason"),
+    [
+        ({"beats_automl": False}, "baseline"),
+        ({"beats_naive": False}, "baseline"),
+        ({"test_year": 2026}, "out_of_time"),
+        ({"test_year": 2028}, "out_of_time"),
+        ({"calibration_error": 0.06}, "calibration"),
+        ({"conformal_coverage": 0.89}, "coverage"),
+        ({"subgroup_pass": False}, "subgroup"),
+        ({"feature_claim_ids": ()}, "claim"),
+        ({"feature_claim_ids": ("missing",)}, "claim"),
+    ],
+)
+def test_each_failed_gate_is_one_reason(overrides, reason):
+    result = promotion_gates(_passing(**overrides))
+    assert result.promoted is False
+    assert result.reasons == (reason,)
+
+
+def test_a_discarded_run_is_recorded_once(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    gate = GateResult(promoted=False, reasons=("baseline",))
+    try:
+        assert record_outcome(
+            con,
+            actor="tester",
+            target_id="Q.ACR",
+            model="glm",
+            where_ran="local",
+            finished_on="2026-10-31",
+            metrics="mae=1",
+            gate=gate,
+            relation="discarded",
+        ) == (True, True)
+        assert record_outcome(
+            con,
+            actor="other",
+            target_id="Q.ACR",
+            model="glm",
+            where_ran="local",
+            finished_on="2026-10-31",
+            metrics="mae=1",
+            gate=gate,
+            relation="discarded",
+        ) == (False, False)
+        runs = con.execute("SELECT count(*) FROM lead_evidence.run").fetchone()[0]
+        findings = con.execute("SELECT count(*) FROM lead_evidence.finding").fetchone()[
+            0
+        ]
+        assert runs == 1
+        assert findings == 1
+    finally:
+        con.close()
+
+
+def test_a_promoted_run_is_recorded(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        wrote = record_outcome(
+            con,
+            actor="tester",
+            target_id="Q.ACR",
+            model="glm",
+            where_ran="local",
+            finished_on="2026-10-31",
+            metrics="mae=1",
+            gate=promotion_gates(_passing()),
+            relation="promoted",
+            claim_id="claim-1",
+        )
+        assert wrote == (True, True)
+    finally:
+        con.close()
+
+
+def test_an_unknown_relation_writes_nothing(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        with pytest.raises(ValueError):
+            record_outcome(
+                con,
+                actor="tester",
+                target_id="Q.ACR",
+                model="glm",
+                where_ran="local",
+                finished_on="2026-10-31",
+                metrics="mae=1",
+                gate=GateResult(promoted=False, reasons=("baseline",)),
+                relation="other",
+            )
+        assert con.execute("SELECT count(*) FROM lead_evidence.run").fetchone()[0] == 0
+        assert (
+            con.execute("SELECT count(*) FROM lead_evidence.finding").fetchone()[0] == 0
+        )
+    finally:
+        con.close()
+
+
+def test_drift_window_includes_both_ends():
+    events = (
+        {"event_on": "2026-10-01", "citation": "start"},
+        {"event_on": "2026-10-31", "citation": "alarm"},
+        {"event_on": "2026-09-30", "citation": "before"},
+        {"event_on": "2026-11-01", "citation": "after"},
+    )
+    assert events_in_window(
+        window_start="2026-10-01", alarm_on="2026-10-31", events=events
+    ) == ("start", "alarm")
diff --git a/tests/lead/test_ledger_lineage.py b/tests/lead/test_ledger_lineage.py
new file mode 100644
index 0000000..8abdb90
--- /dev/null
+++ b/tests/lead/test_ledger_lineage.py
@@ -0,0 +1,16 @@
+"""Ledger tables walk backward to sources and forward to runs."""
+
+from __future__ import annotations
+
+from aco.lake.lineage import build_lineage
+
+
+def test_a_claim_lists_its_inputs_and_its_outputs():
+    graph = build_lineage()
+    deps = graph.table_deps("lead_evidence.claim")
+    assert "lead_evidence.extraction" in deps
+    assert "lead_evidence.source" in deps
+    outputs = graph.table_outputs("lead_evidence.claim")
+    assert "lead_evidence.feature" in outputs
+    assert "lead_evidence.run" in outputs
+    assert "lead_evidence.finding" in outputs
-- 
2.49.1


From 370f64f1933cb48bb42cae93910d6b9993779057 Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:08:31 -0400
Subject: [PATCH 27/30] feat(lead): add the agent tool surface

---
 src/lead/model.py        |  24 ++++
 src/lead/tools.py        | 256 +++++++++++++++++++++++++++++++++++++++
 tests/lead/test_model.py |  22 ++++
 tests/lead/test_tools.py | 236 ++++++++++++++++++++++++++++++++++++
 4 files changed, 538 insertions(+)
 create mode 100644 src/lead/model.py
 create mode 100644 src/lead/tools.py
 create mode 100644 tests/lead/test_model.py
 create mode 100644 tests/lead/test_tools.py

diff --git a/src/lead/model.py b/src/lead/model.py
new file mode 100644
index 0000000..3070b3c
--- /dev/null
+++ b/src/lead/model.py
@@ -0,0 +1,24 @@
+"""Version stamp for a weights hash and an adapter.
+
+Choosing the open-weight model, training a LoRA adapter, and serving the
+weights are not done here.
+"""
+
+from __future__ import annotations
+
+MODEL_GAPS = (
+    "strand 3 has not named an open-weight model",
+    "LoRA training is not run here",
+    "the same weights are not served from this module",
+)
+
+
+def stamp(weights_hash: str, adapter_version: str) -> str:
+    """Join a weights hash and an adapter version.
+
+    Raises:
+        ValueError: either half is empty.
+    """
+    if not weights_hash or not adapter_version:
+        raise ValueError("a stamp needs a weights hash and an adapter version")
+    return f"{weights_hash}+{adapter_version}"
diff --git a/src/lead/tools.py b/src/lead/tools.py
new file mode 100644
index 0000000..05ecad3
--- /dev/null
+++ b/src/lead/tools.py
@@ -0,0 +1,256 @@
+"""Tool surface for the LEAD agent.
+
+These functions are what an MCP server would expose. The MCP SDK is not a
+project dependency, so callers use this module. The agent writes ledger
+rows only. A Databricks run needs an injected transport and room under the
+spend budget. Neither check calls the network.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Mapping, Sequence
+
+from lead.ledger.store import append
+from lead.limits import (
+    earlier_qualification_headroom,
+    normalization_sensitivity,
+    shrinkage_weight,
+)
+from lead.targets import TARGETS, by_id
+from perf import tracer as tracer_factory
+from prisma.search.due import due
+
+
+def _traced(name: str):
+    return tracer_factory("lead.tools").start_as_current_span(name)
+
+
+def _hash(*parts: str) -> str:
+    return hashlib.sha256("\n".join(parts).encode()).hexdigest()
+
+
+def _row(con, table: str, row_id: str) -> dict[str, object] | None:
+    fetched = con.execute(
+        f"SELECT * FROM lead_evidence.{table} WHERE id = ?",
+        [row_id],
+    ).fetchone()
+    if fetched is None:
+        return None
+    columns = [item[0] for item in con.description]
+    return dict(zip(columns, fetched, strict=True))
+
+
+def ledger_search(
+    con, *, target_id: str = "", text: str = ""
+) -> list[dict[str, object]]:
+    """Claims for a target whose subject contains ``text``."""
+    with _traced("ledger_search"):
+        sql = (
+            "SELECT id, target_id, subject, tiers, certainty "
+            "FROM lead_evidence.claim WHERE 1 = 1"
+        )
+        params: list[str] = []
+        if target_id:
+            sql += " AND target_id = ?"
+            params.append(target_id)
+        if text:
+            sql += " AND lower(subject) LIKE ?"
+            params.append(f"%{text.casefold()}%")
+        sql += " ORDER BY id"
+        rows = con.execute(sql, params).fetchall()
+        columns = ("id", "target_id", "subject", "tiers", "certainty")
+        return [dict(zip(columns, row, strict=True)) for row in rows]
+
+
+def ledger_get_claim(con, claim_id: str) -> dict[str, object] | None:
+    """One claim by id, or None."""
+    with _traced("ledger_get_claim"):
+        return _row(con, "claim", claim_id)
+
+
+def targets_list() -> tuple:
+    """Every registered target."""
+    with _traced("targets_list"):
+        return tuple(TARGETS)
+
+
+def limits_compute(kind: str, **kwargs: float):
+    """Dispatch one limits expression.
+
+    Raises:
+        ValueError: ``kind`` is not headroom, normalization, or shrinkage.
+    """
+    with _traced("limits_compute"):
+        if kind == "headroom":
+            return earlier_qualification_headroom(kwargs["prior"], kwargs["caught"])
+        if kind == "normalization":
+            return normalization_sensitivity(kwargs["benchmark"], kwargs["factor"])
+        if kind == "shrinkage":
+            return shrinkage_weight(int(kwargs["aco_n"]), int(kwargs["prior_n"]))
+        raise ValueError(kind)
+
+
+def features_validate(con, *, target_id: str, claim_ids: Sequence[str]) -> None:
+    """Raise unless the target exists and every claim id is stored.
+
+    Raises:
+        KeyError: ``target_id`` is unknown.
+        ValueError: the claim list is empty or a claim id is missing.
+    """
+    with _traced("features_validate"):
+        by_id(target_id)
+        if not claim_ids:
+            raise ValueError("a feature needs a claim id")
+        for claim_id in claim_ids:
+            found = con.execute(
+                "SELECT 1 FROM lead_evidence.claim WHERE id = ?",
+                [claim_id],
+            ).fetchone()
+            if found is None:
+                raise ValueError(claim_id)
+
+
+def features_propose(
+    con,
+    *,
+    target_id: str,
+    claim_ids: Sequence[str],
+    expression: str,
+    actor: str,
+    grain: str = "",
+) -> bool:
+    """Validate the claims, then append one feature. The actor is not hashed."""
+    with _traced("features_propose"):
+        features_validate(con, target_id=target_id, claim_ids=claim_ids)
+        joined = ",".join(claim_ids)
+        content = _hash(target_id, joined, expression, grain)
+        return append(
+            con,
+            "feature",
+            {
+                "id": content,
+                "target_id": target_id,
+                "claim_ids": joined,
+                "expression": expression,
+                "grain": grain,
+                "content_hash": content,
+                "actor": actor,
+            },
+        )
+
+
+def runs_start(
+    con,
+    *,
+    where_ran: str,
+    target_id: str,
+    actor: str,
+    model: str,
+    transport=None,
+    spent: float = 0.0,
+    budget: float = 0.0,
+    finished_on: str = "",
+    metrics: str = "",
+    feature_ids: str = "",
+    gate_results: str = "",
+) -> str:
+    """Record a local run, or a Databricks run through an injected transport.
+
+    A Databricks call raises when the spend is at the budget or when no
+    transport was injected. Both raises happen before the transport is
+    called, and neither writes a row. A local run never calls a transport.
+
+    Returns the run id.
+    """
+    with _traced("runs_start"):
+        by_id(target_id)
+        if where_ran not in {"local", "databricks"}:
+            raise ValueError(where_ran)
+        if where_ran == "databricks":
+            if spent >= budget:
+                raise RuntimeError("databricks budget")
+            if transport is None:
+                raise RuntimeError("databricks transport is required")
+            transport({"target_id": target_id, "model": model})
+        content = _hash(
+            target_id, model, where_ran, finished_on, metrics, feature_ids, gate_results
+        )
+        append(
+            con,
+            "run",
+            {
+                "id": content,
+                "target_id": target_id,
+                "where_ran": where_ran,
+                "finished_on": finished_on,
+                "model": model,
+                "metrics": metrics,
+                "feature_ids": feature_ids,
+                "gate_results": gate_results,
+                "content_hash": content,
+                "actor": actor,
+            },
+        )
+        return content
+
+
+def runs_get(con, run_id: str) -> dict[str, object] | None:
+    """One run by id, or None."""
+    with _traced("runs_get"):
+        return _row(con, "run", run_id)
+
+
+def runs_compare(con, left_id: str, right_id: str) -> dict[str, str]:
+    """Metric strings for two stored runs.
+
+    Raises:
+        KeyError: either id is missing.
+    """
+    with _traced("runs_compare"):
+        left = runs_get(con, left_id)
+        right = runs_get(con, right_id)
+        if left is None or right is None:
+            raise KeyError(left_id if left is None else right_id)
+        return {
+            "left_metrics": str(left.get("metrics") or ""),
+            "right_metrics": str(right.get("metrics") or ""),
+            "same_target": str(left.get("target_id") == right.get("target_id")),
+        }
+
+
+def failures_check(con, target_id: str) -> list[dict[str, object]]:
+    """Runs for this target whose gate results are nonempty."""
+    with _traced("failures_check"):
+        rows = con.execute(
+            """
+            SELECT id, target_id, gate_results, model
+            FROM lead_evidence.run
+            WHERE target_id = ?
+              AND gate_results IS NOT NULL
+              AND gate_results <> ''
+            ORDER BY id
+            """,
+            [target_id],
+        ).fetchall()
+        return [
+            {
+                "id": row[0],
+                "target_id": row[1],
+                "gate_results": row[2],
+                "model": row[3],
+            }
+            for row in rows
+        ]
+
+
+def findings_record(con, row: Mapping[str, object]) -> bool:
+    """Append one finding. The caller supplies the content hash."""
+    with _traced("findings_record"):
+        return append(con, "finding", row)
+
+
+def review_trigger_search(con, *, project: str, now: str) -> tuple[str, ...]:
+    """Reasons a search is due. This does not run a search."""
+    with _traced("review_trigger_search"):
+        return due(con, project=project, now=now)
diff --git a/tests/lead/test_model.py b/tests/lead/test_model.py
new file mode 100644
index 0000000..be6a078
--- /dev/null
+++ b/tests/lead/test_model.py
@@ -0,0 +1,22 @@
+"""Adapter stamps. Training and serving stay gaps."""
+
+from __future__ import annotations
+
+import pytest
+
+from lead.model import MODEL_GAPS, stamp
+
+
+def test_stamp_joins_the_weights_hash_and_the_adapter_version():
+    assert stamp("abc", "v1") == "abc+v1"
+
+
+def test_a_stamp_needs_both_halves():
+    with pytest.raises(ValueError):
+        stamp("", "v1")
+    with pytest.raises(ValueError):
+        stamp("abc", "")
+
+
+def test_model_gaps_are_recorded():
+    assert MODEL_GAPS
diff --git a/tests/lead/test_tools.py b/tests/lead/test_tools.py
new file mode 100644
index 0000000..2dd2e0b
--- /dev/null
+++ b/tests/lead/test_tools.py
@@ -0,0 +1,236 @@
+"""The agent writes ledger rows. Databricks stays behind an injected transport."""
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+
+import pytest
+
+from lead.ledger.store import append, open_ledger
+from lead.tools import (
+    failures_check,
+    features_propose,
+    features_validate,
+    findings_record,
+    ledger_get_claim,
+    ledger_search,
+    limits_compute,
+    review_trigger_search,
+    runs_compare,
+    runs_get,
+    runs_start,
+    targets_list,
+    tracer_factory,
+)
+from perf import tracer
+
+
+def _claim(con, claim_id: str = "claim-1", subject: str = "Frailty index") -> None:
+    append(
+        con,
+        "claim",
+        {
+            "id": claim_id,
+            "kind": "precursor",
+            "subject": subject,
+            "target_id": "Q.ACR",
+            "tiers": "literature",
+            "content_hash": claim_id,
+            "actor": "tester",
+        },
+    )
+
+
+def test_the_default_tracer_factory_is_perf():
+    assert tracer_factory is tracer
+
+
+def test_ledger_search_filters_and_is_traced(tmp_path, monkeypatch):
+    class Recording:
+        def __init__(self) -> None:
+            self.names: list[str] = []
+
+        def __call__(self, scope: str) -> Recording:
+            assert scope == "lead.tools"
+            return self
+
+        @contextmanager
+        def start_as_current_span(self, name: str, **_kwargs: object):
+            self.names.append(name)
+            yield None
+
+    recording = Recording()
+    monkeypatch.setattr("lead.tools.tracer_factory", recording)
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        _claim(con, "claim-1", "Frailty index")
+        _claim(con, "claim-2", "Other topic")
+        found = ledger_search(con, target_id="Q.ACR", text="FRAIL")
+        assert [row["id"] for row in found] == ["claim-1"]
+        assert ledger_get_claim(con, "claim-1")["subject"] == "Frailty index"
+        assert ledger_get_claim(con, "missing") is None
+    finally:
+        con.close()
+    assert "ledger_search" in recording.names
+    assert "ledger_get_claim" in recording.names
+
+
+def test_targets_include_quality_and_high_needs():
+    ids = {target.id for target in targets_list()}
+    assert "Q.ACR" in ids
+    assert "HN.4" in ids
+
+
+def test_limits_compute_dispatches_the_three_expressions():
+    assert limits_compute("headroom", prior=1.0, caught=1.05).scored == 1.04
+    assert limits_compute("normalization", benchmark=1000.0, factor=1.25) == 800.0
+    assert limits_compute("shrinkage", aco_n=100, prior_n=300) == 0.25
+    with pytest.raises(ValueError):
+        limits_compute("nope")
+
+
+def test_a_feature_needs_a_known_target_and_a_stored_claim(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        with pytest.raises(KeyError):
+            features_validate(con, target_id="NO.SUCH", claim_ids=("claim-1",))
+        with pytest.raises(ValueError):
+            features_validate(con, target_id="Q.ACR", claim_ids=())
+        with pytest.raises(ValueError):
+            features_validate(con, target_id="Q.ACR", claim_ids=("missing",))
+        _claim(con)
+        assert (
+            features_propose(
+                con,
+                target_id="Q.ACR",
+                claim_ids=("claim-1",),
+                expression="kim_index",
+                actor="tester",
+            )
+            is True
+        )
+        assert (
+            features_propose(
+                con,
+                target_id="Q.ACR",
+                claim_ids=("claim-1",),
+                expression="kim_index",
+                actor="other",
+            )
+            is False
+        )
+    finally:
+        con.close()
+
+
+def test_local_runs_do_not_call_a_transport(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+
+    def boom(_row: dict) -> None:
+        raise AssertionError("transport called")
+
+    try:
+        run_id = runs_start(
+            con,
+            where_ran="local",
+            target_id="Q.ACR",
+            actor="tester",
+            model="glm",
+            transport=boom,
+            finished_on="2026-10-31",
+            metrics="mae=1",
+        )
+        assert runs_get(con, run_id)["metrics"] == "mae=1"
+    finally:
+        con.close()
+
+
+def test_databricks_refuses_before_calling_a_transport(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    called: list[dict] = []
+    try:
+        with pytest.raises(RuntimeError):
+            runs_start(
+                con,
+                where_ran="databricks",
+                target_id="Q.ACR",
+                actor="tester",
+                model="glm",
+                transport=None,
+                spent=0,
+                budget=1,
+            )
+        with pytest.raises(RuntimeError):
+            runs_start(
+                con,
+                where_ran="databricks",
+                target_id="Q.ACR",
+                actor="tester",
+                model="glm",
+                transport=called.append,
+                spent=1,
+                budget=1,
+            )
+        assert called == []
+        assert con.execute("SELECT count(*) FROM lead_evidence.run").fetchone()[0] == 0
+    finally:
+        con.close()
+
+
+def test_an_injected_databricks_transport_records_the_run(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    called: list[dict] = []
+    try:
+        left = runs_start(
+            con,
+            where_ran="databricks",
+            target_id="Q.ACR",
+            actor="tester",
+            model="glm",
+            transport=called.append,
+            spent=0,
+            budget=1,
+            finished_on="2026-10-31",
+            metrics="mae=1",
+        )
+        right = runs_start(
+            con,
+            where_ran="local",
+            target_id="Q.ACR",
+            actor="tester",
+            model="glm",
+            finished_on="2026-11-01",
+            metrics="mae=2",
+            gate_results="baseline",
+        )
+        assert len(called) == 1
+        assert runs_compare(con, left, right)["left_metrics"] == "mae=1"
+        assert runs_compare(con, left, right)["right_metrics"] == "mae=2"
+        failed = failures_check(con, "Q.ACR")
+        assert [row["id"] for row in failed] == [right]
+        with pytest.raises(KeyError):
+            runs_compare(con, left, "missing")
+    finally:
+        con.close()
+
+
+def test_findings_and_search_triggers_write_only_the_ledger(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        assert (
+            findings_record(
+                con,
+                {
+                    "relation": "contradicts",
+                    "numbers": "1",
+                    "content_hash": "finding-1",
+                    "actor": "tester",
+                },
+            )
+            is True
+        )
+        assert review_trigger_search(
+            con, project="lead-precursors", now="2026-10-31"
+        ) == ("nightly",)
+    finally:
+        con.close()
-- 
2.49.1


From d6590a2357cc8879943c00fb23502f25a71dd91d Mon Sep 17 00:00:00 2001
From: kert 
Date: Sat, 31 Oct 2026 22:28:52 -0400
Subject: [PATCH 28/30] fix(lead): keep every manifest row and refuse a
 repeated Databricks start

---
 src/lead/gates.py           | 11 +++++++--
 src/lead/manifest.py        | 42 ++++++++++++++++++++-----------
 src/lead/tools.py           |  8 +++---
 tests/lead/test_gates.py    |  9 +++++++
 tests/lead/test_manifest.py | 49 ++++++++++++++++++++++++++++++++++++-
 tests/lead/test_tools.py    | 22 +++++++++++++++++
 6 files changed, 121 insertions(+), 20 deletions(-)

diff --git a/src/lead/gates.py b/src/lead/gates.py
index ebc307f..0747173 100644
--- a/src/lead/gates.py
+++ b/src/lead/gates.py
@@ -8,6 +8,7 @@ of either hash.
 from __future__ import annotations
 
 import hashlib
+import math
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 
@@ -49,9 +50,15 @@ def promotion_gates(item: GateInput) -> GateResult:
         reasons.append("baseline")
     if item.test_year != item.train_through + 1:
         reasons.append("out_of_time")
-    if item.calibration_error > item.calibration_tolerance:
+    if (
+        math.isnan(item.calibration_error)
+        or item.calibration_error > item.calibration_tolerance
+    ):
         reasons.append("calibration")
-    if item.conformal_coverage < item.coverage_target:
+    if (
+        math.isnan(item.conformal_coverage)
+        or item.conformal_coverage < item.coverage_target
+    ):
         reasons.append("coverage")
     if not item.subgroup_pass:
         reasons.append("subgroup")
diff --git a/src/lead/manifest.py b/src/lead/manifest.py
index d4bd03d..fb30149 100644
--- a/src/lead/manifest.py
+++ b/src/lead/manifest.py
@@ -80,24 +80,31 @@ def _ensure(con) -> None:
             con.execute(text)
 
 
-def _identity(row: dict[str, str]) -> tuple[str, str, str]:
+def _fields(row: dict[str, str], line_no: int) -> tuple[str, str, str, str, str]:
     source_id = (row.get("source_id") or "").strip()
     release = (row.get("release") or "").strip()
     digest = (row.get("sha256") or "").strip()
+    path = (row.get("path") or "").strip()
+    url = (row.get("url") or "").strip()
     if not source_id:
-        raise ValueError("source_id is required")
+        raise ValueError(f"row {line_no}: source_id is required")
     if not release:
-        raise ValueError("release is required")
-    if not digest:
-        raise ValueError("sha256 is required")
-    return source_id, release, digest
+        raise ValueError(f"row {line_no}: release is required ({source_id})")
+    return source_id, release, digest, path, url
 
 
 def load_manifest(con, csv_path: str, *, actor: str) -> int:
     """Insert new release rows. Return how many were new.
 
+    A row is identified by source, release, sha256, path, and URL, so two
+    paths that share a file hash both stay. A file whose every row lacks a
+    sha256 raises and inserts nothing. A blank hash beside real rows is
+    stored with an empty sha256 and layer ``gap``. The file hash is not
+    invented.
+
     Raises:
-        ValueError: a required column or a required key is missing.
+        ValueError: a required column is missing, a row has no source or
+            release, or no row has a sha256.
     """
     _ensure(con)
     with Path(csv_path).open(newline="") as handle:
@@ -106,22 +113,29 @@ def load_manifest(con, csv_path: str, *, actor: str) -> int:
         if missing:
             raise ValueError(f"missing columns: {missing}")
         raw = list(reader)
-    identities = [_identity(row) for row in raw]
+    parsed = [(i, row, _fields(row, i)) for i, row in enumerate(raw, start=1)]
+    if parsed and all(not fields[2] for _line, _row, fields in parsed):
+        named = "; ".join(
+            f"row {line_no}: sha256 is required ({fields[0]} {fields[1]})"
+            for line_no, _row, fields in parsed
+        )
+        raise ValueError(named)
     inserted = 0
-    for row, (source_id, release, digest) in zip(raw, identities, strict=True):
+    for _line, row, (source_id, release, digest, path, url) in parsed:
         content = hashlib.sha256(
-            f"{source_id}\n{release}\n{digest}\n".encode()
+            f"{source_id}\n{release}\n{digest}\n{path}\n{url}\n".encode()
         ).hexdigest()
         existing = con.execute(
             """
             SELECT 1 FROM lead_evidence.public_release
             WHERE source_id = ? AND release = ? AND sha256 = ?
+              AND path = ? AND url = ?
             """,
-            [source_id, release, digest],
+            [source_id, release, digest, path, url],
         ).fetchone()
         if existing:
             continue
-        layer = layer_of(source_id, row.get("lead_use") or "")
+        layer = "gap" if not digest else layer_of(source_id, row.get("lead_use") or "")
         con.execute(
             """
             INSERT INTO lead_evidence.public_release (
@@ -136,13 +150,13 @@ def load_manifest(con, csv_path: str, *, actor: str) -> int:
                 row.get("title") or "",
                 row.get("publisher") or "",
                 row.get("kind") or "",
-                row.get("url") or "",
+                url,
                 row.get("api_endpoint") or "",
                 row.get("landing_page") or "",
                 release,
                 row.get("fetched_utc") or "",
                 row.get("status") or "",
-                row.get("path") or "",
+                path,
                 digest,
                 row.get("rows") or "",
                 row.get("size_bytes") or "",
diff --git a/src/lead/tools.py b/src/lead/tools.py
index 05ecad3..6d8de62 100644
--- a/src/lead/tools.py
+++ b/src/lead/tools.py
@@ -167,15 +167,17 @@ def runs_start(
         by_id(target_id)
         if where_ran not in {"local", "databricks"}:
             raise ValueError(where_ran)
+        content = _hash(
+            target_id, model, where_ran, finished_on, metrics, feature_ids, gate_results
+        )
+        if _row(con, "run", content) is not None:
+            return content
         if where_ran == "databricks":
             if spent >= budget:
                 raise RuntimeError("databricks budget")
             if transport is None:
                 raise RuntimeError("databricks transport is required")
             transport({"target_id": target_id, "model": model})
-        content = _hash(
-            target_id, model, where_ran, finished_on, metrics, feature_ids, gate_results
-        )
         append(
             con,
             "run",
diff --git a/tests/lead/test_gates.py b/tests/lead/test_gates.py
index 261a97c..ed91caf 100644
--- a/tests/lead/test_gates.py
+++ b/tests/lead/test_gates.py
@@ -57,6 +57,15 @@ def test_each_failed_gate_is_one_reason(overrides, reason):
     assert result.reasons == (reason,)
 
 
+def test_a_nan_metric_does_not_promote():
+    calibrated = promotion_gates(_passing(calibration_error=float("nan")))
+    covered = promotion_gates(_passing(conformal_coverage=float("nan")))
+    assert calibrated.promoted is False
+    assert calibrated.reasons == ("calibration",)
+    assert covered.promoted is False
+    assert covered.reasons == ("coverage",)
+
+
 def test_a_discarded_run_is_recorded_once(tmp_path):
     con = open_ledger(str(tmp_path / "ledger.duckdb"))
     gate = GateResult(promoted=False, reasons=("baseline",))
diff --git a/tests/lead/test_manifest.py b/tests/lead/test_manifest.py
index 900482d..e3fea0a 100644
--- a/tests/lead/test_manifest.py
+++ b/tests/lead/test_manifest.py
@@ -63,7 +63,7 @@ def test_a_blank_sha256_raises_and_inserts_nothing(tmp_path):
     )
     con = open_ledger(str(tmp_path / "ledger.duckdb"))
     try:
-        with pytest.raises(ValueError):
+        with pytest.raises(ValueError, match=r"row 1:.*ahrq.*2024"):
             load_manifest(con, str(csv_path), actor="tester")
         count = con.execute(
             "SELECT count(*) FROM lead_evidence.public_release"
@@ -73,6 +73,53 @@ def test_a_blank_sha256_raises_and_inserts_nothing(tmp_path):
         con.close()
 
 
+def test_a_blank_hash_among_real_rows_is_stored_as_a_gap(tmp_path):
+    csv_path = _csv(
+        tmp_path,
+        "ahrq,Title,Pub,table,https://example.test/a,,,2024,2026-10-31,ok,"
+        "/no/such/lead-file,abc,1,2,public,benchmark\n"
+        "cms_mds,Full file,CMS,data,https://example.test/full,,,2026-04-01,"
+        "2026-10-31,over_cap:10,,,10,public,benchmark\n",
+    )
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        assert load_manifest(con, str(csv_path), actor="tester") == 2
+        assert load_manifest(con, str(csv_path), actor="tester") == 0
+        gap = con.execute(
+            """
+            SELECT sha256, layer, path
+            FROM lead_evidence.public_release
+            WHERE source_id = 'cms_mds'
+            """
+        ).fetchone()
+        assert gap == ("", "gap", "")
+    finally:
+        con.close()
+
+
+def test_the_same_bytes_at_two_paths_both_stay(tmp_path):
+    csv_path = _csv(
+        tmp_path,
+        "ahrq,Office A,Pub,table,https://example.test/a,,,2024,2026-10-31,ok,"
+        "ahrq/a.csv,abc,1,2,public,benchmark\n"
+        "ahrq,Office B,Pub,table,https://example.test/b,,,2024,2026-10-31,ok,"
+        "ahrq/b.csv,abc,1,2,public,benchmark\n",
+    )
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    try:
+        assert load_manifest(con, str(csv_path), actor="tester") == 2
+        assert load_manifest(con, str(csv_path), actor="tester") == 0
+        paths = con.execute(
+            """
+            SELECT path FROM lead_evidence.public_release
+            WHERE source_id = 'ahrq' ORDER BY path
+            """
+        ).fetchall()
+        assert paths == [("ahrq/a.csv",), ("ahrq/b.csv",)]
+    finally:
+        con.close()
+
+
 def test_release_ddl_and_delta_schema_omit_the_catalog_token():
     assert "aco" not in RELEASE_DDL
     assert "aco" not in schema_sql("delta")
diff --git a/tests/lead/test_tools.py b/tests/lead/test_tools.py
index 2dd2e0b..f69aacf 100644
--- a/tests/lead/test_tools.py
+++ b/tests/lead/test_tools.py
@@ -177,6 +177,28 @@ def test_databricks_refuses_before_calling_a_transport(tmp_path):
         con.close()
 
 
+def test_a_repeated_databricks_start_does_not_call_the_transport_again(tmp_path):
+    con = open_ledger(str(tmp_path / "ledger.duckdb"))
+    called: list[dict] = []
+    kwargs = {
+        "where_ran": "databricks",
+        "target_id": "Q.ACR",
+        "actor": "tester",
+        "model": "glm",
+        "finished_on": "2026-10-31",
+        "metrics": "mae=1",
+        "spent": 0,
+        "budget": 1,
+    }
+    try:
+        first = runs_start(con, transport=called.append, **kwargs)
+        second = runs_start(con, transport=called.append, **kwargs)
+        assert first == second
+        assert len(called) == 1
+    finally:
+        con.close()
+
+
 def test_an_injected_databricks_transport_records_the_run(tmp_path):
     con = open_ledger(str(tmp_path / "ledger.duckdb"))
     called: list[dict] = []
-- 
2.49.1


From bb747f272d065398284475a5f0224495371b72d2 Mon Sep 17 00:00:00 2001
From: kert 
Date: Mon, 2 Nov 2026 00:22:35 -0500
Subject: [PATCH 29/30] feat(lead): load the public mirror and serve the
 explorer

Local files land in the DuckDB mirror with the source headers kept: group labels, stacked IRS names, headerless code maps, and the HCPCS action key. The read-only explorer and the review page are wired at lead.fhirworx.io. OEWS now includes 2025.
---
 compose.yml                        |   52 +
 dev/scripts/bootstrap_sso.py       |    1 +
 infra/coredns/hosts                |    1 +
 infra/nginx/index.html             |   10 +
 infra/traefik/dynamic/services.yml |    1 +
 src/bls/oews.py                    |    2 +-
 src/cli/__init__.py                |    6 +-
 src/cli/lead.py                    |   79 ++
 src/lead/explore/__init__.py       |    8 +
 src/lead/explore/__main__.py       |   34 +
 src/lead/explore/app.py            |   46 +
 src/lead/explore/catalog.py        |  551 +++++++++
 src/lead/explore/guard.py          |   48 +
 src/lead/explore/review_page.py    |  181 +++
 src/lead/explore/views.py          |  366 ++++++
 src/lead/explore/writeup.py        |  433 +++++++
 src/lead/load/__init__.py          |   12 +
 src/lead/load/errors.py            |   15 +
 src/lead/load/frame.py             |  412 +++++++
 src/lead/load/irs.py               |   55 +
 src/lead/load/public.py            | 1801 ++++++++++++++++++++++++++++
 src/lead/load/spine_map.py         |   94 ++
 src/lead/load/strand5.py           |  579 +++++++++
 src/lead/review/__init__.py        |   21 +
 src/lead/review/codes.py           |   49 +
 src/lead/review/library.py         |  146 +++
 src/lead/review/store.py           |  154 +++
 stack.toml                         |    2 +-
 tests/bls/test_oews.py             |    2 +-
 tests/lead/test_explore.py         |  359 ++++++
 tests/lead/test_load_public.py     | 1591 ++++++++++++++++++++++++
 tests/lead/test_load_spine.py      |   54 +
 tests/lead/test_load_strand5.py    |  279 +++++
 tests/lead/test_review.py          |  188 +++
 34 files changed, 7628 insertions(+), 4 deletions(-)
 create mode 100644 src/lead/explore/__init__.py
 create mode 100644 src/lead/explore/__main__.py
 create mode 100644 src/lead/explore/app.py
 create mode 100644 src/lead/explore/catalog.py
 create mode 100644 src/lead/explore/guard.py
 create mode 100644 src/lead/explore/review_page.py
 create mode 100644 src/lead/explore/views.py
 create mode 100644 src/lead/explore/writeup.py
 create mode 100644 src/lead/load/__init__.py
 create mode 100644 src/lead/load/errors.py
 create mode 100644 src/lead/load/frame.py
 create mode 100644 src/lead/load/irs.py
 create mode 100644 src/lead/load/public.py
 create mode 100644 src/lead/load/spine_map.py
 create mode 100644 src/lead/load/strand5.py
 create mode 100644 src/lead/review/__init__.py
 create mode 100644 src/lead/review/codes.py
 create mode 100644 src/lead/review/library.py
 create mode 100644 src/lead/review/store.py
 create mode 100644 tests/lead/test_explore.py
 create mode 100644 tests/lead/test_load_public.py
 create mode 100644 tests/lead/test_load_spine.py
 create mode 100644 tests/lead/test_load_strand5.py
 create mode 100644 tests/lead/test_review.py

diff --git a/compose.yml b/compose.yml
index 6c8050a..1bc399d 100644
--- a/compose.yml
+++ b/compose.yml
@@ -710,6 +710,58 @@ services:
       - no-new-privileges:true
     restart: unless-stopped
 
+  # LEAD public-data explorer. Read-only Streamlit at lead.fhirworx.io
+  # behind git-sso. Reuses the bcda image (streamlit + editable /app/src)
+  # and does not build. The mirror is mounted read-only; it is not copied
+  # into the image.
+  lead:
+    image: ${IMAGE_PREFIX:-fhirworx}/bcda:${COMMIT_SHA:-latest}
+    pull_policy: if_not_present
+    container_name: lead
+    networks:
+      - gateway
+      - observability
+    environment:
+      - LEAD_PUBLIC_DB=/data/lead/public.duckdb
+      - LEAD_ZOTERO_DB=/data/zotero/zotero.sqlite
+      - LEAD_ZOTERO_STORAGE=/data/zotero/storage
+      - LEAD_REVIEW_DB=/data/lead/review/labels.sqlite
+    volumes:
+      - ./src:/app/src:ro
+      - ./data/lead:/data/lead:ro
+      - ./data/lead/review:/data/lead/review
+      - ./data/zotero/data:/data/zotero:ro
+    command:
+      - uv
+      - run
+      - --no-sync
+      - python
+      - -m
+      - lead.explore
+      - --server.port
+      - "8501"
+      - --server.address
+      - 0.0.0.0
+      - --server.headless
+      - "true"
+    healthcheck:
+      test:
+        - CMD
+        - curl
+        - -sf
+        - --max-time
+        - "4"
+        - http://localhost:8501/_stcore/health
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 40s
+    labels:
+      - "promtail=true"
+    security_opt:
+      - no-new-privileges:true
+    restart: unless-stopped
+
   # SSO-guarded RAG chat UI (P34). Routed at llm.fhirworx.io via Traefik's
   # git-sso middleware ($reef in infra/traefik/dynamic/services.yml). Joins
   # gateway (Traefik), storage (postgres:5432 / pgvector), and data
diff --git a/dev/scripts/bootstrap_sso.py b/dev/scripts/bootstrap_sso.py
index 9654ed6..862954a 100644
--- a/dev/scripts/bootstrap_sso.py
+++ b/dev/scripts/bootstrap_sso.py
@@ -276,6 +276,7 @@ SUBDOMAINS = [
     "auth",
     "llm",
     "bcda",  # BCDA manager (P52) — Streamlit behind git-sso
+    "lead",  # LEAD public-data explorer — Streamlit behind git-sso
 ]
 
 
diff --git a/infra/coredns/hosts b/infra/coredns/hosts
index 167aa6d..5fb5f2d 100644
--- a/infra/coredns/hosts
+++ b/infra/coredns/hosts
@@ -18,3 +18,4 @@
 192.168.1.192 s3console.fhirworx.io
 192.168.1.192 llm.fhirworx.io
 192.168.1.192 bcda.fhirworx.io
+192.168.1.192 lead.fhirworx.io
diff --git a/infra/nginx/index.html b/infra/nginx/index.html
index dd5e971..05fc17e 100644
--- a/infra/nginx/index.html
+++ b/infra/nginx/index.html
@@ -73,6 +73,16 @@
                 
    
             
 
+            
+                
+                
+                
    LEAD
    
+                
    Public mirror
    
+                
    Read-only explorer over the public-data DuckDB mirror
    
+                
+                
    
+                
+
             
                 
                 
diff --git a/infra/traefik/dynamic/services.yml b/infra/traefik/dynamic/services.yml
index 546f7a7..1f5703b 100644
--- a/infra/traefik/dynamic/services.yml
+++ b/infra/traefik/dynamic/services.yml
@@ -9,6 +9,7 @@
   "api"                (dict "port" "8000"  "theme" false "mw" "secure-headers")
   "llm"                (dict "port" "8000"  "theme" false "mw" "git-sso,secure-headers")
   "bcda"               (dict "port" "8501"  "theme" false "mw" "git-sso,secure-headers")
+  "lead"               (dict "port" "8501"  "theme" false "mw" "git-sso,secure-headers")
   "nessie"             (dict "port" "19120" "theme" false "mw" "git-sso,infra-headers")
   "trino"              (dict "port" "8080"  "theme" true  "mw" "git-sso,infra-headers")
   "polaris"            (dict "port" "8181"  "theme" false "mw" "git-sso,infra-headers")
diff --git a/src/bls/oews.py b/src/bls/oews.py
index 1c2e87a..be312be 100644
--- a/src/bls/oews.py
+++ b/src/bls/oews.py
@@ -33,7 +33,7 @@ import httpx
 from bib.item import Source
 from bib.tag import Tag
 
-YEARS = range(2013, 2025)
+YEARS = range(2013, 2026)
 TABLE = "bls.oews"
 _HEADERS = {
     "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0",
diff --git a/src/cli/__init__.py b/src/cli/__init__.py
index 746c8f7..f277179 100644
--- a/src/cli/__init__.py
+++ b/src/cli/__init__.py
@@ -61,7 +61,11 @@ app.add_typer(
     help="CMS rulemaking comment text extraction & analysis.",
 )
 app.add_typer(lake_app, name="lake", help="Lakehouse schema and data.")
-app.add_typer(lead_app, name="lead", help="LEAD target registry and evidence ledger.")
+app.add_typer(
+    lead_app,
+    name="lead",
+    help="LEAD target registry, evidence ledger, and public-data mirror.",
+)
 app.add_typer(llm_app, name="llm", help="Local RAG — index, search, tag.")
 app.add_typer(db_app, name="db", help="DuckDB utilities.")
 app.add_typer(docs_app, name="docs", help="Documentation generation.")
diff --git a/src/cli/lead.py b/src/cli/lead.py
index af26045..ceff1ab 100644
--- a/src/cli/lead.py
+++ b/src/cli/lead.py
@@ -2,10 +2,14 @@
 
 from __future__ import annotations
 
+from collections import Counter
+
 import typer
 
 from lead.ledger.store import open_ledger
 from lead.ledger.store import status as ledger_status
+from lead.load.public import load_manifest_files
+from lead.load.strand5 import load_strand5
 from lead.manifest import load_manifest
 from lead.targets import TARGETS
 
@@ -85,3 +89,78 @@ def manifest_load(
     finally:
         con.close()
     typer.echo(f"inserted\t{inserted}")
+
+
+@app.command("explore")
+def explore(
+    db: str = typer.Option(
+        "/home/kert/stack/data/lead/public.duckdb",
+        "--db",
+        help="Path to the local public-data mirror.",
+    ),
+    port: int = typer.Option(8502, help="Port for the Streamlit server."),
+    headless: bool = typer.Option(True, help="Do not open a browser."),
+) -> None:
+    """Open the read-only explorer on the local public-data mirror."""
+    import os
+    import subprocess
+    import sys
+
+    env = os.environ.copy()
+    env["LEAD_PUBLIC_DB"] = db
+    cmd = [
+        sys.executable,
+        "-m",
+        "lead.explore",
+        "--server.port",
+        str(port),
+        "--server.headless",
+        "true" if headless else "false",
+    ]
+    typer.echo(f"LEAD public mirror on http://localhost:{port}")
+    raise typer.Exit(subprocess.call(cmd, env=env))
+
+
+@app.command("load")
+def load_data(
+    db: str = typer.Option(..., "--db", help="Path to the local DuckDB mirror."),
+    manifest: str = typer.Option("", "--manifest", help="Enrichment manifest CSV."),
+    root: str = typer.Option(
+        "", "--root", help="Directory the manifest paths are under."
+    ),
+    strand5: str = typer.Option(
+        "", "--strand5", help="Directory that holds files.tsv."
+    ),
+    actor: str = typer.Option("stack", "--actor", help="Who loaded the rows."),
+) -> None:
+    """Load local public files into DuckDB. This does not download or publish."""
+    import duckdb
+
+    if not manifest and not strand5:
+        raise typer.BadParameter("pass --manifest or --strand5")
+    if manifest and not root:
+        raise typer.BadParameter("--manifest requires --root")
+    con = duckdb.connect(db)
+    try:
+        con.execute("SET preserve_insertion_order = false")
+        con.execute("SET memory_limit = '16GB'")
+        con.execute("SET threads = 4")
+        results = []
+        if manifest:
+            results.extend(load_manifest_files(con, manifest, root=root, actor=actor))
+        if strand5:
+            results.extend(load_strand5(con, strand5, actor=actor))
+    finally:
+        con.close()
+    counts = Counter(row.status for row in results)
+    for status in (
+        "loaded",
+        "skipped",
+        "gap",
+        "mismatch",
+        "missing",
+        "absent",
+        "unreadable",
+    ):
+        if counts[status]:
+            typer.echo(f"{status}\t{counts[status]}")
diff --git a/src/lead/explore/__init__.py b/src/lead/explore/__init__.py
new file mode 100644
index 0000000..d1d9f72
--- /dev/null
+++ b/src/lead/explore/__init__.py
@@ -0,0 +1,8 @@
+"""Read-only explorer for the local public-data mirror.
+
+``stack lead explore`` serves this package. It does not download, does
+not publish, and does not open the evidence ledger. The Labels page
+writes gold labels to their own sqlite file and does not tag Zotero.
+"""
+
+from __future__ import annotations
diff --git a/src/lead/explore/__main__.py b/src/lead/explore/__main__.py
new file mode 100644
index 0000000..f47c43b
--- /dev/null
+++ b/src/lead/explore/__main__.py
@@ -0,0 +1,34 @@
+"""``python -m lead.explore`` — the public-data explorer.
+
+The fhirworx theme file ships with ``bcda.ui`` and is passed as
+``--theme.base``, so this app and the BCDA manager share one look.
+Extra arguments are forwarded to ``streamlit run``.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+
+def main(argv: list[str] | None = None) -> int:
+    from streamlit.web import cli as stcli
+
+    from bcda.ui import brand
+    from lead.explore import app
+
+    args = list(sys.argv[1:] if argv is None else argv)
+    if not any(
+        item == "--theme.base" or item.startswith("--theme.base=") for item in args
+    ):
+        args = ["--theme.base", str(brand.THEME_PATH), *args]
+    sys.argv = ["streamlit", "run", str(Path(app.__file__)), *args]
+    try:
+        stcli.main()
+    except SystemExit as exc:
+        return int(exc.code or 0)
+    return 0
+
+
+if __name__ == "__main__":  # pragma: no cover — module entrypoint
+    sys.exit(main())
diff --git a/src/lead/explore/app.py b/src/lead/explore/app.py
new file mode 100644
index 0000000..a4052ed
--- /dev/null
+++ b/src/lead/explore/app.py
@@ -0,0 +1,46 @@
+"""LEAD public-data explorer.
+
+stack lead explore --db /home/kert/stack/data/lead/public.duckdb
+"""
+
+from __future__ import annotations
+
+import streamlit as st
+
+from bcda.ui import brand
+from lead.explore import views
+from lead.explore.review_page import review
+
+PAGES = [
+    st.Page(views.writeup, title="Write-up", icon=":material/article:", default=True),
+    st.Page(review, title="Labels", icon=":material/rate_review:"),
+    st.Page(views.coverage, title="Coverage", icon=":material/inventory:"),
+    st.Page(views.browse, title="Tables", icon=":material/table:"),
+    st.Page(views.place, title="Place", icon=":material/location_on:"),
+    st.Page(views.series, title="Series", icon=":material/show_chart:"),
+    st.Page(views.reader, title="SQL", icon=":material/code:"),
+]
+
+
+def main() -> None:
+    st.set_page_config(
+        page_title="LEAD public mirror",
+        page_icon=brand.icon(),
+        layout="wide",
+    )
+    brand.inject()
+    page = st.navigation(PAGES, position="hidden")
+    with st.sidebar:
+        brand.masthead("LEAD", "public mirror")
+        for item in PAGES:
+            st.page_link(item, label=item.title or "", icon=item.icon)
+        st.link_button("OSF registration", "https://osf.io/8wq3m/")
+        st.caption(views.db_path())
+        if st.button("Reconnect"):
+            views.connection.clear()
+            st.rerun()
+    page.run()
+
+
+if __name__ == "__main__":  # pragma: no cover — streamlit run executes this file
+    main()
diff --git a/src/lead/explore/catalog.py b/src/lead/explore/catalog.py
new file mode 100644
index 0000000..162a3d5
--- /dev/null
+++ b/src/lead/explore/catalog.py
@@ -0,0 +1,551 @@
+"""Questions the explorer asks the public-data mirror.
+
+Every function takes an open DuckDB connection. None of them download,
+publish, or write. A table whose load log recorded at least
+:data:`HEAVY_ROWS` is not aggregated unless the caller passes a year or
+a series, because a geography predicate alone still reads the file.
+"""
+
+from __future__ import annotations
+
+import re
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+
+import duckdb
+
+from lead.explore.guard import prepare
+
+DEFAULT_DB = "/home/kert/stack/data/lead/public.duckdb"
+HEAVY_ROWS = 2_000_000
+SAMPLE_CAP = 500
+SQL_CAP = 2_000
+
+_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+_HIDDEN = frozenset({"information_schema", "pg_catalog"})
+PROVENANCE = (
+    "release",
+    "content_sha256",
+    "source_path",
+    "member",
+    "geo_fips",
+    "geo_zcta",
+    "geo_zip",
+    "geo_hsa",
+    "geo_hrr",
+    "geo_cbsa",
+    "spine_year",
+    "spine_quarter",
+    "spine_month",
+    "time_key",
+    "file_year",
+)
+YEAR_COLUMNS = ("spine_year", "file_year", "year")
+GEO_COLUMNS = (
+    "geo_fips",
+    "geo_zcta",
+    "geo_zip",
+    "geo_hsa",
+    "geo_hrr",
+    "geo_cbsa",
+)
+_YEAR_ROLES = frozenset(YEAR_COLUMNS) | {"spine_quarter", "spine_month"}
+_NUMERIC = frozenset(
+    {
+        "TINYINT",
+        "SMALLINT",
+        "INTEGER",
+        "BIGINT",
+        "HUGEINT",
+        "UTINYINT",
+        "USMALLINT",
+        "UINTEGER",
+        "UBIGINT",
+        "UHUGEINT",
+        "FLOAT",
+        "DOUBLE",
+        "REAL",
+        "DECIMAL",
+    }
+)
+
+
+class NeedsFilter(ValueError):
+    """A large table needs a year or a series before it is aggregated."""
+
+    def __init__(self, table: str) -> None:
+        self.table = table
+        super().__init__(f"{table} needs a year or a series before it is read")
+
+
+class QueryTimeout(TimeoutError):
+    """DuckDB interrupted the scan."""
+
+    def __init__(self, seconds: float) -> None:
+        self.seconds = seconds
+        super().__init__(f"stopped after {seconds:g}s")
+
+
+@dataclass(frozen=True)
+class FileRow:
+    source_id: str
+    release: str
+    path: str
+    table_name: str
+    status: str
+    row_count: int
+    reason: str
+
+
+@dataclass(frozen=True)
+class TableCard:
+    schema: str
+    name: str
+    columns: tuple[tuple[str, str], ...]
+    loaded_files: int
+    loaded_rows: int
+    releases: tuple[str, ...]
+
+    @property
+    def qualified(self) -> str:
+        return f"{self.schema}.{self.name}"
+
+    @property
+    def heavy(self) -> bool:
+        return self.loaded_rows >= HEAVY_ROWS
+
+    def column_names(self) -> set[str]:
+        return {name for name, _type in self.columns}
+
+    def year_column(self) -> str:
+        have = self.column_names()
+        for name in YEAR_COLUMNS:
+            if name in have:
+                return name
+        return ""
+
+    def measure_columns(self) -> tuple[str, ...]:
+        """Numeric columns that are not year or quarter markers."""
+        return tuple(
+            name
+            for name, kind in self.columns
+            if name not in _YEAR_ROLES and _base_type(kind) in _NUMERIC
+        )
+
+
+@dataclass(frozen=True)
+class SeriesSpan:
+    points: int
+    first_year: int | None
+    last_year: int | None
+    first_value: float | None
+    last_value: float | None
+    change: float | None
+
+
+def quote(name: str) -> str:
+    if _IDENT.fullmatch(name or "") is None:
+        raise ValueError(name)
+    return '"' + name + '"'
+
+
+def qualify(schema: str, name: str) -> str:
+    return f"{quote(schema)}.{quote(name)}"
+
+
+def open_mirror(path: str | Path):
+    """Open *path* read-only. A missing file raises ``FileNotFoundError``."""
+    src = Path(path)
+    if not src.is_file():
+        raise FileNotFoundError(src)
+    con = duckdb.connect(str(src), read_only=True)
+    con.execute("SET preserve_insertion_order = false")
+    con.execute("SET memory_limit = '8GB'")
+    con.execute("SET threads = 4")
+    return con
+
+
+def bounded_execute(con, sql: str, params: list | None = None, *, seconds: float = 45):
+    """Run *sql* and interrupt it if it is still going after *seconds*."""
+    timer = threading.Timer(seconds, con.interrupt)
+    timer.start()
+    try:
+        return con.execute(sql, params or [])
+    except duckdb.InterruptException as exc:
+        raise QueryTimeout(seconds) from exc
+    finally:
+        timer.cancel()
+
+
+def latest_files(con) -> tuple[FileRow, ...]:
+    """Newest load-log row for each source and path.
+
+    A rerun that finds the file already loaded does not append a row, so
+    the newest row still carries the count from the read that inserted.
+    """
+    found = con.execute(
+        """
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = 'lead_public' AND table_name = 'load_log'
+        """
+    ).fetchone()
+    if found is None:
+        return ()
+    rows = con.execute(
+        """
+        SELECT source_id, release, path, table_name, status, row_count, reason
+        FROM (
+            SELECT source_id, release, path, table_name, status, row_count, reason,
+                   row_number() OVER (
+                       PARTITION BY source_id, path
+                       ORDER BY rowid DESC
+                   ) AS _n
+            FROM lead_public.load_log
+        ) t
+        WHERE _n = 1
+        ORDER BY source_id, release, path
+        """
+    ).fetchall()
+    return tuple(
+        FileRow(
+            source_id=str(row[0] or ""),
+            release=str(row[1] or ""),
+            path=str(row[2] or ""),
+            table_name=str(row[3] or ""),
+            status=str(row[4] or ""),
+            row_count=int(row[5] or 0),
+            reason=str(row[6] or ""),
+        )
+        for row in rows
+    )
+
+
+def status_counts(files: tuple[FileRow, ...]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for row in files:
+        counts[row.status] = counts.get(row.status, 0) + 1
+    return counts
+
+
+def tables(con, files: tuple[FileRow, ...] | None = None) -> tuple[TableCard, ...]:
+    """Every base table outside the system schemas, with log totals attached."""
+    logged = latest_files(con) if files is None else files
+    totals: dict[str, list] = {}
+    for row in logged:
+        if row.status != "loaded" or not row.table_name:
+            continue
+        slot = totals.setdefault(row.table_name, [0, 0, set()])
+        slot[0] += 1
+        slot[1] += row.row_count
+        if row.release:
+            slot[2].add(row.release)
+    meta = con.execute(
+        """
+        SELECT table_schema, table_name, column_name, data_type
+        FROM information_schema.columns
+        WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
+        ORDER BY table_schema, table_name, ordinal_position
+        """
+    ).fetchall()
+    grouped: dict[tuple[str, str], list[tuple[str, str]]] = {}
+    for schema, name, column, kind in meta:
+        if schema in _HIDDEN:
+            continue
+        grouped.setdefault((str(schema), str(name)), []).append(
+            (str(column), str(kind))
+        )
+    cards = []
+    for schema, name in sorted(grouped):
+        qualified_name = f"{schema}.{name}"
+        files_n, rows_n, releases = totals.get(qualified_name, [0, 0, set()])
+        cards.append(
+            TableCard(
+                schema=schema,
+                name=name,
+                columns=tuple(grouped[(schema, name)]),
+                loaded_files=files_n,
+                loaded_rows=rows_n,
+                releases=tuple(sorted(releases)),
+            )
+        )
+    return tuple(cards)
+
+
+def card_for(
+    cards: tuple[TableCard, ...] | list[TableCard], qualified_name: str
+) -> TableCard:
+    for card in cards:
+        if card.qualified == qualified_name:
+            return card
+    raise KeyError(qualified_name)
+
+
+def _where(
+    card: TableCard,
+    *,
+    geo: tuple[str, str] | None,
+    year: int | None,
+    series: tuple[str, str] | None,
+) -> tuple[str, list]:
+    names = card.column_names()
+    clauses: list[str] = []
+    params: list = []
+    if geo is not None:
+        column, value = geo
+        if column not in GEO_COLUMNS or column not in names:
+            raise ValueError(column)
+        clauses.append(f"{quote(column)} = ?")
+        params.append(value)
+    if year is not None:
+        column = card.year_column()
+        if not column:
+            raise ValueError("no year column")
+        clauses.append(f"{quote(column)} = ?")
+        params.append(int(year))
+    if series is not None:
+        column, value = series
+        if column not in names:
+            raise ValueError(column)
+        clauses.append(f"{quote(column)} = ?")
+        params.append(value)
+    text = (" WHERE " + " AND ".join(clauses)) if clauses else ""
+    return text, params
+
+
+def sample(
+    con,
+    card: TableCard,
+    *,
+    geo: tuple[str, str] | None = None,
+    year: int | None = None,
+    series: tuple[str, str] | None = None,
+    limit: int = 100,
+    seconds: float = 20,
+) -> list[dict]:
+    """Return up to *limit* rows. No ``ORDER BY``: a sort would read the file.
+
+    An unfiltered sample stops after *limit* rows. A filter on a heavy
+    table still has to find those rows, so it needs a year or a series.
+    """
+    if card.heavy and (geo is not None or year is not None or series is not None):
+        if year is None and series is None:
+            raise NeedsFilter(card.qualified)
+    where, params = _where(card, geo=geo, year=year, series=series)
+    bound = int(limit)
+    if bound < 1 or bound > SAMPLE_CAP:
+        raise ValueError("limit")
+    sql = f"SELECT * FROM {qualify(card.schema, card.name)}{where} LIMIT {bound}"
+    return _dicts(bounded_execute(con, sql, params, seconds=seconds))
+
+
+def count_rows(
+    con,
+    card: TableCard,
+    *,
+    geo: tuple[str, str] | None = None,
+    year: int | None = None,
+    seconds: float = 45,
+) -> int:
+    if card.heavy and year is None:
+        raise NeedsFilter(card.qualified)
+    where, params = _where(card, geo=geo, year=year, series=None)
+    sql = f"SELECT count(*) FROM {qualify(card.schema, card.name)}{where}"
+    return int(bounded_execute(con, sql, params, seconds=seconds).fetchone()[0])
+
+
+def series_ids(
+    con, card: TableCard, column: str = "series_id", *, limit: int = 500
+) -> tuple[str, ...]:
+    if card.heavy:
+        raise NeedsFilter(card.qualified)
+    if column not in card.column_names():
+        raise ValueError(column)
+    bound = min(int(limit), SAMPLE_CAP)
+    sql = (
+        f"SELECT DISTINCT {quote(column)} FROM {qualify(card.schema, card.name)} "
+        f"WHERE {quote(column)} IS NOT NULL ORDER BY 1 LIMIT {bound}"
+    )
+    return tuple(
+        str(row[0]) for row in bounded_execute(con, sql, seconds=20).fetchall()
+    )
+
+
+def series_points(
+    con,
+    card: TableCard,
+    *,
+    series_column: str,
+    series_id: str,
+    value_column: str,
+    seconds: float = 30,
+) -> list[dict]:
+    names = card.column_names()
+    if series_column not in names or value_column not in names:
+        raise ValueError("column")
+    if not series_id:
+        raise NeedsFilter(card.qualified)
+    year = card.year_column()
+    month = (
+        "spine_month" if "spine_month" in names else "month" if "month" in names else ""
+    )
+    picked = [series_column, value_column]
+    for extra in (year, month, "period", "time_key"):
+        if extra and extra not in picked and extra in names:
+            picked.append(extra)
+    cols = ", ".join(quote(name) for name in picked)
+    order = f" ORDER BY {quote(year)}" if year else ""
+    if month and year:
+        order += f", {quote(month)}"
+    sql = (
+        f"SELECT {cols} FROM {qualify(card.schema, card.name)} "
+        f"WHERE {quote(series_column)} = ?{order}"
+    )
+    return _dicts(bounded_execute(con, sql, [series_id], seconds=seconds))
+
+
+def span(points: list[dict], *, year_key: str, value_key: str) -> SeriesSpan:
+    """Last numeric value minus the first, in year order. Not a model."""
+    usable: list[tuple[int, float]] = []
+    for row in points:
+        raw_year = row.get(year_key)
+        raw_value = row.get(value_key)
+        if raw_year is None or raw_value is None:
+            continue
+        try:
+            usable.append((int(raw_year), float(raw_value)))
+        except (TypeError, ValueError):
+            continue
+    if not usable:
+        return SeriesSpan(0, None, None, None, None, None)
+    usable.sort()
+    change = usable[-1][1] - usable[0][1] if len(usable) > 1 else None
+    return SeriesSpan(
+        len(usable),
+        usable[0][0],
+        usable[-1][0],
+        usable[0][1],
+        usable[-1][1],
+        change,
+    )
+
+
+def profile(
+    con, card: TableCard, *, year: int | None = None, seconds: float = 45
+) -> dict:
+    """One scan: row count, year range, and non-null counts for spine columns."""
+    if card.heavy and year is None:
+        raise NeedsFilter(card.qualified)
+    if (
+        card.loaded_rows == 0
+        and card.qualified != "lead_public.load_log"
+        and year is None
+    ):
+        raise NeedsFilter(card.qualified)
+    names = card.column_names()
+    spine = [name for name in PROVENANCE if name in names]
+    pieces = ["count(*)"]
+    for name in spine:
+        pieces.append(f"count({quote(name)})")
+    year_name = card.year_column()
+    if year_name:
+        pieces.append(f"min({quote(year_name)})")
+        pieces.append(f"max({quote(year_name)})")
+    where, params = ("", [])
+    if year is not None:
+        if not year_name:
+            raise ValueError("no year column")
+        where, params = f" WHERE {quote(year_name)} = ?", [int(year)]
+    sql = f"SELECT {', '.join(pieces)} FROM {qualify(card.schema, card.name)}{where}"
+    row = bounded_execute(con, sql, params, seconds=seconds).fetchone()
+    out: dict = {"rows": int(row[0])}
+    cursor = 1
+    for name in spine:
+        out[name] = int(row[cursor])
+        cursor += 1
+    if year_name:
+        out["year_min"] = row[cursor]
+        out["year_max"] = row[cursor + 1]
+        out["year_column"] = year_name
+    return out
+
+
+def place_summary(
+    con,
+    cards: tuple[TableCard, ...] | list[TableCard],
+    *,
+    geo_column: str,
+    geo_value: str,
+    year: int | None = None,
+    seconds: float = 45,
+) -> list[dict]:
+    """Count *geo_value* in every table that carries *geo_column*.
+
+    A heavy table is not scanned until *year* is set. A timeout on one
+    table is recorded on that row and the rest still run.
+    """
+    if geo_column not in GEO_COLUMNS:
+        raise ValueError(geo_column)
+    rows: list[dict] = []
+    for card in cards:
+        if geo_column not in card.column_names():
+            continue
+        entry = {
+            "table": card.qualified,
+            "rows": None,
+            "year_column": card.year_column(),
+            "note": "",
+        }
+        use_year = year if card.year_column() else None
+        if year is not None and not card.year_column():
+            entry["note"] = "no year column"
+        if card.heavy and use_year is None:
+            entry["note"] = "needs a year"
+            rows.append(entry)
+            continue
+        try:
+            entry["rows"] = count_rows(
+                con,
+                card,
+                geo=(geo_column, geo_value),
+                year=use_year,
+                seconds=seconds,
+            )
+        except QueryTimeout:
+            entry["note"] = "timed out"
+        rows.append(entry)
+    return rows
+
+
+def run_select(
+    con,
+    sql: str,
+    cards: tuple[TableCard, ...] | list[TableCard],
+    *,
+    cap: int = SQL_CAP,
+    seconds: float = 45,
+) -> tuple[list[str], list[tuple]]:
+    """Run one guarded read. A heavy table without ``WHERE`` is refused."""
+    statement = prepare(sql, cap=cap)
+    if statement.lower().startswith("select"):
+        folded = sql.replace('"', "")
+        for card in cards:
+            if not card.heavy:
+                continue
+            pattern = (
+                rf"(?i)(? list[dict]:
+    headers = [item[0] for item in cursor.description]
+    return [dict(zip(headers, row, strict=True)) for row in cursor.fetchall()]
+
+
+def _base_type(kind: str) -> str:
+    return kind.split("(", 1)[0].upper()
diff --git a/src/lead/explore/guard.py b/src/lead/explore/guard.py
new file mode 100644
index 0000000..87eb107
--- /dev/null
+++ b/src/lead/explore/guard.py
@@ -0,0 +1,48 @@
+"""One read-only statement, capped, for the SQL page.
+
+The connection is opened read-only as well. This guard is what stops a
+typed statement from scanning the large tables without a bound.
+"""
+
+from __future__ import annotations
+
+import re
+
+_START = re.compile(r"(?is)^\s*(select|with|describe|show|explain)\b")
+_EXPLAIN_ANALYZE = re.compile(r"(?is)\bexplain\s+analyze\b")
+_WRITE = re.compile(
+    r"(?is)\b("
+    r"insert|update|delete|drop|alter|create|copy|attach|detach|"
+    r"install|export|import|pragma|call|checkpoint|vacuum|truncate"
+    r")\b"
+)
+_BARE = re.compile(r"(?is)^\s*(describe|show|explain)\b")
+
+
+class SqlRejected(ValueError):
+    """The statement is not a single capped read."""
+
+
+def prepare(sql: str, *, cap: int = 2000) -> str:
+    """Return *sql* wrapped so it cannot return more than *cap* rows.
+
+    ``DESCRIBE``, ``SHOW``, and ``EXPLAIN`` are returned unchanged.
+    ``EXPLAIN ANALYZE`` is rejected because it runs the query.
+    """
+    text = (sql or "").strip().rstrip(";").strip()
+    if not text:
+        raise SqlRejected("empty statement")
+    if ";" in text:
+        raise SqlRejected("one statement")
+    if _EXPLAIN_ANALYZE.search(text):
+        raise SqlRejected("explain analyze runs the query")
+    if _WRITE.search(text):
+        raise SqlRejected("only a read is allowed")
+    if _START.match(text) is None:
+        raise SqlRejected("start with select, with, describe, show, or explain")
+    bound = int(cap)
+    if bound < 1:
+        raise SqlRejected("cap")
+    if _BARE.match(text):
+        return text
+    return f"SELECT * FROM ({text}) AS _lead_q LIMIT {bound}"
diff --git a/src/lead/explore/review_page.py b/src/lead/explore/review_page.py
new file mode 100644
index 0000000..c81978a
--- /dev/null
+++ b/src/lead/explore/review_page.py
@@ -0,0 +1,181 @@
+"""The human labeling page. Reads Zotero. Writes only the label file."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import streamlit as st
+
+from lead.review.codes import (
+    DECISIONS,
+    GATE,
+    PROJECTS,
+    STAGES,
+    criteria,
+    reason_codes,
+    reasons,
+)
+from lead.review.library import (
+    connect,
+    load_item,
+    next_unlabeled,
+    pass_two_keys,
+    project_keys,
+)
+from lead.review.store import Store
+from prisma.audit import disagreements
+from prisma.export import _extract_fulltext
+
+_ZOTERO = os.environ.get(
+    "LEAD_ZOTERO_DB",
+    "/home/kert/stack/data/zotero/data/zotero.sqlite",
+)
+_STORAGE = os.environ.get(
+    "LEAD_ZOTERO_STORAGE",
+    "/home/kert/stack/data/zotero/data/storage",
+)
+_LABELS = os.environ.get(
+    "LEAD_REVIEW_DB",
+    "/home/kert/stack/data/lead/review/labels.sqlite",
+)
+
+
+def review() -> None:
+    st.header("Labels")
+    st.caption(
+        "Gold labels for the calibration gate. Include and uncertain are "
+        "the positive class. A saved label is not a screen: tag and not a "
+        "stage: tag. Nothing here starts a production screen, and no "
+        "sensitivity is computed."
+    )
+    project = st.selectbox("Strand", PROJECTS)
+    stage = st.selectbox(
+        "Stage",
+        STAGES,
+        format_func=lambda name: (
+            "Title and abstract" if name == "title_abstract" else "Full text"
+        ),
+    )
+    pass_n = st.radio("Pass", (1, 2), horizontal=True)
+    try:
+        store = _store(_LABELS)
+        lib = _library(_ZOTERO)
+    except Exception as exc:
+        st.error(str(exc))
+        return
+
+    labeled = store.latest(project, stage, pass_n)
+    first = store.latest(project, stage, 1)
+    target = GATE[stage]
+    st.write(
+        f"Pass 1: {len(first)} of {target}. "
+        f"Pass 2: {store.count(project, stage, 2)} of {target}."
+    )
+    st.caption(
+        "The queue is Zotero key order. A stratified calibration sample "
+        "has not been drawn, so the first items are not that sample."
+    )
+    with st.expander("Where the two passes disagree"):
+        _agreement(first, store.latest(project, stage, 2))
+
+    keys = project_keys(lib, project)
+    if pass_n == 1:
+        pending = next_unlabeled(keys, set(labeled))
+        left = sum(1 for key in keys if key not in labeled)
+    else:
+        queue = pass_two_keys(keys, set(first), set(labeled))
+        pending = queue[0] if queue else None
+        left = len(queue)
+    st.write(f"{left} unlabeled on this pass.")
+    if pending is None:
+        if pass_n == 2 and not first:
+            st.info("Pass 2 waits until pass 1 has a label.")
+        else:
+            st.success("Nothing unlabeled on this pass.")
+        return
+
+    item = load_item(lib, pending, Path(_STORAGE))
+    st.subheader(item.title or "(no title)")
+    byline = ", ".join(item.authors)
+    bits = [part for part in (byline, item.year, item.doi) if part]
+    if bits:
+        st.write(" · ".join(bits))
+    if stage == "full_text":
+        text = _extract_fulltext(list(item.attachment_paths), max_chars=20_000)
+        if text:
+            st.text_area("Full text", text, height=360, disabled=True)
+        elif item.attachment_paths:
+            st.warning("A file is attached and no text came out of it.")
+        else:
+            st.warning(
+                "No full text is attached. "
+                "unavailable-fulltext is the exclude reason for that."
+            )
+    st.text_area(
+        "Abstract",
+        item.abstract.strip() or "(no abstract)",
+        height=240,
+        disabled=True,
+    )
+    with st.expander("Criteria"):
+        st.markdown(criteria(project))
+    with st.expander("Exclusion reasons"):
+        st.markdown(reasons(project))
+
+    codes = reason_codes(project)
+    with st.form("gold"):
+        decision = st.radio("Decision", DECISIONS, horizontal=True)
+        reason = st.selectbox("Reason, exclude only", ("", *codes))
+        rationale = st.text_area("Why, in a sentence that names the criterion")
+        submitted = st.form_submit_button("Save label")
+    if submitted:
+        try:
+            store.record(
+                zotero_key=item.key,
+                project=project,
+                stage=stage,
+                pass_n=pass_n,
+                decision=decision,
+                reason=reason,
+                rationale=rationale,
+                actor=_actor(),
+            )
+        except ValueError as exc:
+            st.error(str(exc))
+        else:
+            st.rerun()
+
+
+def _agreement(first: dict[str, str], second: dict[str, str]) -> None:
+    shared = set(first) & set(second)
+    if not shared:
+        st.write("No item has both passes yet.")
+        return
+    a = {key: first[key] for key in shared}
+    b = {key: second[key] for key in shared}
+    n = len(disagreements(a, b))
+    st.write(f"{n} of {len(shared)} items labeled in both passes disagree.")
+
+
+def _actor() -> str:
+    headers = getattr(getattr(st, "context", None), "headers", None) or {}
+    for name in ("X-Auth-Request-Email", "x-auth-request-email"):
+        value = headers.get(name)
+        if value:
+            return str(value)
+    for name in ("X-Auth-Request-User", "x-auth-request-user"):
+        value = headers.get(name)
+        if value:
+            return str(value)
+    return "reviewer"
+
+
+@st.cache_resource(show_spinner=False)
+def _library(path: str):
+    return connect(path)
+
+
+@st.cache_resource(show_spinner=False)
+def _store(path: str) -> Store:
+    return Store(path)
diff --git a/src/lead/explore/views.py b/src/lead/explore/views.py
new file mode 100644
index 0000000..ff17d8b
--- /dev/null
+++ b/src/lead/explore/views.py
@@ -0,0 +1,366 @@
+"""Streamlit pages. Queries live in :mod:`lead.explore.catalog`."""
+
+from __future__ import annotations
+
+import os
+
+import duckdb
+import pandas as pd
+import streamlit as st
+
+from bcda.ui import brand
+from lead.explore.catalog import (
+    DEFAULT_DB,
+    GEO_COLUMNS,
+    NeedsFilter,
+    QueryTimeout,
+    card_for,
+    latest_files,
+    open_mirror,
+    place_summary,
+    profile,
+    run_select,
+    sample,
+    series_ids,
+    series_points,
+    span,
+    status_counts,
+    tables,
+)
+from lead.explore.guard import SqlRejected
+from lead.explore.writeup import build
+
+_SQL = """SELECT source_id, status, count(*) AS files, sum(row_count) AS rows_recorded
+FROM lead_public.load_log
+GROUP BY 1, 2
+ORDER BY rows_recorded DESC"""
+
+
+def db_path() -> str:
+    return os.environ.get("LEAD_PUBLIC_DB", DEFAULT_DB)
+
+
+@st.cache_resource(show_spinner=False)
+def connection(path: str):
+    return open_mirror(path)
+
+
+def _open():
+    path = db_path()
+    try:
+        return connection(path), path, None
+    except Exception as exc:
+        return None, path, exc
+
+
+def _show_error(path: str, exc: BaseException) -> None:
+    text = str(exc)
+    if "lock" in text.lower():
+        st.error(
+            f"{path} is locked by a writer. Leave this closed until "
+            "`stack lead load` exits."
+        )
+    elif isinstance(exc, FileNotFoundError):
+        st.error(f"No mirror at {path}.")
+    else:
+        st.error(text)
+
+
+def _files_frame(rows) -> pd.DataFrame:
+    return pd.DataFrame(
+        [
+            {
+                "source_id": row.source_id,
+                "release": row.release,
+                "status": row.status,
+                "row_count": row.row_count,
+                "reason": row.reason,
+                "table": row.table_name,
+                "path": row.path,
+            }
+            for row in rows
+        ]
+    )
+
+
+def writeup() -> None:
+    brand.title("Public data mirror", "lead · evidence chassis")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    for section in build(con):
+        st.subheader(section.heading)
+        for paragraph in section.body:
+            st.markdown(paragraph)
+        if section.rows:
+            frame = pd.DataFrame(section.rows, columns=list(section.headers))
+            st.dataframe(frame, hide_index=True, width="stretch")
+
+
+def coverage() -> None:
+    brand.title("Coverage", "lead · load log")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    files = latest_files(con)
+    counts = status_counts(files)
+    columns = st.columns(min(4, max(len(counts), 1)))
+    for index, (status, count) in enumerate(sorted(counts.items())):
+        columns[index % len(columns)].metric(status, f"{count:,}")
+    statuses = ["all", *sorted(counts)]
+    picked = st.selectbox("Status", statuses)
+    needle = st.text_input("Source or path contains")
+    chosen = files
+    if picked != "all":
+        chosen = tuple(row for row in chosen if row.status == picked)
+    if needle.strip():
+        text = needle.strip().lower()
+        chosen = tuple(
+            row
+            for row in chosen
+            if text in row.source_id.lower() or text in row.path.lower()
+        )
+    st.caption(f"{len(chosen):,} files")
+    if chosen:
+        st.dataframe(_files_frame(chosen), hide_index=True, width="stretch")
+
+
+def browse() -> None:
+    brand.title("Tables", "lead · one table")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    cards = tables(con)
+    names = [card.qualified for card in cards]
+    if not names:
+        st.info("This file has no tables.")
+        return
+    needle = st.text_input("Filter names")
+    shown = [name for name in names if needle.strip().lower() in name.lower()]
+    if not shown:
+        st.info("No table name matches.")
+        return
+    name = st.selectbox("Table", shown)
+    card = card_for(cards, name)
+    left, right, third = st.columns(3)
+    left.metric("Files loaded", f"{card.loaded_files:,}")
+    right.metric("Rows recorded", f"{card.loaded_rows:,}")
+    third.metric("Heavy", "yes" if card.heavy else "no")
+    st.dataframe(
+        pd.DataFrame(card.columns, columns=["column", "type"]),
+        hide_index=True,
+        width="stretch",
+    )
+    year_text = ""
+    geo_column = ""
+    geo_value = ""
+    if card.heavy:
+        st.caption(
+            "This table is heavy. A sample with no filter is allowed, "
+            "because it stops after 100 rows. A total needs a year or a series."
+        )
+    with st.form("slice"):
+        if card.year_column():
+            year_text = st.text_input("Year", placeholder=card.year_column())
+        geos = [column for column in GEO_COLUMNS if column in card.column_names()]
+        if geos:
+            geo_column = st.selectbox("Geography column", ["", *geos])
+            geo_value = st.text_input("Geography value")
+        show = st.form_submit_button("Show 100 rows")
+        profile_it = st.form_submit_button("Profile")
+    try:
+        year = _year(year_text)
+    except ValueError:
+        st.warning("Year must be a number.")
+        year = None
+        show = False
+        profile_it = False
+    geo = (geo_column, geo_value.strip()) if geo_column and geo_value.strip() else None
+    if show:
+        try:
+            rows = sample(con, card, geo=geo, year=year, limit=100)
+        except (NeedsFilter, QueryTimeout, ValueError) as exc:
+            st.warning(str(exc))
+        else:
+            st.dataframe(pd.DataFrame(rows), hide_index=True, width="stretch")
+    elif not card.heavy and not profile_it:
+        rows = sample(con, card, limit=100)
+        st.dataframe(pd.DataFrame(rows), hide_index=True, width="stretch")
+    if profile_it:
+        try:
+            got = profile(con, card, year=year)
+        except (NeedsFilter, QueryTimeout, ValueError) as exc:
+            st.warning(str(exc))
+        else:
+            st.dataframe(
+                pd.DataFrame(
+                    [{"field": key, "value": value} for key, value in got.items()]
+                ),
+                hide_index=True,
+                width="stretch",
+            )
+
+
+def place() -> None:
+    brand.title("Place", "lead · one geography")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    cards = tables(con)
+    present = [
+        column
+        for column in GEO_COLUMNS
+        if any(column in card.column_names() for card in cards)
+    ]
+    if not present:
+        st.info("No table carries a geography column.")
+        return
+    with st.form("place"):
+        column = st.selectbox(
+            "Column",
+            present,
+            index=present.index("geo_fips") if "geo_fips" in present else 0,
+        )
+        value = st.text_input("Value", placeholder="01001")
+        year_text = st.text_input("Year (required for a heavy table)")
+        go = st.form_submit_button("Look up")
+    if not go:
+        return
+    if not value.strip():
+        st.warning("Enter a value.")
+        return
+    if column == "geo_fips" and len(value.strip()) != 5:
+        st.caption(
+            "A county FIPS is five digits. A state total ending in 000 "
+            "was stored with geo_fips empty."
+        )
+    try:
+        year = _year(year_text)
+    except ValueError:
+        st.warning("Year must be a number.")
+        return
+    try:
+        rows = place_summary(
+            con, cards, geo_column=column, geo_value=value.strip(), year=year
+        )
+    except ValueError as exc:
+        st.warning(str(exc))
+        return
+    hit = [row for row in rows if row["rows"]]
+    st.caption(
+        f"{len(hit):,} tables have this value. "
+        f"{sum(1 for row in rows if row['note'] == 'needs a year'):,} heavy tables were not scanned."
+    )
+    st.dataframe(pd.DataFrame(rows), hide_index=True, width="stretch")
+
+
+def series() -> None:
+    brand.title("Series", "lead · one measure over time")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    cards = [
+        card
+        for card in tables(con)
+        if "series_id" in card.column_names() and card.measure_columns()
+    ]
+    if not cards:
+        st.info("No table has both series_id and a numeric measure.")
+        return
+    card = card_for(
+        cards,
+        st.selectbox("Table", [item.qualified for item in cards]),
+    )
+    measures = card.measure_columns()
+    value_column = st.selectbox("Measure", measures)
+    series_id = ""
+    if card.heavy:
+        series_id = st.text_input("series_id")
+        st.caption("This table is heavy, so the id is not listed from a scan.")
+    else:
+        try:
+            found = series_ids(con, card)
+        except (NeedsFilter, QueryTimeout) as exc:
+            st.warning(str(exc))
+            found = ()
+        if found:
+            series_id = st.selectbox("series_id", found)
+        else:
+            series_id = st.text_input("series_id")
+    if not st.button("Plot") or not series_id:
+        return
+    try:
+        points = series_points(
+            con,
+            card,
+            series_column="series_id",
+            series_id=series_id,
+            value_column=value_column,
+        )
+    except (NeedsFilter, QueryTimeout, ValueError) as exc:
+        st.warning(str(exc))
+        return
+    frame = pd.DataFrame(points)
+    if frame.empty:
+        st.info("No rows for that id.")
+        return
+    year_key = card.year_column()
+    got = span(points, year_key=year_key, value_key=value_column) if year_key else None
+    if got and got.points:
+        st.metric(
+            "Last minus first",
+            "—" if got.change is None else f"{got.change:,.4g}",
+            help=(
+                f"{got.first_value} in {got.first_year} to "
+                f"{got.last_value} in {got.last_year}. Not a model."
+            ),
+        )
+    if year_key and year_key in frame.columns and value_column in frame.columns:
+        chart = frame.dropna(subset=[year_key, value_column]).sort_values(year_key)
+        st.line_chart(chart, x=year_key, y=value_column)
+    st.dataframe(frame, hide_index=True, width="stretch")
+
+
+def reader() -> None:
+    brand.title("SQL", "lead · one read")
+    con, path, err = _open()
+    st.caption(path)
+    if err is not None or con is None:
+        _show_error(path, err or FileNotFoundError(path))
+        return
+    st.markdown(
+        "One statement. `SELECT` and `WITH` return at most 2,000 rows. "
+        "A heavy table needs a `WHERE`. Writes are rejected, and the "
+        "connection is read-only anyway."
+    )
+    with st.form("sql"):
+        text = st.text_area("Statement", value=_SQL, height=180)
+        go = st.form_submit_button("Run")
+    if not go:
+        return
+    cards = tables(con)
+    try:
+        headers, rows = run_select(con, text, cards)
+    except (SqlRejected, NeedsFilter, QueryTimeout, duckdb.Error) as exc:
+        st.warning(str(exc).split("\n", 1)[0][:500])
+        return
+    st.dataframe(pd.DataFrame(rows, columns=headers), hide_index=True, width="stretch")
+
+
+def _year(text: str) -> int | None:
+    raw = (text or "").strip()
+    if not raw:
+        return None
+    if not raw.isdigit():
+        raise ValueError("year")
+    return int(raw)
diff --git a/src/lead/explore/writeup.py b/src/lead/explore/writeup.py
new file mode 100644
index 0000000..550f907e
--- /dev/null
+++ b/src/lead/explore/writeup.py
@@ -0,0 +1,433 @@
+"""The narrative the explorer shows first.
+
+Sentences about the loader's contract are fixed, because they describe
+how a row was built. Every count is computed from the open connection.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from lead.explore.catalog import FileRow, TableCard, latest_files, status_counts, tables
+
+_FAMILIES: tuple[tuple[str, str, str], ...] = (
+    (
+        "cms",
+        "CMS",
+        "Public CMS files: ACO and REACH, HCC software and mappings, "
+        "enrollment, cost reports, geographic variation, dialysis, "
+        "market saturation, and the other data.cms.gov extracts.",
+    ),
+    (
+        "bls",
+        "Bureau of Labor Statistics",
+        "PPI and JOLTS as dated series, CES health-care employment, "
+        "national OEWS wages, and QCEW employment and wages. "
+        "QCEW's source id is qcew.",
+    ),
+    (
+        "irs",
+        "IRS Statistics of Income",
+        "County and ZIP income in econ.irs_soi. A00100 keeps that name. "
+        "A ZIP is geo_zip and is not a ZCTA.",
+    ),
+    (
+        "census",
+        "Census Bureau",
+        "ACS summary-file extracts and geographic reference files.",
+    ),
+    (
+        "cdc",
+        "CDC",
+        "PLACES, the Social Vulnerability Index, and nonmedical factors.",
+    ),
+    (
+        "ahrq",
+        "AHRQ HCUP",
+        "Clinical Classifications Software and CCSR mappings.",
+    ),
+    (
+        "nlm",
+        "National Library of Medicine",
+        "RxClass and the prescribable RxNorm release. "
+        "A concept string is in str_, because str is not a column name.",
+    ),
+    (
+        "hrsa",
+        "HRSA",
+        "Health center locations and shortage-area files.",
+    ),
+    (
+        "ecqi",
+        "eCQI",
+        "Electronic clinical quality measure specifications.",
+    ),
+    (
+        "ma",
+        "Medicare Advantage penetration",
+        "County penetration by month. The year and month come from the "
+        "parent directory, not from the file name.",
+    ),
+    (
+        "fred",
+        "FRED",
+        "Not fetched. An absent row means the file was never read, "
+        "and this mirror then has no econ.fred table.",
+    ),
+)
+
+_FAMILY_ORDER = {key: index for index, (key, _title, _about) in enumerate(_FAMILIES)}
+_ABOUT = {key: (title, about) for key, title, about in _FAMILIES}
+
+_SPINE: tuple[tuple[str, str], ...] = (
+    ("release", "Release label from the manifest, or the file name."),
+    ("content_sha256", "SHA-256 of the file that was read."),
+    ("source_path", "Resolved path of that file."),
+    ("member", "Zip member. A nested zip is outer::inner. Empty for a loose file."),
+    (
+        "geo_fips",
+        "Five-digit county. A state total (a code ending in 000) is left empty.",
+    ),
+    ("geo_zcta", "ZCTA. This is not a ZIP code, and nothing here crosswalks the two."),
+    ("geo_zip", "ZIP code. This is not a ZCTA."),
+    ("geo_hsa", "Hospital service area, as published. Not collapsed onto a county."),
+    ("geo_hrr", "Hospital referral region, as published."),
+    ("geo_cbsa", "CBSA, as published."),
+    (
+        "spine_year",
+        "Year taken from the file's own columns or, failing that, its name.",
+    ),
+    (
+        "spine_quarter",
+        "Quarter, when the file has one. A token such as q1-q4 is not a quarter.",
+    ),
+    (
+        "spine_month",
+        "Month. PPI and JOLTS annual M13 do not become a month. "
+        "CES keeps month 13, because that file uses it as the annual average.",
+    ),
+    ("time_key", "YYYYQq only when a year and a quarter both exist. Otherwise empty."),
+    (
+        "file_year",
+        "A single year token from the file name, when the columns did not carry one.",
+    ),
+)
+
+
+@dataclass(frozen=True)
+class Section:
+    heading: str
+    body: tuple[str, ...]
+    headers: tuple[str, ...] = ()
+    rows: tuple[tuple, ...] = ()
+
+
+def _n(count: int, word: str) -> str:
+    suffix = "" if count == 1 or word.endswith("s") else "s"
+    return f"{count:,} {word}{suffix}"
+
+
+def family_of(source_id: str) -> str:
+    sid = (source_id or "").lower()
+    if sid.startswith("cms"):
+        return "cms"
+    if sid.startswith("bls") or sid == "qcew":
+        return "bls"
+    if sid.startswith("irs"):
+        return "irs"
+    if sid.startswith("census"):
+        return "census"
+    if sid.startswith("cdc"):
+        return "cdc"
+    if sid.startswith("ahrq"):
+        return "ahrq"
+    if sid.startswith("nlm"):
+        return "nlm"
+    if sid.startswith("hrsa"):
+        return "hrsa"
+    if sid.startswith("ecqi"):
+        return "ecqi"
+    if sid.startswith("ma_") or sid == "ma":
+        return "ma"
+    if sid == "fred":
+        return "fred"
+    return "other"
+
+
+def build(con) -> tuple[Section, ...]:
+    files = latest_files(con)
+    cards = tables(con, files)
+    counts = status_counts(files)
+    loaded = [row for row in files if row.status == "loaded"]
+    loaded_rows = sum(row.row_count for row in loaded)
+    sources = len({row.source_id for row in loaded})
+    return (
+        _registration(),
+        _mirror(files, cards),
+        _recorded(counts, loaded_rows, sources),
+        _largest(cards),
+        _families(loaded),
+        _keyed(),
+        _geography(),
+        _gaps(files, cards),
+        _excluded(files, cards),
+        _exploring(cards),
+    )
+
+
+def _registration() -> Section:
+    """The frozen OSF registration. Screening does not start ahead of it."""
+    return Section(
+        "Registration",
+        (
+            "The review protocol was registered on OSF before screening. "
+            "The frozen record is "
+            "[10.17605/OSF.IO/8WQ3M](https://doi.org/10.17605/OSF.IO/8WQ3M), "
+            "project [osf.io/8wq3m](https://osf.io/8wq3m/).",
+        ),
+    )
+
+
+def _mirror(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
+    if not files and not cards:
+        state = "This file has no load log and no observation tables."
+    elif not files:
+        state = (
+            f"This file has {len(cards)} tables and no load log, "
+            "so the explorer cannot say which file they came from."
+        )
+    else:
+        whose = "its" if len(files) == 1 else "their"
+        state = (
+            f"The log has {_n(len(files), 'file')} in {whose} latest state, "
+            f"and the file holds {_n(len(cards), 'table')}."
+        )
+    return Section(
+        "The mirror",
+        (
+            "This is the local DuckDB mirror of public files for the LEAD "
+            "evidence chassis. `stack lead load` reads files that are already "
+            "on disk, from the enrichment manifest and from the strand-5 "
+            "inventory, and writes them here. The app opens that file read-only. "
+            "It does not download, it does not publish, and it does not write "
+            "Unity Catalog.",
+            state,
+            "A row count in this write-up is the number the loader recorded "
+            "for the latest successful read of each file. Opening this page "
+            "does not scan QCEW, the dialysis file, or the IRS extracts. "
+            "A second load of a file that is already present does not append "
+            "a new log row, so the count stays the one from the read that inserted.",
+        ),
+    )
+
+
+def _recorded(counts: dict[str, int], loaded_rows: int, sources: int) -> Section:
+    order = (
+        "loaded",
+        "skipped",
+        "absent",
+        "gap",
+        "mismatch",
+        "missing",
+        "unreadable",
+    )
+    rows = tuple((status, counts[status]) for status in order if counts.get(status))
+    for status in sorted(counts):
+        if status not in order:
+            rows = (*rows, (status, counts[status]))
+    loaded_n = counts.get("loaded", 0)
+    return Section(
+        "What the load recorded",
+        (
+            f"{_n(loaded_n, 'file')} loaded, {loaded_rows:,} rows recorded, "
+            f"across {_n(sources, 'source id')}.",
+            "Skipped is a file the loader refused on purpose: the manifest "
+            "kind was not data, the member was a program or a readme, or the "
+            "same bytes were already stored from a loose file. Absent is a "
+            "file the inventory marked missing before any read, including a "
+            "404 and FRED. A hash mismatch is recorded and that file is not "
+            "inserted. A file the reader could not open is not in this log: "
+            "the load command prints it as unreadable, and a later run retries it.",
+        ),
+        ("Status", "Files"),
+        rows,
+    )
+
+
+def _largest(cards: tuple[TableCard, ...]) -> Section:
+    ranked = tuple(
+        sorted(
+            (card for card in cards if card.name != "load_log"),
+            key=lambda card: card.loaded_rows,
+            reverse=True,
+        )
+    )[:15]
+    return Section(
+        "Largest tables",
+        (
+            "These are the fifteen tables with the most rows recorded. "
+            "A table at or above 2,000,000 rows is marked heavy: the other "
+            "pages will not aggregate it until you give a year or a series.",
+        ),
+        ("Table", "Files", "Rows recorded", "Heavy"),
+        tuple(
+            (
+                card.qualified,
+                card.loaded_files,
+                f"{card.loaded_rows:,}",
+                "yes" if card.heavy else "",
+            )
+            for card in ranked
+        ),
+    )
+
+
+def _families(loaded: list[FileRow]) -> Section:
+    buckets: dict[str, list] = {}
+    for row in loaded:
+        key = family_of(row.source_id)
+        slot = buckets.setdefault(key, [0, 0, set()])
+        slot[0] += 1
+        slot[1] += row.row_count
+        if row.table_name:
+            slot[2].add(row.table_name)
+    keys = sorted(buckets, key=lambda key: (_FAMILY_ORDER.get(key, 100), key))
+    rows = []
+    for key in keys:
+        title, about = _ABOUT.get(key, (key, "A source id outside the named groups."))
+        files_n, rows_n, names = buckets[key]
+        rows.append((title, files_n, f"{rows_n:,}", len(names), about))
+    return Section(
+        "Families",
+        (
+            "Each source id is grouped by who publishes the file. "
+            "The note is what that group is for. The counts are from the log.",
+        ),
+        ("Family", "Files", "Rows recorded", "Tables", "What it holds"),
+        tuple(rows),
+    )
+
+
+def _keyed() -> Section:
+    return Section(
+        "How a row is keyed",
+        (
+            "Observation tables are `lead_public.`, except the "
+            "series and the income files: `bls.ppi`, `bls.jolts`, `bls.ces`, "
+            "`bls.oews`, `bls.qcew`, `econ.irs_soi`, and "
+            "`lead_public.ma_county_penetration`. Several releases of one "
+            "source stack in the same table. The release and the source path "
+            "say which file a row came from.",
+            "Most measure columns are text, including a suppressed cell such "
+            "as '-'. `bls.ppi.value` and `bls.jolts.value` are doubles, and "
+            "on those two tables a blank or a lone dot is null. CES and OEWS "
+            "use the typed columns of their own writers (`value`, employment, "
+            "wages) and do not carry the spine columns below. OEWS is national.",
+        ),
+        ("Column", "Meaning"),
+        _SPINE,
+    )
+
+
+def _geography() -> Section:
+    return Section(
+        "Geography and time",
+        (
+            "A county is `geo_fips`. A ZIP is `geo_zip`. A ZCTA is `geo_zcta`. "
+            "The loader does not crosswalk one onto another, and a state total "
+            "whose code ends in 000 is stored with `geo_fips` empty. HSA, HRR, "
+            "and CBSA stay on their own columns.",
+            "`time_key` is `YYYYQq` only when the row has both a year and a "
+            "quarter. An annual QCEW row has a null time key; a quarterly row "
+            "has a key such as 1990Q1. PPI and JOLTS period M13 is the annual "
+            "average and does not become month 13. CES month 13 stays 13, "
+            "because that series uses it as the annual average. Two four-digit "
+            "years in one file name are not treated as a year.",
+        ),
+    )
+
+
+def _gaps(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
+    absent = [row for row in files if row.status == "absent"]
+    skipped: dict[str, int] = {}
+    for row in files:
+        if row.status == "skipped":
+            reason = row.reason or "(blank)"
+            skipped[reason] = skipped.get(reason, 0) + 1
+    fred_rows = [row for row in files if row.source_id == "fred"]
+    has_fred_table = any(card.qualified == "econ.fred" for card in cards)
+    if any(row.status == "absent" for row in fred_rows) and not has_fred_table:
+        fred = (
+            "FRED is absent. The log records it that way, and there is no "
+            "`econ.fred` table. Those series were not fetched."
+        )
+    elif has_fred_table:
+        fred = "`econ.fred` is present in this file."
+    elif fred_rows:
+        fred = (
+            "The log has a FRED row whose latest status is "
+            f"{fred_rows[-1].status}, reason {fred_rows[-1].reason or 'blank'}."
+        )
+    else:
+        fred = "The load log has no FRED row, and `econ.fred` is not in this file."
+    rows = tuple(
+        (row.source_id, _base(row.path) or row.release, row.reason or "")
+        for row in absent
+    )
+    skip_line = (
+        "Skipped files by reason: "
+        + ", ".join(f"{reason} {count}" for reason, count in sorted(skipped.items()))
+        + "."
+        if skipped
+        else "No file's latest status is skipped."
+    )
+    return Section(
+        "Gaps",
+        (
+            fred,
+            f"{len(absent):,} files have latest status absent.",
+            skip_line,
+        ),
+        ("Source", "File", "Reason"),
+        rows,
+    )
+
+
+def _excluded(files: tuple[FileRow, ...], cards: tuple[TableCard, ...]) -> Section:
+    del files, cards
+    return Section(
+        "What this file is not",
+        (
+            "This is not the evidence ledger. Claims, searches, screening "
+            "decisions, and calibration labels are a different database, and "
+            "this app does not open it. Literature claims are not rows here.",
+            "This is not Unity Catalog. The load that filled this file did "
+            "not publish there.",
+            "There is no beneficiary, no claim line, and no CCLF extract. "
+            "The files are public statistical releases. A cell is not a person.",
+        ),
+    )
+
+
+def _exploring(cards: tuple[TableCard, ...]) -> Section:
+    heavy = sum(1 for card in cards if card.heavy and card.name != "load_log")
+    return Section(
+        "Exploring it",
+        (
+            "Coverage lists every file in the log. Tables opens one table: "
+            "a short sample when the table is small, and a year or a series "
+            "before any total when it is not. Place looks up one geography "
+            f"in every table that carries it. {heavy:,} tables are heavy.",
+            "Series plots `series_id` against a numeric column and reports "
+            "last value minus first value, in year order. That difference is "
+            "not a model and it is not a LEAD benchmark.",
+            "SQL runs one statement. A read is wrapped so it returns at most "
+            "2,000 rows. A heavy table is refused unless the statement has a "
+            "WHERE. A scan that runs past 45 seconds is interrupted.",
+        ),
+    )
+
+
+def _base(path: str) -> str:
+    name = path.rsplit("/", 1)[-1]
+    return name.rsplit("\\", 1)[-1]
diff --git a/src/lead/load/__init__.py b/src/lead/load/__init__.py
new file mode 100644
index 0000000..59fd7f9
--- /dev/null
+++ b/src/lead/load/__init__.py
@@ -0,0 +1,12 @@
+"""Load local public files into a DuckDB mirror.
+
+The loaders read files the caller already has. They do not download,
+they do not open URLs, and they do not publish to a warehouse.
+"""
+
+from __future__ import annotations
+
+from lead.load.public import LoadResult, load_file, load_manifest_files
+from lead.load.strand5 import load_strand5
+
+__all__ = ["LoadResult", "load_file", "load_manifest_files", "load_strand5"]
diff --git a/src/lead/load/errors.py b/src/lead/load/errors.py
new file mode 100644
index 0000000..a3a47d2
--- /dev/null
+++ b/src/lead/load/errors.py
@@ -0,0 +1,15 @@
+"""Failures that must not leave observation rows behind."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class HashMismatch(ValueError):
+    """The file bytes are not the sha256 the caller supplied."""
+
+    def __init__(self, path: Path, expected: str, actual: str) -> None:
+        self.path = path
+        self.expected = expected
+        self.actual = actual
+        super().__init__(f"sha256 mismatch: {path}")
diff --git a/src/lead/load/frame.py b/src/lead/load/frame.py
new file mode 100644
index 0000000..f956e59
--- /dev/null
+++ b/src/lead/load/frame.py
@@ -0,0 +1,412 @@
+"""Shared DuckDB writes for the public-data mirror.
+
+The log DDL lives here, not in ``lead.ledger.schema``, so the Delta
+schema string stays free of a catalog token. Observation tables are
+created when a file is loaded. This module does not download and does
+not open a URL.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+from cms.datacms import sanitize_column_name
+from lead.load.errors import HashMismatch
+from lead.load.spine_map import SpineMap, file_quarter, file_year, spine_columns
+
+LOAD_LOG_DDL = """
+CREATE SCHEMA IF NOT EXISTS lead_public;
+CREATE TABLE IF NOT EXISTS lead_public.load_log (
+    source_id VARCHAR,
+    release VARCHAR,
+    sha256 VARCHAR,
+    path VARCHAR,
+    table_name VARCHAR,
+    status VARCHAR,
+    row_count BIGINT,
+    reason VARCHAR,
+    actor VARCHAR,
+    created_at VARCHAR
+)
+"""
+
+_IDENT = re.compile(r"[a-z_][a-z0-9_]*")
+_RESERVED = frozenset(
+    {
+        "release",
+        "content_sha256",
+        "source_path",
+        "member",
+        "geo_fips",
+        "geo_zcta",
+        "geo_zip",
+        "geo_hsa",
+        "geo_hrr",
+        "geo_cbsa",
+        "spine_year",
+        "spine_quarter",
+        "spine_month",
+        "time_key",
+        "file_year",
+    }
+)
+_INTS = frozenset({"spine_year", "spine_quarter", "spine_month", "file_year"})
+_PROVENANCE = (
+    "release",
+    "content_sha256",
+    "source_path",
+    "member",
+    "geo_fips",
+    "geo_zcta",
+    "geo_zip",
+    "geo_hsa",
+    "geo_hrr",
+    "geo_cbsa",
+    "spine_year",
+    "spine_quarter",
+    "spine_month",
+    "time_key",
+    "file_year",
+)
+
+
+def ensure_log(con) -> None:
+    for statement in LOAD_LOG_DDL.split(";"):
+        text = statement.strip()
+        if text:
+            con.execute(text)
+
+
+def qualify(table: str) -> str:
+    """``schema.name``, defaulting a bare name to ``lead_public``."""
+    parts = table.split(".")
+    if len(parts) == 1:
+        parts = ["lead_public", parts[0]]
+    if len(parts) != 2 or any(_IDENT.fullmatch(part) is None for part in parts):
+        raise ValueError(table)
+    return ".".join(parts)
+
+
+def hash_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def require_hash(path: Path, expected: str) -> str:
+    """Hash *path* and require it to equal *expected*.
+
+    Raises:
+        ValueError: *expected* is blank.
+        HashMismatch: the bytes are not that digest. Nothing was inserted.
+    """
+    wanted = (expected or "").strip().lower()
+    if not wanted:
+        raise ValueError("sha256 is required")
+    actual = hash_file(path)
+    if actual != wanted:
+        raise HashMismatch(path, wanted, actual)
+    return actual
+
+
+def already_loaded(con, *, source_id: str, path: str, sha256: str) -> bool:
+    ensure_log(con)
+    row = con.execute(
+        """
+        SELECT 1 FROM lead_public.load_log
+        WHERE source_id = ? AND path = ? AND sha256 = ? AND status = 'loaded'
+        """,
+        [source_id, path, sha256.strip().lower()],
+    ).fetchone()
+    return row is not None
+
+
+def record(
+    con,
+    *,
+    source_id: str,
+    release: str,
+    sha256: str,
+    path: str,
+    table: str,
+    status: str,
+    rows: int,
+    reason: str,
+    actor: str,
+) -> None:
+    ensure_log(con)
+    con.execute(
+        """
+        INSERT INTO lead_public.load_log (
+            source_id, release, sha256, path, table_name, status,
+            row_count, reason, actor, created_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """,
+        [
+            source_id,
+            release,
+            sha256,
+            path,
+            table,
+            status,
+            rows,
+            reason,
+            actor,
+            datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+        ],
+    )
+
+
+def latest(con, *, source_id: str, path: str) -> tuple[str, str]:
+    row = con.execute(
+        """
+        SELECT status, reason FROM lead_public.load_log
+        WHERE source_id = ? AND path = ?
+        ORDER BY rowid DESC
+        LIMIT 1
+        """,
+        [source_id, path],
+    ).fetchone()
+    if row is None:
+        return "", ""
+    return str(row[0]), str(row[1] or "")
+
+
+def quote_ident(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def unique_names(raw_names: list[str]) -> list[str]:
+    """SQL-safe, unique, non-reserved names in header order."""
+    seen: dict[str, int] = {}
+    out: list[str] = []
+    for raw in raw_names:
+        name = sanitize_column_name(raw) or "col"
+        if name in _RESERVED:
+            name = "src_" + name
+        seen[name] = seen.get(name, 0) + 1
+        if seen[name] > 1:
+            name = f"{name}_{seen[name]}"
+        out.append(name)
+    return out
+
+
+def _typed(name: str) -> str:
+    if name in _INTS:
+        return "INTEGER"
+    return "VARCHAR"
+
+
+def ensure_table(con, table: str, columns: list[str]) -> None:
+    schema, name = qualify(table).split(".")
+    con.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
+    found = con.execute(
+        """
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = ? AND table_name = ?
+        """,
+        [schema, name],
+    ).fetchone()
+    qualified = f"{schema}.{name}"
+    if found is None:
+        defs = ", ".join(f"{column} {_typed(column)}" for column in columns)
+        con.execute(f"CREATE TABLE {qualified} ({defs})")
+        return
+    have = {row[0] for row in con.execute(f"DESCRIBE {qualified}").fetchall()}
+    for column in columns:
+        if column not in have:
+            con.execute(f"ALTER TABLE {qualified} ADD COLUMN {column} {_typed(column)}")
+
+
+def _five(column: str, *, drop_state_total: bool) -> str:
+    state = f" AND right(trim({column}), 3) <> '000'" if drop_state_total else ""
+    return (
+        f"CASE WHEN regexp_full_match(trim({column}), '[0-9]{{5}}'){state} "
+        f"THEN trim({column}) END"
+    )
+
+
+def _compose(state: str, county: str) -> str:
+    return (
+        f"CASE WHEN regexp_full_match(trim({state}), '[0-9]{{1,2}}') "
+        f"AND regexp_full_match(trim({county}), '[0-9]{{1,3}}') "
+        f"AND lpad(trim({county}), 3, '0') <> '000' "
+        f"THEN lpad(trim({state}), 2, '0') || lpad(trim({county}), 3, '0') END"
+    )
+
+
+def _int_expr(column: str, low: int, high: int) -> str:
+    cast = f"TRY_CAST(TRY_CAST(trim({column}) AS DOUBLE) AS INTEGER)"
+    return f"CASE WHEN {cast} BETWEEN {low} AND {high} THEN {cast} END"
+
+
+def _fips_expr(spine: SpineMap) -> str:
+    parts: list[str] = []
+    if spine.fips:
+        parts.append(_five(spine.fips, drop_state_total=True))
+    if spine.county_fips:
+        parts.append(_five(spine.county_fips, drop_state_total=True))
+        if spine.state_fips:
+            parts.append(_compose(spine.state_fips, spine.county_fips))
+    if not parts:
+        return "NULL"
+    if len(parts) == 1:
+        return parts[0]
+    return "COALESCE(" + ", ".join(parts) + ")"
+
+
+def _text_expr(column: str) -> str:
+    return f"NULLIF(trim({column}), '')"
+
+
+def spine_projection(
+    columns: list[str],
+    *,
+    file_name: str,
+    when: tuple[int, int] | None,
+) -> tuple[str, str, str, str]:
+    """Return SQL for year, quarter-before-month, month, and file_year.
+
+    Quarter-from-month is applied by the caller, which can see the month.
+    """
+    spine = spine_columns(columns)
+    named_year = file_year(file_name)
+    named_quarter = file_quarter(file_name)
+    if spine.year:
+        year_sql = _int_expr(spine.year, 1900, 2100)
+    elif when is not None:
+        year_sql = str(int(when[0]))
+    elif named_year is not None:
+        year_sql = str(named_year)
+    else:
+        year_sql = "NULL"
+    quarter_parts: list[str] = []
+    if spine.quarter:
+        quarter_parts.append(_int_expr(spine.quarter, 1, 4))
+    if spine.period:
+        quarter_parts.append(
+            "CASE WHEN regexp_full_match(trim(period), 'Q0[1-4]') "
+            "THEN TRY_CAST(substr(trim(period), 3) AS INTEGER) END"
+        )
+    quarter_sql = (
+        "COALESCE(" + ", ".join(quarter_parts) + ")" if quarter_parts else "NULL"
+    )
+    month_parts: list[str] = []
+    if spine.month:
+        month_parts.append(_int_expr(spine.month, 1, 12))
+    if spine.period:
+        month_parts.append(
+            "CASE WHEN regexp_full_match(trim(period), 'M(0[1-9]|1[0-2])') "
+            "THEN TRY_CAST(substr(trim(period), 2) AS INTEGER) END"
+        )
+    if when is not None and not month_parts:
+        month_parts.append(str(int(when[1])))
+    month_sql = "COALESCE(" + ", ".join(month_parts) + ")" if month_parts else "NULL"
+    file_year_sql = "NULL" if named_year is None else str(named_year)
+    # Stash the filename quarter in the quarter SQL when the file has no
+    # quarter column and no month. A month still wins, in the outer query.
+    if named_quarter is not None and quarter_sql == "NULL" and month_sql == "NULL":
+        quarter_sql = str(named_quarter)
+    return year_sql, quarter_sql, month_sql, file_year_sql
+
+
+def insert_frame(
+    con,
+    table: str,
+    temp: str,
+    columns: list[str],
+    *,
+    release: str,
+    sha256: str,
+    source_path: str,
+    member: str,
+    file_name: str,
+    when: tuple[int, int] | None,
+    member_column: str = "",
+) -> int:
+    """Append *temp* into *table* with spine and provenance columns.
+
+    *member_column*, when set, is a column of *temp* and supplies each
+    row's member name. *member* is used for every row otherwise.
+    """
+    qualified = qualify(table)
+    year_sql, quarter_sql, month_sql, file_year_sql = spine_projection(
+        columns, file_name=file_name, when=when
+    )
+    spine = spine_columns(columns)
+    count = con.execute(f"SELECT COUNT(*) FROM {temp}").fetchone()[0]
+    carried = columns if not member_column else [*columns, member_column]
+    select_cols = ", ".join(carried)
+    con.execute(
+        f"""
+        CREATE OR REPLACE TEMP TABLE _lead_spine AS
+        SELECT {select_cols},
+               ({year_sql}) AS spine_year,
+               ({quarter_sql}) AS quarter_from_column,
+               ({month_sql}) AS spine_month,
+               ({file_year_sql}) AS file_year,
+               ({_fips_expr(spine)}) AS geo_fips,
+               ({_five(spine.zcta, drop_state_total=False) if spine.zcta else "NULL"}) AS geo_zcta,
+               ({_zip_expr(spine.zip) if spine.zip else "NULL"}) AS geo_zip,
+               ({_text_expr(spine.hsa) if spine.hsa else "NULL"}) AS geo_hsa,
+               ({_text_expr(spine.hrr) if spine.hrr else "NULL"}) AS geo_hrr,
+               ({_text_expr(spine.cbsa) if spine.cbsa else "NULL"}) AS geo_cbsa
+        FROM {temp}
+        """
+    )
+    out_columns = [
+        *columns,
+        *_PROVENANCE,
+    ]
+    ensure_table(con, qualified, out_columns)
+    member_sql = member_column if member_column else "?"
+    params: list[object] = [release, sha256, source_path]
+    if not member_column:
+        params.append(member)
+    con.execute(
+        f"""
+        INSERT INTO {qualified} ({", ".join(out_columns)})
+        SELECT {", ".join(columns)},
+               ?,
+               ?,
+               ?,
+               {member_sql},
+               geo_fips, geo_zcta, geo_zip, geo_hsa, geo_hrr, geo_cbsa,
+               spine_year,
+               COALESCE(
+                   quarter_from_column,
+                   CASE WHEN spine_month BETWEEN 1 AND 12
+                        THEN 1 + (spine_month - 1) // 3 END
+               ),
+               spine_month,
+               CASE
+                   WHEN spine_year IS NOT NULL
+                    AND COALESCE(
+                        quarter_from_column,
+                        CASE WHEN spine_month BETWEEN 1 AND 12
+                             THEN 1 + (spine_month - 1) // 3 END
+                    ) BETWEEN 1 AND 4
+                   THEN spine_year || 'Q' || COALESCE(
+                        quarter_from_column,
+                        CASE WHEN spine_month BETWEEN 1 AND 12
+                             THEN 1 + (spine_month - 1) // 3 END
+                   )
+               END,
+               file_year
+        FROM _lead_spine
+        """,
+        params,
+    )
+    return int(count)
+
+
+def _zip_expr(column: str) -> str:
+    return (
+        f"CASE WHEN regexp_full_match(trim({column}), '[0-9]{{5}}') THEN trim({column}) "
+        f"WHEN regexp_full_match(trim({column}), '[0-9]{{9}}') THEN left(trim({column}), 5) END"
+    )
diff --git a/src/lead/load/irs.py b/src/lead/load/irs.py
new file mode 100644
index 0000000..6cd9c4a
--- /dev/null
+++ b/src/lead/load/irs.py
@@ -0,0 +1,55 @@
+"""Which members of an IRS SOI zip are the county, ZIP, or CBSA tables.
+
+A zip that contains a national CSV also contains one workbook per state
+and, often, an ``.xls`` twin of that CSV. Loading every member would
+repeat the same returns. State workbooks are the table only when the
+zip has no national extract.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+_TABULAR = {".csv", ".txt", ".dat", ".xls", ".xlsx"}
+_DELIMITED = {".csv", ".txt", ".dat"}
+_NATIONAL = ("allagi", "allnoagi", "cyall", "zpall", "incyall")
+
+
+def _kind(name: str) -> str:
+    base = Path(name).name.casefold()
+    suffix = Path(name).suffix.casefold()
+    if not base or suffix not in _TABULAR:
+        return ""
+    if "docguide" in base or "guide" in base:
+        return ""
+    if "cbsa" in base or "incbsa" in base:
+        return "cbsa"
+    if any(token in base for token in _NATIONAL):
+        return "national"
+    # 2005-2007 name the national ZIP file ``zipcode05.csv``, not ``zpall``.
+    stem = base[: -len(suffix)] if suffix else base
+    if suffix == ".csv" and stem.startswith("zipcode") and stem[7:].isdigit():
+        return "national"
+    if suffix == ".txt":
+        return ""
+    return "state"
+
+
+def choose_irs_members(members: list[str]) -> list[str]:
+    """National extracts and CBSA tables. State workbooks if that is all."""
+    classified = [(name, _kind(name)) for name in members]
+    classified = [(name, kind) for name, kind in classified if kind]
+    has_delimited_national = any(
+        kind == "national" and Path(name).suffix.casefold() in _DELIMITED
+        for name, kind in classified
+    )
+    has_national = any(kind == "national" for _, kind in classified)
+    chosen: list[str] = []
+    for name, kind in classified:
+        suffix = Path(name).suffix.casefold()
+        if kind == "national" and has_delimited_national and suffix not in _DELIMITED:
+            continue
+        if kind == "state" and has_national:
+            continue
+        chosen.append(name)
+    return chosen
diff --git a/src/lead/load/public.py b/src/lead/load/public.py
new file mode 100644
index 0000000..f6b5a5f
--- /dev/null
+++ b/src/lead/load/public.py
@@ -0,0 +1,1801 @@
+"""Load local public files into a DuckDB mirror.
+
+The caller supplies the path and the sha256. This module does not
+download, does not open the file's URL, and does not publish.
+"""
+
+from __future__ import annotations
+
+import codecs
+import csv
+import json
+import re
+import shutil
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from lead.load.errors import HashMismatch
+from lead.load.frame import (
+    already_loaded,
+    ensure_log,
+    insert_frame,
+    qualify,
+    quote_ident,
+    record,
+    require_hash,
+    unique_names,
+)
+
+_MARKUP = {".html", ".htm", ".pdf", ".doc", ".docx"}
+_TABULAR = {".csv", ".txt", ".dat", ".rrf", ".xlsx", ".xls", ".json"}
+_DELIMITED = {".csv", ".txt", ".dat", ".rrf"}
+# RxNorm rich-release files are pipe-delimited and have no header row.
+_RRF_COLUMNS = {
+    "rxnconso": (
+        "RXCUI",
+        "LAT",
+        "TS",
+        "LUI",
+        "STT",
+        "SUI",
+        "ISPREF",
+        "RXAUI",
+        "SAUI",
+        "SCUI",
+        "SDUI",
+        "SAB",
+        "TTY",
+        "CODE",
+        "STR",
+        "SRL",
+        "SUPPRESS",
+        "CVF",
+    ),
+    "rxnrel": (
+        "RXCUI1",
+        "RXAUI1",
+        "STYPE1",
+        "REL",
+        "RXCUI2",
+        "RXAUI2",
+        "STYPE2",
+        "RELA",
+        "RUI",
+        "SRUI",
+        "SAB",
+        "SL",
+        "RG",
+        "DIR",
+        "SUPPRESS",
+        "CVF",
+    ),
+    "rxnsat": (
+        "RXCUI",
+        "LUI",
+        "SUI",
+        "RXAUI",
+        "STYPE",
+        "CODE",
+        "ATUI",
+        "SATUI",
+        "ATN",
+        "SAB",
+        "ATV",
+        "SUPPRESS",
+        "CVF",
+    ),
+}
+
+
+@dataclass(frozen=True)
+class LoadResult:
+    source_id: str
+    release: str
+    path: str
+    sha256: str
+    status: str
+    rows: int
+    table: str
+    reason: str
+
+
+class _NeedsSerial(Exception):
+    """DuckDB's parallel CSV reader refused this file. Read it on one thread."""
+
+
+def _parallel_refused(exc: BaseException) -> bool:
+    text = str(exc).lower()
+    return "parallel" in text and "csv" in text
+
+
+class _Results(list):
+    """Result list that reports each file as it finishes."""
+
+    def append(self, result: LoadResult) -> None:
+        print(
+            f"{result.status}\t{result.source_id}\t{result.release}\t{result.rows}",
+            flush=True,
+        )
+        super().append(result)
+
+
+def _resolved(path: Path) -> str:
+    return str(path.resolve())
+
+
+def _is_program(path: Path) -> bool:
+    """SAS macros and comment banners are not tables, even when they contain commas."""
+    with path.open("rb") as handle:
+        head = handle.read(80).lstrip().lower()
+    return head.startswith((b"%macro", b"/*", b"*"))
+
+
+def _markup(path: Path) -> bool:
+    if path.suffix.casefold() in _MARKUP:
+        return True
+    with path.open("rb") as handle:
+        head = handle.read(64).lstrip().lower()
+    return head.startswith(b"<")
+
+
+def _delim(path: Path) -> str | None:
+    with path.open("rb") as handle:
+        line = handle.readline(1_000_000)
+    if line.startswith(b"\xef\xbb\xbf"):
+        line = line[3:]
+    text = line.decode("utf-8", "replace")
+    counts = {token: text.count(token) for token in (",", "|", "\t")}
+    best = max(counts, key=counts.get)
+    if counts[best] == 0:
+        return None
+    return best
+
+
+def _is_bls_payload(payload: object) -> bool:
+    items = payload if isinstance(payload, list) else [payload]
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        results = item.get("Results") or item.get("results")
+        if isinstance(results, dict) and isinstance(results.get("series"), list):
+            return True
+    return False
+
+
+def _bls_rows(payload: object) -> list[tuple[str, str, str, str]]:
+    items = payload if isinstance(payload, list) else [payload]
+    rows: list[tuple[str, str, str, str]] = []
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        results = item.get("Results") or item.get("results") or {}
+        if not isinstance(results, dict):
+            continue
+        for series in results.get("series") or []:
+            if not isinstance(series, dict):
+                continue
+            series_id = str(series.get("seriesID") or series.get("seriesId") or "")
+            for obs in series.get("data") or []:
+                if not isinstance(obs, dict):
+                    continue
+                rows.append(
+                    (
+                        series_id,
+                        str(obs.get("year") or ""),
+                        str(obs.get("period") or ""),
+                        str(obs.get("value") if obs.get("value") is not None else ""),
+                    )
+                )
+    return rows
+
+
+def _csv_facts(path: Path) -> tuple[str, bool]:
+    """Return ``(encoding, quoted_newline)``.
+
+    CMS exports put a Windows-1252 apostrophe in an otherwise ASCII file.
+    DuckDB rejects that byte as UTF-8 and does not accept ``latin-1`` for it.
+    A newline inside quotes cannot use the parallel scanner together with
+    ``null_padding``, so those files are read with ``parallel=false``.
+    """
+    decoder = codecs.getincrementaldecoder("utf-8")()
+    encoding = "utf-8"
+    in_quote = False
+    quoted_newline = False
+    with path.open("rb") as handle:
+        while chunk := handle.read(1024 * 1024):
+            if not quoted_newline:
+                for byte in chunk:
+                    if byte == 0x22:
+                        in_quote = not in_quote
+                    elif in_quote and byte in (0x0A, 0x0D):
+                        quoted_newline = True
+                        break
+            if encoding == "utf-8":
+                try:
+                    decoder.decode(chunk)
+                except UnicodeDecodeError:
+                    encoding = "cp1252"
+    if encoding == "utf-8":
+        try:
+            decoder.decode(b"", final=True)
+        except UnicodeDecodeError:
+            encoding = "cp1252"
+    return encoding, quoted_newline
+
+
+def _shared_dialect(
+    pairs: list[tuple[str, Path]],
+) -> tuple[str, str, bool, bool] | None:
+    """Delimiter, encoding, SAS-quote flag, and parallel flag when members agree.
+
+    Parallel is false when any member has a newline inside quotes.
+    """
+    first = pairs[0][1]
+    delim = _delim(first) or ","
+    encoding, quoted = _csv_facts(first)
+    sas = _sas_wrapped(first)
+    for _, path in pairs[1:]:
+        other_encoding, other_quoted = _csv_facts(path)
+        if (_delim(path) or ",") != delim or other_encoding != encoding:
+            return None
+        if _sas_wrapped(path) != sas:
+            return None
+        quoted = quoted or other_quoted
+    return delim, encoding, sas, not quoted
+
+
+def _sas_wrapped(path: Path) -> bool:
+    """HCUP wraps the header in single quotes and comma-bearing fields in double quotes."""
+    with path.open("rb") as handle:
+        line = handle.readline(100_000)
+    if line.startswith(b"\xef\xbb\xbf"):
+        line = line[3:]
+    return line.startswith(b"'") and b"','" in line
+
+
+def _read_csv_sql(
+    *, filename: bool, parallel: bool, skip: int = 0, header: bool = True
+) -> str:
+    # Quote is forced to `"`. Auto-detect treats a suppressed `'-` cell, and
+    # an HCUP single-quoted code, as the quote character and then merges rows.
+    # ``skip = 0`` makes DuckDB fail to sniff some text members, so it is
+    # omitted unless a title banner actually has lines to skip.
+    names = "\n            filename = true," if filename else ""
+    union = "\n            union_by_name = true," if filename else ""
+    skipped = "\n            skip = ?," if skip else ""
+    flag = "true" if parallel else "false"
+    header_sql = "true" if header else "false"
+    return f"""
+        SELECT * FROM read_csv(
+            ?,
+            header = {header_sql},
+            all_varchar = true,
+            delim = ?,
+            quote = '"',
+            escape = '"',
+            encoding = ?,{union}{names}{skipped}
+            strict_mode = false,
+            null_padding = true,
+            parallel = {flag},
+            max_line_size = 10000000
+        )
+    """
+
+
+def _cell_text(value: object) -> str:
+    """One cell as text. Whole numbers lose a trailing ``.0``."""
+    if value is None:
+        return ""
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, float):
+        if value.is_integer():
+            return str(int(value))
+        return str(value)
+    text = str(value).strip()
+    if re.fullmatch(r"-?\d+\.0+", text):
+        return text.split(".", 1)[0]
+    return text
+
+
+def _filled(row: list[str]) -> list[str]:
+    return [cell.strip() for cell in row if cell and cell.strip()]
+
+
+def _is_number(cell: str) -> bool:
+    text = cell.strip().replace(",", "").replace("%", "")
+    if text.startswith("(") and text.endswith(")") and len(text) > 2:
+        text = "-" + text[1:-1]
+    if text in {"", "-", ".", "'-"}:
+        return False
+    try:
+        float(text)
+    except ValueError:
+        return False
+    return True
+
+
+def _is_year(cell: str) -> bool:
+    text = cell.strip()
+    if text.endswith(".0"):
+        text = text[:-2]
+    return len(text) == 4 and text.isdigit() and 1900 <= int(text) <= 2100
+
+
+def _is_code(cell: str) -> bool:
+    text = cell.strip().strip("'\"")
+    if len(text) < 2 or len(text) > 8 or any(ch.isspace() for ch in text):
+        return False
+    if not any(ch.isdigit() for ch in text):
+        return False
+    return text.replace(".", "").isalnum()
+
+
+def _year_header(row: list[str]) -> bool:
+    filled = _filled(row)
+    if len(filled) < 5:
+        return False
+    years = sum(1 for cell in filled if _is_year(cell))
+    return years >= 5 and years * 2 >= len(filled)
+
+
+def _band(row: list[str]) -> bool:
+    """A row that repeats one non-numeric label, such as ``Historical``."""
+    filled = _filled(row)
+    if len(filled) < 5 or any(_is_number(cell) for cell in filled):
+        return False
+    first = filled[0].casefold()
+    same = sum(1 for cell in filled if cell.casefold() == first)
+    return same / len(filled) >= 0.8
+
+
+def _legend(row: list[str]) -> bool:
+    """IRS prints ``-1, -2, -3`` or ``(1), (2)`` under the header as indexes."""
+    filled = _filled(row)
+    if len(filled) < 3:
+        return False
+    for cell in filled:
+        if any(ch.isalpha() for ch in cell):
+            return False
+        text = cell.strip().replace(",", "")
+        if text.endswith(".0"):
+            text = text[:-2]
+        if len(text) >= 3 and text[0] == "(" and text[-1] == ")":
+            text = text[1:-1].strip()
+        if not re.fullmatch(r"-?\d+", text) or abs(int(text)) > 30:
+            return False
+    return True
+
+
+def _dense(row: list[str], width: int) -> bool:
+    filled = _filled(row)
+    count = len(filled)
+    if count < 2:
+        return False
+    span = max(width, len(row), 1)
+    if count >= 3 and count * 2 >= span:
+        return True
+    return span <= 4 and count * 2 >= span
+
+
+def _suppressed(cell: str) -> bool:
+    """IRS withholds a count with ``.`` or ``'-``."""
+    return cell.strip() in {".", "-", "'-"}
+
+
+def _numeric_data(row: list[str]) -> bool:
+    filled = _filled(row)
+    if len(filled) < 3 or _year_header(row) or _legend(row):
+        return False
+    numbers = [cell for cell in filled if _is_number(cell) and not _is_year(cell)]
+    if len(numbers) >= 3 and len(numbers) * 2 >= len(filled):
+        return True
+    # A ZIP row can be mostly suppression dots. That is still an observation.
+    # One real number keeps a row of dots from becoming a header.
+    suppressed = sum(1 for cell in filled if _suppressed(cell))
+    observed = len(numbers) + suppressed
+    return len(numbers) >= 1 and observed >= 3 and observed * 2 >= len(filled)
+
+
+def _code_data(row: list[str]) -> bool:
+    filled = _filled(row)
+    return len(filled) >= 2 and _is_code(filled[0])
+
+
+def _opens_with_data(rows: list[list[str]]) -> bool:
+    """The first record is an observation, so the file has no header.
+
+    A header that merely starts with a code-like token (``ICD10, CC``) is
+    not data: its second cell is a name. A mapping line (``A0104, 92``) is.
+    """
+    if not rows:
+        return False
+    if _numeric_data(rows[0]):
+        return True
+    filled = _filled(rows[0])
+    return len(filled) >= 2 and _is_code(filled[0]) and _is_number(filled[1])
+
+
+def _placeholder(name: str) -> bool:
+    """A name invented for a blank header cell, not a field from the file."""
+    return re.fullmatch(r"(?:column\d+|__unnamed__\d*|col(?:_\d+)?)", name) is not None
+
+
+def _interior_blank(row: list[str]) -> bool:
+    """An empty cell with a name on both sides. A trailing blank does not count."""
+    seen = False
+    gap = False
+    for cell in row:
+        if cell and cell.strip():
+            if gap:
+                return True
+            seen = True
+        elif seen:
+            gap = True
+    return False
+
+
+def _label_fragment(row: list[str]) -> bool:
+    """A wrapped header piece, such as ``NUMBER OF`` above ``RETURNS``.
+
+    A title sentence and a one-cell note are not pieces of the header.
+    """
+    filled = _filled(row)
+    if not filled or _numeric_data(row) or _band(row) or _legend(row):
+        return False
+    if any(len(cell) > 40 or "," in cell for cell in filled):
+        return False
+    if len(filled) == 1:
+        text = filled[0]
+        return " " not in text and len(text) <= 20
+    return True
+
+
+def _compose_header(rows: list[list[str]], index: int) -> list[str]:
+    """Join a stacked label onto a header that has a hole in it.
+
+    Physician-fee files put a wrapped label row above a complete header.
+    Those headers have no hole, so they stay as they are. Old IRS county
+    sheets leave the county-code cell blank and split ``NUMBER OF RETURNS``
+    across two rows. Only that shape is joined.
+    """
+    header = list(rows[index])
+    if not _interior_blank(header):
+        return header
+    above: list[list[str]] = []
+    for cursor in range(index - 1, -1, -1):
+        row = rows[cursor]
+        if not _filled(row) or not _label_fragment(row):
+            break
+        above.append(row)
+    above.reverse()
+    width = max([len(header), *(len(row) for row in above)] or [len(header)])
+    composed: list[str] = []
+    for column in range(width):
+        bits: list[str] = []
+        for row in [*above, header]:
+            cell = row[column].strip() if column < len(row) else ""
+            if cell and cell not in bits:
+                bits.append(cell)
+        composed.append(" ".join(bits))
+    last_filled = max(
+        (cursor for cursor, cell in enumerate(composed) if cell.strip()),
+        default=-1,
+    )
+    merged: list[str] = []
+    carried = ""
+    for cursor, cell in enumerate(composed):
+        if cell.strip():
+            carried = cell.strip()
+            merged.append(carried)
+        elif cursor < last_filled and carried:
+            merged.append(carried)
+        else:
+            merged.append("")
+    return merged
+
+
+def _single_letter(cell: str) -> bool:
+    text = cell.strip()
+    return len(text) == 1 and text.isalpha()
+
+
+def _letter_key(row: list[str]) -> bool:
+    """One letter and a phrase, as in an HCPCS action-code key."""
+    filled = _filled(row)
+    return len(filled) == 2 and _single_letter(filled[0]) and not _is_number(filled[1])
+
+
+def _letter_key_at(rows: list[list[str]]) -> int | None:
+    """Index of a headerless ``A / Add`` list, after a one-cell title."""
+    for index, row in enumerate(rows[:40]):
+        if not _filled(row):
+            continue
+        if _letter_key(row):
+            nxt = rows[index + 1] if index + 1 < len(rows) else []
+            return index if _letter_key(nxt) else None
+        if len(_filled(row)) != 1:
+            return None
+    return None
+
+
+def _complete_short_header(
+    rows: list[list[str]], index: int, header: list[str]
+) -> list[str]:
+    """Join the words above a one-letter header cell onto that cell.
+
+    Old IRS sheets end the Schedule C column at ``C``, with ``Schedules``
+    on the row above. A finished name such as ``HCPCS`` has no one-letter
+    cell, so it stays as it is.
+    """
+    if not any(_single_letter(cell) for cell in header):
+        return header
+    completed = list(header)
+    for column, cell in enumerate(completed):
+        if not _single_letter(cell):
+            continue
+        bits: list[str] = []
+        for cursor in range(index - 1, -1, -1):
+            row = rows[cursor]
+            # A one-cell state name is a group label, not a header piece.
+            if not _filled(row) or not _label_fragment(row) or len(_filled(row)) == 1:
+                break
+            above = row[column].strip() if column < len(row) else ""
+            if above and above not in bits:
+                bits.append(above)
+        if not bits:
+            continue
+        bits.reverse()
+        completed[column] = " ".join([*bits, cell.strip()])
+    return completed
+
+
+def _spacer_label(cell: str) -> bool:
+    """A CMS ``BLANK`` row, a footnote, or a source line. Not a group label."""
+    text = cell.strip()
+    if not text:
+        return True
+    folded = text.casefold().rstrip(":")
+    if folded in {"blank", "n/a", "na"}:
+        return True
+    if folded.startswith(("note", "source", "see note")):
+        return True
+    return text[0] in "¹²³⁴⁵⁶⁷⁸⁹⁰*†"
+
+
+def _section_label(cells: list[str]) -> str | None:
+    """The one cell of a group row, such as ``Aged Beneficiaries`` above a year."""
+    filled = _filled(cells)
+    if len(filled) != 1 or _spacer_label(filled[0]):
+        return None
+    return filled[0]
+
+
+def _header_index(rows: list[list[str]]) -> int | None:
+    """Record index of the header, or None when the sheet is prose.
+
+    The header is the last name row before the first data row. A wrapped
+    label row above ``HCPCS,MOD,...`` is not the header. A title sentence
+    and a repeated ``Historical`` band are not either. ``0`` is row 1.
+    """
+    sample = rows[:40]
+    if not sample:
+        return None
+    width = max(len(row) for row in sample)
+    data_at = None
+    for index, row in enumerate(sample):
+        if _numeric_data(row) or _code_data(row):
+            data_at = index
+            break
+    if data_at is None:
+        for index, row in enumerate(sample):
+            if _dense(row, width) and not _band(row) and not _legend(row):
+                if any(_filled(later) for later in sample[index + 1 : index + 6]):
+                    return index
+        return None
+    if data_at == 0:
+        return 0
+    chosen = None
+    for index in range(data_at):
+        row = sample[index]
+        if _band(row) or _legend(row):
+            continue
+        if _year_header(row) or (
+            _dense(row, width) and not _numeric_data(row) and not _code_data(row)
+        ):
+            chosen = index
+    return chosen
+
+
+def _repeats_header(cells: list[str], names: list[str]) -> bool:
+    """A banner skip can leave the header text as the first data row."""
+    width = max(len(cells), len(names))
+    left = [cell.strip() for cell in cells] + [""] * (width - len(cells))
+    right = [name.strip() for name in names] + [""] * (width - len(names))
+    if not any(left):
+        return False
+    return [cell.casefold() for cell in left] == [name.casefold() for name in right]
+
+
+def _drop_junk_rows(con, ignore: set[str] | None = None) -> None:
+    """Drop a repeated label band, a column-index legend, and one-cell notes."""
+    skipped = {name.casefold() for name in (ignore or set())}
+    described = [row[0] for row in con.execute("DESCRIBE _lead_raw").fetchall()]
+    kept = [
+        index for index, name in enumerate(described) if name.casefold() not in skipped
+    ]
+    if not kept:
+        return
+    drop_ids: list[int] = []
+    for raw_row in con.execute("SELECT rowid, * FROM _lead_raw").fetchall():
+        cells = [_cell_text(raw_row[index + 1]) for index in kept]
+        if (
+            len(_filled(cells)) < 2
+            or _legend(cells)
+            or _band(cells)
+            or _repeats_header(cells, [described[index] for index in kept])
+        ):
+            drop_ids.append(raw_row[0])
+    if drop_ids:
+        con.executemany(
+            "DELETE FROM _lead_raw WHERE rowid = ?", [(row_id,) for row_id in drop_ids]
+        )
+
+
+def _banner_skip(path: Path, delim: str, encoding: str) -> tuple[int, bool]:
+    """Physical lines before the header, and whether that row is a header.
+
+    CMS files put a title, a note, and sometimes a wrapped label row above
+    the header. The header itself may be split across quoted lines. ``0``
+    means row 1 is the header. RxNorm files, and a mapping whose first
+    record is already a code, have no header.
+    """
+    if path.suffix.casefold() == ".rrf":
+        return 0, False
+    with path.open(newline="", encoding=encoding, errors="replace") as handle:
+        reader = csv.reader(handle, delimiter=delim, quotechar='"')
+        rows: list[list[str]] = []
+        starts: list[int] = []
+        for _ in range(40):
+            start = reader.line_num
+            try:
+                row = next(reader)
+            except StopIteration:
+                break
+            rows.append([_cell_text(cell) for cell in row])
+            starts.append(start)
+    if not rows:
+        return 0, True
+    if _opens_with_data(rows):
+        return 0, False
+    index = _header_index(rows)
+    if index is None:
+        return 0, True
+    return starts[index], True
+
+
+def _read_delimited(con, path: Path, *, serial: bool = False) -> bool:
+    """Load *path* into ``_lead_raw``. Return whether single-quote wraps should be stripped."""
+    encoding, quoted_newline = _csv_facts(path)
+    delim = _delim(path) or ","
+    skip, has_header = _banner_skip(path, delim, encoding)
+    params: list[object] = [str(path), delim, encoding]
+    if skip:
+        params.append(skip)
+    parallel = not serial and not quoted_newline
+    try:
+        con.execute(
+            "CREATE OR REPLACE TEMP TABLE _lead_raw AS "
+            + _read_csv_sql(
+                filename=False, parallel=parallel, skip=skip, header=has_header
+            ),
+            params,
+        )
+    except Exception as exc:
+        if parallel and _parallel_refused(exc):
+            raise _NeedsSerial from exc
+        raise
+    if skip:
+        _drop_junk_rows(con)
+    if not has_header:
+        _rename_rrf(con, path)
+    return _sas_wrapped(path)
+
+
+def _rename_rrf(con, path: Path) -> None:
+    """Give a headerless RxNorm file its published column names."""
+    wanted = _RRF_COLUMNS.get(path.stem.casefold())
+    if not wanted:
+        return
+    found = [row[0] for row in con.execute("DESCRIBE _lead_raw").fetchall()]
+    if len(found) == len(wanted) + 1:
+        found = found[:-1]
+    if len(found) != len(wanted):
+        return
+    pairs = ", ".join(
+        f"{quote_ident(old)} AS {quote_ident(new)}" for old, new in zip(found, wanted)
+    )
+    con.execute(
+        f"CREATE OR REPLACE TEMP TABLE _lead_named AS SELECT {pairs} FROM _lead_raw"
+    )
+    con.execute("DROP TABLE _lead_raw")
+    con.execute("ALTER TABLE _lead_named RENAME TO _lead_raw")
+
+
+def _read_json(con, path: Path) -> None:
+    payload = json.loads(path.read_text())
+    if _is_bls_payload(payload):
+        con.execute(
+            """
+            CREATE OR REPLACE TEMP TABLE _lead_raw (
+                series_id VARCHAR, year VARCHAR, period VARCHAR, value VARCHAR
+            )
+            """
+        )
+        rows = _bls_rows(payload)
+        if rows:
+            con.executemany("INSERT INTO _lead_raw VALUES (?, ?, ?, ?)", rows)
+        return
+    con.execute(
+        """
+        CREATE OR REPLACE TEMP TABLE _lead_raw AS
+        SELECT * FROM read_json_auto(?)
+        """,
+        [str(path)],
+    )
+
+
+def _biff_refused(exc: BaseException) -> bool:
+    text = str(exc).lower()
+    return "calamine" in text or "not an office document" in text or "cfb" in text
+
+
+def _empty_sheet(exc: BaseException) -> bool:
+    text = str(exc).lower()
+    return "empty" in text and "sheet" in text
+
+
+def _xlrd_sheets(path: Path) -> list[str]:
+    import xlrd
+
+    return [str(name) for name in xlrd.open_workbook(str(path)).sheet_names()]
+
+
+def _unique_headers(cells: list[str], width: int) -> list[str]:
+    seen: dict[str, int] = {}
+    header: list[str] = []
+    padded = cells + [""] * (width - len(cells))
+    for index, value in enumerate(padded[:width]):
+        text = value.strip() or f"column{index}"
+        seen[text] = seen.get(text, 0) + 1
+        header.append(text if seen[text] == 1 else f"{text}_{seen[text]}")
+    return header
+
+
+def _promote_rows(rows: list[list[str]]):
+    """Use the detected header. A prose sheet returns None.
+
+    A one-cell group label (``Aged Beneficiaries`` above a run of years) is
+    kept in ``section``. A sheet whose header says to not delete it is an
+    Excel dropdown list, not a table.
+    """
+    import polars as pl
+
+    if not rows:
+        return None
+    key_at = _letter_key_at(rows)
+    if key_at is not None:
+        kept = [_filled(row) for row in rows[key_at:] if _letter_key(row)]
+        if not kept:
+            return None
+        return pl.DataFrame(kept, schema=["action_code", "action_label"], orient="row")
+    if _opens_with_data(rows):
+        width = max(len(row) for row in rows)
+        names = [f"column{index}" for index in range(width)]
+        kept = []
+        for row in rows:
+            cells = [_cell_text(value) for value in row]
+            cells = cells[:width] + [""] * (width - len(cells))
+            if len(_filled(cells)) < 2 or _legend(cells) or _band(cells):
+                continue
+            kept.append(cells)
+        if not kept:
+            return None
+        return pl.DataFrame(kept, schema=names, orient="row")
+    index = _header_index(rows)
+    if index is None:
+        return None
+    if any("do not delete" in cell.casefold() for cell in rows[index]):
+        return None
+    width = max(len(row) for row in rows)
+    header = _complete_short_header(rows, index, _compose_header(rows, index))
+    names = _unique_headers(header, width)
+    pending: list[tuple[str, list[str]]] = []
+    section = ""
+    saw_section = False
+    for row in rows[index + 1 :]:
+        cells = [_cell_text(value) for value in row]
+        cells = cells[:width] + [""] * (width - len(cells))
+        label = _section_label(cells)
+        if label is not None:
+            section = label
+            saw_section = True
+            continue
+        if (
+            len(_filled(cells)) < 2
+            or _legend(cells)
+            or _band(cells)
+            or _repeats_header(cells, names)
+        ):
+            continue
+        pending.append((section, cells))
+    if not pending:
+        return None
+    if saw_section:
+        # A group label can sit under the first data rows. Every row gets
+        # the column, empty until the first label, so the width stays put.
+        names = (
+            ["section", *names] if "section" not in names else ["group_label", *names]
+        )
+        kept = [[label, *cells] for label, cells in pending]
+    else:
+        kept = [cells for _, cells in pending]
+    names = _name_leading_stub(names, kept)
+    return pl.DataFrame(kept, schema=names, orient="row")
+
+
+def _name_leading_stub(names: list[str], kept: list[list[str]]) -> list[str]:
+    """A blank first header cell that holds the row stub is ``category``.
+
+    CMS home-health sheets leave that cell empty and put ``Total`` and
+    ``1-9`` under it. An empty placeholder is left for the drop step.
+    """
+    offset = 1 if names and names[0] in {"section", "group_label"} else 0
+    if offset >= len(names) or names[offset] != "column0":
+        return names
+    if not any(len(row) > offset and row[offset].strip() for row in kept):
+        return names
+    renamed = list(names)
+    stub = "category" if "category" not in renamed else "category_2"
+    renamed[offset] = stub
+    return renamed
+
+
+def _rows_from_frame(frame) -> list[list[str]]:
+    return [[_cell_text(value) for value in row] for row in frame.iter_rows()]
+
+
+def _xlrd_frame(path: Path, sheet: str):
+    import xlrd
+
+    book = xlrd.open_workbook(str(path))
+    found = book.sheet_by_name(sheet)
+    if found.nrows == 0:
+        return None
+    rows = [
+        [_cell_text(value) for value in found.row_values(index)]
+        for index in range(found.nrows)
+    ]
+    return _promote_rows(rows)
+
+
+def _register_frame(con, frame) -> None:
+    import polars as pl
+
+    frame = frame.select(pl.all().cast(pl.Utf8))
+    con.register("_lead_arrow", frame.to_arrow())
+    con.execute("CREATE OR REPLACE TEMP TABLE _lead_raw AS SELECT * FROM _lead_arrow")
+    con.unregister("_lead_arrow")
+
+
+def _read_excel(con, path: Path, sheet: str) -> bool:
+    """Load one sheet into ``_lead_raw``. A title-only sheet loads nothing."""
+    import polars as pl
+
+    try:
+        frame = pl.read_excel(
+            path,
+            sheet_name=sheet,
+            has_header=False,
+            infer_schema_length=0,
+            drop_empty_rows=False,
+            drop_empty_cols=False,
+            raise_if_empty=False,
+        )
+    except Exception as exc:
+        if _empty_sheet(exc):
+            return False
+        if not _biff_refused(exc):
+            raise
+        frame = _xlrd_frame(path, sheet)
+        if frame is None or frame.height == 0:
+            return False
+        _register_frame(con, frame)
+        return True
+    if frame.height == 0:
+        return False
+    promoted = _promote_rows(_rows_from_frame(frame))
+    if promoted is None or promoted.height == 0:
+        return False
+    _register_frame(con, promoted)
+    return True
+
+
+_FIXED_COLUMNS = {
+    "codes": ("code", "description"),
+    "order": (
+        "order_number",
+        "code",
+        "header_flag",
+        "short_description",
+        "long_description",
+    ),
+    "codes_addenda": ("action", "code", "description"),
+    "order_addenda": (
+        "action",
+        "header_flag",
+        "code",
+        "short_description",
+        "long_description",
+    ),
+}
+_ORDER_ADDENDA = re.compile(
+    r"^(Add|Delete|Revise from|Revise to):\s+([01])\s+(\S+)\s+(.*\S)\s*$",
+    re.IGNORECASE,
+)
+_CODES_ADDENDA = re.compile(
+    r"^(Add|Delete|Revise from|Revise to):\s+(\S+)\s+(.*\S)\s*$",
+    re.IGNORECASE,
+)
+
+
+def _sample_lines(path: Path, limit: int) -> list[str]:
+    found: list[str] = []
+    with path.open(encoding="utf-8", errors="replace") as handle:
+        for raw in handle:
+            text = raw.rstrip("\n\r")
+            if text.strip():
+                found.append(text)
+            if len(found) >= limit:
+                break
+    return found
+
+
+def _parse_codes_line(line: str) -> tuple[str, ...] | None:
+    if len(line) < 8 or not re.fullmatch(r"[A-Za-z0-9 ]{7}", line[:7]):
+        return None
+    code = line[:7].strip()
+    description = line[7:].strip()
+    if not code or not description or not any(ch.isdigit() for ch in code):
+        return None
+    return (code, description)
+
+
+def _parse_order_line(line: str) -> tuple[str, ...] | None:
+    if (
+        len(line) < 16
+        or not line[:5].isdigit()
+        or line[5] != " "
+        or line[14] not in "01"
+    ):
+        return None
+    code = line[6:13].strip()
+    if not code:
+        return None
+    short = line[16:76].strip()
+    long = line[76:].strip() if len(line) > 76 else ""
+    return (line[:5], code, line[14], short, long)
+
+
+def _split_pair(text: str) -> tuple[str, str]:
+    parts = re.split(r"\s{2,}", text.strip(), maxsplit=1)
+    if len(parts) == 1:
+        return parts[0], ""
+    return parts[0], parts[1].strip()
+
+
+def _parse_codes_addenda(line: str) -> tuple[str, ...] | None:
+    matched = _CODES_ADDENDA.match(line)
+    if matched is None or matched.group(2) in {"0", "1"}:
+        return None
+    return (matched.group(1).capitalize(), matched.group(2), matched.group(3).strip())
+
+
+def _parse_order_addenda(line: str) -> tuple[str, ...] | None:
+    matched = _ORDER_ADDENDA.match(line)
+    if matched is None:
+        return None
+    short, long = _split_pair(matched.group(4))
+    action = matched.group(1)
+    if action.lower() == "add":
+        action = "Add"
+    elif action.lower() == "delete":
+        action = "Delete"
+    else:
+        action = action[0].upper() + action[1:]
+    return (action, matched.group(2), matched.group(3), short, long)
+
+
+def _fixed_kind(path: Path) -> str | None:
+    """ICD code, order, and addenda files are fixed width, not CSV."""
+    if path.suffix.casefold() not in {".txt", ".dat"}:
+        return None
+    name = path.name.casefold()
+    if "addenda" in name and any(token in name for token in ("icd", "codes", "order")):
+        lines = _sample_lines(path, 40)
+        order_hits = sum(1 for line in lines if _ORDER_ADDENDA.match(line))
+        code_hits = sum(1 for line in lines if _CODES_ADDENDA.match(line))
+        # An order-addenda line also matches the codes pattern, so the
+        # flag match wins when the two counts are equal.
+        if order_hits and order_hits >= code_hits:
+            return "order_addenda"
+        if code_hits:
+            return "codes_addenda"
+        return None
+    if "icd10" in name and "order" in name:
+        lines = _sample_lines(path, 15)
+        hits = sum(1 for line in lines if _parse_order_line(line))
+        return "order" if hits else None
+    if "icd10" in name and "codes" in name:
+        lines = _sample_lines(path, 15)
+        hits = sum(1 for line in lines if _parse_codes_line(line))
+        return "codes" if hits else None
+    return None
+
+
+def _parse_fixed_line(line: str, kind: str) -> tuple[str, ...] | None:
+    if kind == "codes":
+        return _parse_codes_line(line)
+    if kind == "order":
+        return _parse_order_line(line)
+    if kind == "codes_addenda":
+        return _parse_codes_addenda(line)
+    return _parse_order_addenda(line)
+
+
+def _read_fixed(con, path: Path, kind: str) -> bool:
+    columns = _FIXED_COLUMNS[kind]
+    rows: list[tuple[str, ...]] = []
+    with path.open(encoding="utf-8", errors="replace") as handle:
+        for raw in handle:
+            parsed = _parse_fixed_line(raw.rstrip("\n\r"), kind)
+            if parsed:
+                rows.append(parsed)
+    if not rows:
+        return False
+    import polars as pl
+
+    frame = pl.DataFrame(
+        {name: [row[index] for row in rows] for index, name in enumerate(columns)}
+    )
+    _register_frame(con, frame)
+    return True
+
+
+def _as_varchar(src: str, *, strip_single: bool) -> str:
+    expr = f"CAST({quote_ident(src)} AS VARCHAR)"
+    if not strip_single:
+        return expr
+    wrapped = f"regexp_full_match({expr}, '^''.*''$')"
+    return f"CASE WHEN {wrapped} THEN substr({expr}, 2, length({expr}) - 2) ELSE {expr} END"
+
+
+def _replace_clean(con, pairs: str) -> None:
+    con.execute(
+        f"CREATE OR REPLACE TEMP TABLE _lead_renamed AS SELECT {pairs} FROM _lead_clean"
+    )
+    con.execute("DROP TABLE _lead_clean")
+    con.execute("ALTER TABLE _lead_renamed RENAME TO _lead_clean")
+
+
+def _rename_icd_map(con, names: list[str]) -> list[str]:
+    """Name a headerless ``code, number, D`` file the way the later CSV does.
+
+    CMS ships the ICD-to-CC map as a tab file with no header. The first
+    record is a code, so it must not become the column names. ``D`` marks
+    the second CC for a code that maps twice.
+    """
+    if len(names) not in {2, 3} or not all(_placeholder(name) for name in names):
+        return names
+    listed = ", ".join(quote_ident(name) for name in names)
+    sample = con.execute(f"SELECT {listed} FROM _lead_clean LIMIT 40").fetchall()
+    if not sample:
+        return names
+    for row in sample:
+        code = _cell_text(row[0])
+        number = _cell_text(row[1])
+        # A letter plus digits. A bare FIPS or a header token must not match.
+        if re.fullmatch(r"[A-Za-z][A-Za-z0-9]{2,6}", code) is None or not _is_number(
+            number
+        ):
+            return names
+        if len(names) == 3 and _cell_text(row[2]) not in {"", "D"}:
+            return names
+    renamed = ["icd10", "cc"] + (["duplicate_cc"] if len(names) == 3 else [])
+    pairs = ", ".join(
+        f"{quote_ident(old)} AS {quote_ident(new)}"
+        for old, new in zip(names, renamed, strict=True)
+    )
+    _replace_clean(con, pairs)
+    return renamed
+
+
+def _drop_empty_placeholders(con, names: list[str]) -> list[str]:
+    """Drop a blank header that never holds a value. A filled one stays."""
+    kept: list[str] = []
+    for name in names:
+        if _placeholder(name):
+            # A blank header that holds only a backtick or a suppression dot
+            # is not a field. A digit or a letter means the cell is real.
+            ident = quote_ident(name)
+            hit = con.execute(
+                f"SELECT 1 FROM _lead_clean WHERE {ident} IS NOT NULL "
+                f"AND regexp_full_match(trim(CAST({ident} AS VARCHAR)), "
+                f"'.*[A-Za-z0-9].*') LIMIT 1"
+            ).fetchone()
+            if hit is None:
+                continue
+        kept.append(name)
+    if not kept or len(kept) == len(names):
+        return names
+    _replace_clean(con, ", ".join(quote_ident(name) for name in kept))
+    return kept
+
+
+def _clean(con, *, strip_single: bool = False) -> list[str]:
+    raw = [row[0] for row in con.execute("DESCRIBE _lead_raw").fetchall()]
+    names = unique_names(raw)
+    pairs = ", ".join(
+        f"{_as_varchar(src, strip_single=strip_single)} AS {dst}"
+        for src, dst in zip(raw, names, strict=True)
+    )
+    con.execute(
+        f"CREATE OR REPLACE TEMP TABLE _lead_clean AS SELECT {pairs} FROM _lead_raw"
+    )
+    names = _rename_icd_map(con, names)
+    return _drop_empty_placeholders(con, names)
+
+
+def _excel_sheets(path: Path) -> list[str]:
+    try:
+        from fastexcel import read_excel
+
+        return list(read_excel(path).sheet_names)
+    except Exception as exc:
+        if not _biff_refused(exc):
+            raise
+        return _xlrd_sheets(path)
+
+
+def _textual_table(path: Path) -> bool:
+    """Prose that mentions a comma once is not a table.
+
+    A real extract repeats its delimiter on the next line. A readme does not.
+    """
+    delim = _delim(path)
+    if delim is None:
+        return False
+    token = delim.encode()
+    with path.open("rb") as handle:
+        first = handle.readline(100_000)
+        second = handle.readline(100_000)
+    if token not in first:
+        return False
+    return not second or token in second
+
+
+def _zip_members(path: Path, only: list[str] | None) -> list[str]:
+    with zipfile.ZipFile(path) as archive:
+        names = [name for name in archive.namelist() if not name.endswith("/")]
+    if only is not None:
+        wanted = set(only)
+        return [name for name in names if name in wanted]
+    chosen: list[str] = []
+    for name in names:
+        suffix = Path(name).suffix.casefold()
+        if suffix in _TABULAR:
+            chosen.append(name)
+        elif suffix == ".zip":
+            chosen.extend(_nested(path, name))
+    return chosen
+
+
+def _nested(path: Path, member: str) -> list[str]:
+    """One extra zip level. The returned names are ``outer::inner``."""
+    import io
+
+    found: list[str] = []
+    with zipfile.ZipFile(path) as outer, outer.open(member) as handle:
+        blob = handle.read()
+    if not blob.startswith(b"PK"):
+        return []
+    with zipfile.ZipFile(io.BytesIO(blob)) as inner:
+        for name in inner.namelist():
+            if name.endswith("/"):
+                continue
+            if Path(name).suffix.casefold() in _TABULAR:
+                found.append(f"{member}::{name}")
+    return found
+
+
+def _materialize(archive: Path, member: str, dest: Path) -> Path:
+    if "::" in member:
+        outer_name, inner_name = member.split("::", 1)
+        target = dest / inner_name
+        target.parent.mkdir(parents=True, exist_ok=True)
+        import io
+
+        with zipfile.ZipFile(archive) as outer, outer.open(outer_name) as handle:
+            blob = handle.read()
+        with zipfile.ZipFile(io.BytesIO(blob)) as inner, inner.open(inner_name) as src:
+            with target.open("wb") as dst:
+                shutil.copyfileobj(src, dst)
+        return target
+    target = dest / member
+    target.parent.mkdir(parents=True, exist_ok=True)
+    with (
+        zipfile.ZipFile(archive) as zf,
+        zf.open(member) as src,
+        target.open("wb") as dst,
+    ):
+        shutil.copyfileobj(src, dst)
+    return target
+
+
+def _load_path_into(
+    con,
+    path: Path,
+    *,
+    table: str,
+    release: str,
+    sha256: str,
+    source_path: str,
+    member: str,
+    file_name: str,
+    when: tuple[int, int] | None,
+    serial: bool = False,
+) -> int:
+    suffix = path.suffix.casefold()
+    strip_single = False
+    fixed = _fixed_kind(path)
+    if fixed:
+        if not _read_fixed(con, path, fixed):
+            return 0
+        columns = _clean(con)
+        return insert_frame(
+            con,
+            table,
+            "_lead_clean",
+            columns,
+            release=release,
+            sha256=sha256,
+            source_path=source_path,
+            member=member,
+            file_name=file_name,
+            when=when,
+        )
+    if suffix in {".xlsx", ".xls"}:
+        total = 0
+        for sheet in _excel_sheets(path):
+            if not _read_excel(con, path, sheet):
+                continue
+            columns = _clean(con)
+            label = sheet if member in {"", path.name} else f"{member}#{sheet}"
+            if member and suffix in {".xlsx", ".xls"} and member != path.name:
+                label = f"{member}#{sheet}"
+            elif member in {"", path.name}:
+                label = sheet
+            total += insert_frame(
+                con,
+                table,
+                "_lead_clean",
+                columns,
+                release=release,
+                sha256=sha256,
+                source_path=source_path,
+                member=label,
+                file_name=file_name,
+                when=when,
+            )
+        return total
+    if suffix == ".json":
+        _read_json(con, path)
+    elif suffix in _DELIMITED or _delim(path) is not None:
+        if suffix in {".txt", ".dat"} and not _textual_table(path):
+            return 0
+        strip_single = _read_delimited(con, path, serial=serial)
+    else:
+        return 0
+    columns = _clean(con, strip_single=strip_single)
+    return insert_frame(
+        con,
+        table,
+        "_lead_clean",
+        columns,
+        release=release,
+        sha256=sha256,
+        source_path=source_path,
+        member=member,
+        file_name=file_name,
+        when=when,
+    )
+
+
+def _load_delimited_batch(
+    con,
+    pairs: list[tuple[str, Path]],
+    *,
+    table: str,
+    release: str,
+    sha256: str,
+    source_path: str,
+    file_name: str,
+    when: tuple[int, int] | None,
+    serial: bool = False,
+) -> int:
+    """Read delimited members. Files with different banners are not one scan."""
+    buckets: dict[tuple[int, bool], list[tuple[str, Path]]] = {}
+    for member, path in pairs:
+        delim = _delim(path) or ","
+        encoding, _quoted = _csv_facts(path)
+        skip, has_header = _banner_skip(path, delim, encoding)
+        buckets.setdefault((skip, has_header), []).append((member, path))
+    total = 0
+    for (skip, has_header), group in buckets.items():
+        total += _load_delimited_group(
+            con,
+            group,
+            table=table,
+            release=release,
+            sha256=sha256,
+            source_path=source_path,
+            file_name=file_name,
+            when=when,
+            serial=serial,
+            skip=skip,
+            has_header=has_header,
+        )
+    return total
+
+
+def _load_delimited_group(
+    con,
+    pairs: list[tuple[str, Path]],
+    *,
+    table: str,
+    release: str,
+    sha256: str,
+    source_path: str,
+    file_name: str,
+    when: tuple[int, int] | None,
+    serial: bool = False,
+    skip: int = 0,
+    has_header: bool = True,
+) -> int:
+    """Read members that share one banner depth. Each row keeps its member."""
+    dialect = _shared_dialect(pairs)
+    # Headerless members do not share a column list, so one read_csv would
+    # take the first record of each file as its header.
+    if (
+        not has_header
+        or dialect is None
+        or len(pairs) == 1
+        or any(path.suffix.casefold() == ".rrf" for _, path in pairs)
+    ):
+        total = 0
+        for member, path in pairs:
+            total += _load_path_into(
+                con,
+                path,
+                table=table,
+                release=release,
+                sha256=sha256,
+                source_path=source_path,
+                member=member,
+                file_name=file_name,
+                when=when,
+                serial=serial,
+            )
+        return total
+    delim, encoding, strip_single, parallel = dialect
+    parallel = parallel and not serial
+    paths = [str(path) for _, path in pairs]
+    params: list[object] = [paths, delim, encoding]
+    if skip:
+        params.append(skip)
+    try:
+        con.execute(
+            "CREATE OR REPLACE TEMP TABLE _lead_raw AS "
+            + _read_csv_sql(filename=True, parallel=parallel, skip=skip),
+            params,
+        )
+    except Exception as exc:
+        if parallel and _parallel_refused(exc):
+            raise _NeedsSerial from exc
+        raise
+    if skip:
+        _drop_junk_rows(con, ignore={"filename"})
+    con.execute(
+        "CREATE OR REPLACE TEMP TABLE _lead_map (filename VARCHAR, zip_member VARCHAR)"
+    )
+    mapped: dict[str, str] = {}
+    for member, path in pairs:
+        mapped[str(path)] = member
+        mapped[str(path.resolve())] = member
+    con.executemany("INSERT INTO _lead_map VALUES (?, ?)", list(mapped.items()))
+    con.execute(
+        """
+        CREATE OR REPLACE TEMP TABLE _lead_joined AS
+        SELECT raw.* EXCLUDE (filename), COALESCE(map.zip_member, '') AS zip_member
+        FROM _lead_raw AS raw
+        LEFT JOIN _lead_map AS map ON raw.filename = map.filename
+        """
+    )
+    con.execute("DROP TABLE _lead_raw")
+    con.execute("ALTER TABLE _lead_joined RENAME TO _lead_raw")
+    columns = [
+        name for name in _clean(con, strip_single=strip_single) if name != "zip_member"
+    ]
+    return insert_frame(
+        con,
+        table,
+        "_lead_clean",
+        columns,
+        release=release,
+        sha256=sha256,
+        source_path=source_path,
+        member="",
+        file_name=file_name,
+        when=when,
+        member_column="zip_member",
+    )
+
+
+def load_file(
+    con,
+    path: Path | str,
+    *,
+    table: str,
+    source_id: str,
+    release: str,
+    sha256: str,
+    actor: str,
+    when: tuple[int, int] | None = None,
+    members: list[str] | None = None,
+    _serial: bool = False,
+) -> int:
+    """Load one local file. Return how many observation rows were inserted.
+
+    A second call with the same source, path, and sha256 inserts nothing.
+    A sha256 that does not match the bytes raises :class:`HashMismatch`
+    and inserts nothing.
+
+    Raises:
+        FileNotFoundError: *path* does not exist.
+        HashMismatch: the bytes are not *sha256*.
+        ValueError: *sha256* is blank, or *table* is not an identifier.
+    """
+    src = Path(path)
+    qualified = qualify(table)
+    stored = _resolved(src)
+    ensure_log(con)
+    if already_loaded(con, source_id=source_id, path=stored, sha256=sha256):
+        return 0
+    if not src.exists():
+        raise FileNotFoundError(src)
+    digest = require_hash(src, sha256)
+    if _markup(src) or (
+        src.suffix.casefold() not in _TABULAR | {".zip"} and _delim(src) is None
+    ):
+        record(
+            con,
+            source_id=source_id,
+            release=release,
+            sha256=digest,
+            path=stored,
+            table=qualified,
+            status="skipped",
+            rows=0,
+            reason="not_tabular",
+            actor=actor,
+        )
+        return 0
+    con.execute("BEGIN")
+    try:
+        if _table_exists(con, qualified):
+            con.execute(f"DELETE FROM {qualified} WHERE source_path = ?", [stored])
+        if src.suffix.casefold() == ".zip":
+            chosen = _zip_members(src, members)
+            if not chosen:
+                record(
+                    con,
+                    source_id=source_id,
+                    release=release,
+                    sha256=digest,
+                    path=stored,
+                    table=qualified,
+                    status="skipped",
+                    rows=0,
+                    reason="not_tabular",
+                    actor=actor,
+                )
+                con.execute("COMMIT")
+                return 0
+            import tempfile
+
+            total = 0
+            with tempfile.TemporaryDirectory(prefix="lead-load-") as tmp:
+                dest = Path(tmp)
+                delimited: list[tuple[str, Path]] = []
+                other: list[tuple[str, Path]] = []
+                for member in chosen:
+                    extracted = _materialize(src, member, dest)
+                    if _markup(extracted) or _is_program(extracted):
+                        continue
+                    suffix = extracted.suffix.casefold()
+                    if _fixed_kind(extracted):
+                        other.append((member, extracted))
+                        continue
+                    if suffix in {".txt", ".dat"} and not _textual_table(extracted):
+                        continue
+                    if suffix in _DELIMITED:
+                        delimited.append((member, extracted))
+                    else:
+                        other.append((member, extracted))
+                if delimited:
+                    total += _load_delimited_batch(
+                        con,
+                        delimited,
+                        table=qualified,
+                        release=release,
+                        sha256=digest,
+                        source_path=stored,
+                        file_name=src.name,
+                        when=when,
+                        serial=_serial,
+                    )
+                for member, extracted in other:
+                    total += _load_path_into(
+                        con,
+                        extracted,
+                        table=qualified,
+                        release=release,
+                        sha256=digest,
+                        source_path=stored,
+                        member=member,
+                        file_name=src.name,
+                        when=when,
+                        serial=_serial,
+                    )
+            if total == 0:
+                record(
+                    con,
+                    source_id=source_id,
+                    release=release,
+                    sha256=digest,
+                    path=stored,
+                    table=qualified,
+                    status="skipped",
+                    rows=0,
+                    reason="not_tabular",
+                    actor=actor,
+                )
+                con.execute("COMMIT")
+                return 0
+        else:
+            total = _load_path_into(
+                con,
+                src,
+                table=qualified,
+                release=release,
+                sha256=digest,
+                source_path=stored,
+                member="",
+                file_name=src.name,
+                when=when,
+                serial=_serial,
+            )
+            if total == 0 and src.suffix.casefold() not in _TABULAR:
+                record(
+                    con,
+                    source_id=source_id,
+                    release=release,
+                    sha256=digest,
+                    path=stored,
+                    table=qualified,
+                    status="skipped",
+                    rows=0,
+                    reason="not_tabular",
+                    actor=actor,
+                )
+                con.execute("COMMIT")
+                return 0
+        record(
+            con,
+            source_id=source_id,
+            release=release,
+            sha256=digest,
+            path=stored,
+            table=qualified,
+            status="loaded",
+            rows=total,
+            reason="",
+            actor=actor,
+        )
+        con.execute("COMMIT")
+    except _NeedsSerial:
+        con.execute("ROLLBACK")
+        if _serial:
+            raise
+        return load_file(
+            con,
+            src,
+            table=table,
+            source_id=source_id,
+            release=release,
+            sha256=sha256,
+            actor=actor,
+            when=when,
+            members=members,
+            _serial=True,
+        )
+    except Exception:
+        con.execute("ROLLBACK")
+        raise
+    return total
+
+
+def _table_exists(con, qualified: str) -> bool:
+    schema, name = qualified.split(".")
+    row = con.execute(
+        """
+        SELECT 1 FROM information_schema.tables
+        WHERE table_schema = ? AND table_name = ?
+        """,
+        [schema, name],
+    ).fetchone()
+    return row is not None
+
+
+def load_manifest_files(
+    con, csv_path: Path | str, *, root: Path | str, actor: str
+) -> tuple[LoadResult, ...]:
+    """Load every ``kind=data`` row under *root*. Other kinds are skipped.
+
+    A blank sha256 is a gap and is not read. A hash mismatch does not
+    insert that file and does not stop the rest of the manifest.
+    """
+    ensure_log(con)
+    base = Path(root)
+    results: list[LoadResult] = _Results()
+    with Path(csv_path).open(newline="") as handle:
+        reader = csv.DictReader(handle)
+        for row in reader:
+            source_id = (row.get("source_id") or "").strip()
+            release = (row.get("release") or "").strip()
+            kind = (row.get("kind") or "").strip().casefold()
+            rel = (row.get("path") or "").strip()
+            digest = (row.get("sha256") or "").strip().lower()
+            path = Path(rel) if Path(rel).is_absolute() else base / rel
+            table = qualify(source_id)
+            if kind != "data":
+                record(
+                    con,
+                    source_id=source_id,
+                    release=release,
+                    sha256=digest,
+                    path=rel,
+                    table=table,
+                    status="skipped",
+                    rows=0,
+                    reason=f"kind:{kind}",
+                    actor=actor,
+                )
+                results.append(
+                    LoadResult(
+                        source_id,
+                        release,
+                        rel,
+                        digest,
+                        "skipped",
+                        0,
+                        table,
+                        f"kind:{kind}",
+                    )
+                )
+                continue
+            if not digest:
+                results.append(
+                    LoadResult(
+                        source_id,
+                        release,
+                        rel,
+                        "",
+                        "gap",
+                        0,
+                        table,
+                        "sha256 is required",
+                    )
+                )
+                continue
+            if not path.exists():
+                results.append(
+                    LoadResult(
+                        source_id,
+                        release,
+                        rel,
+                        digest,
+                        "missing",
+                        0,
+                        table,
+                        "file is missing",
+                    )
+                )
+                continue
+            try:
+                inserted = load_file(
+                    con,
+                    path,
+                    table=table,
+                    source_id=source_id,
+                    release=release,
+                    sha256=digest,
+                    actor=actor,
+                )
+            except HashMismatch:
+                results.append(
+                    LoadResult(
+                        source_id,
+                        release,
+                        rel,
+                        digest,
+                        "mismatch",
+                        0,
+                        table,
+                        "sha256 mismatch",
+                    )
+                )
+                continue
+            except Exception as exc:
+                results.append(
+                    LoadResult(
+                        source_id,
+                        release,
+                        rel,
+                        digest,
+                        "unreadable",
+                        0,
+                        table,
+                        str(exc).split("\n", 1)[0][:400],
+                    )
+                )
+                continue
+            status, reason = "loaded", ""
+            if inserted == 0:
+                logged, why = _log_status(con, source_id, _resolved(path))
+                if logged == "skipped":
+                    status, reason = logged, why
+            results.append(
+                LoadResult(
+                    source_id, release, rel, digest, status, inserted, table, reason
+                )
+            )
+    return tuple(results)
+
+
+def _log_status(con, source_id: str, path: str) -> tuple[str, str]:
+    row = con.execute(
+        """
+        SELECT status, reason FROM lead_public.load_log
+        WHERE source_id = ? AND path = ?
+        ORDER BY rowid DESC
+        LIMIT 1
+        """,
+        [source_id, path],
+    ).fetchone()
+    if row is None:
+        return "loaded", ""
+    return str(row[0]), str(row[1] or "")
diff --git a/src/lead/load/spine_map.py b/src/lead/load/spine_map.py
new file mode 100644
index 0000000..a9346b9
--- /dev/null
+++ b/src/lead/load/spine_map.py
@@ -0,0 +1,94 @@
+"""Which columns and filename tokens feed the public-data spine.
+
+This does not crosswalk a FIPS to a ZCTA, an HSA, an HRR, or a CBSA.
+A row keeps every grain it already carries.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+_FOUR = re.compile(r"(? str:
+    for alias in aliases:
+        if alias in names:
+            return alias
+    return ""
+
+
+def spine_columns(names: list[str] | tuple[str, ...]) -> SpineMap:
+    """Map sanitized headers onto spine roles.
+
+    A column literally named ``fips`` wins over ``area_fips``. State and
+    county pieces stay separate here; the loader composes a county FIPS
+    only when it writes a row.
+    """
+    have = set(names)
+    return SpineMap(
+        fips=_pick(have, ("fips", "fips_code", "fips_cd", "area_fips")),
+        zcta=_pick(have, ("zcta", "zcta5", "zcta_code")),
+        zip=_pick(have, ("zipcode", "zip_code", "zip_cd", "zip")),
+        hsa=_pick(have, ("hsa", "hsa_num", "hsanum")),
+        hrr=_pick(have, ("hrr", "hrr_num", "hrrnum")),
+        cbsa=_pick(have, ("cbsa", "cbsa_code", "cbsa_cd")),
+        year=_pick(have, ("year", "calendar_year", "data_year", "yr")),
+        quarter=_pick(have, ("quarter", "qtr")),
+        month=_pick(have, ("month", "mon")),
+        state_fips=_pick(have, ("statefips", "state_fips", "fipsst")),
+        county_fips=_pick(have, ("countyfips", "county_fips", "fipscnty")),
+        period=_pick(have, ("period",)),
+    )
+
+
+def file_year(name: str) -> int | None:
+    """One year token from a published filename.
+
+    Two different four-digit years are not a year. A two-digit IRS SOI
+    prefix (``23incy``, ``12cyall``, ``14zpall``) is 2000 plus that prefix.
+    """
+    years = _FOUR.findall(Path(name).name)
+    distinct = set(years)
+    if len(distinct) > 1:
+        return None
+    if len(distinct) == 1:
+        return int(years[0])
+    matched = _IRS2.search(Path(name).name.casefold())
+    if matched:
+        return 2000 + int(matched.group(1))
+    return None
+
+
+def file_quarter(name: str) -> int | None:
+    """One ``Q1``–``Q4`` token. A span such as ``q1-q4`` is not a quarter."""
+    stem = Path(name).name
+    if _QUARTER_SPAN.search(stem):
+        return None
+    found = set(_QUARTER.findall(stem))
+    if len(found) == 1:
+        return int(next(iter(found)))
+    return None
diff --git a/src/lead/load/strand5.py b/src/lead/load/strand5.py
new file mode 100644
index 0000000..8a5e5c2
--- /dev/null
+++ b/src/lead/load/strand5.py
@@ -0,0 +1,579 @@
+"""Load the strand-5 files already on disk.
+
+``files.tsv`` is the inventory. A row whose status is not ``ok`` is not
+read. FRED stays absent when the inventory says the file was never
+retrieved. The MDS frequency extract is skipped here because the
+enrichment manifest loads that file.
+"""
+
+from __future__ import annotations
+
+import re
+import zipfile
+from dataclasses import dataclass
+from pathlib import Path
+
+from bls.ces import SERIES, CesRow
+from bls.ces import write as write_ces
+from bls.oews import read_national
+from bls.oews import to_rows as oews_rows
+from bls.oews import write as write_oews
+from bls.ppi import parse_series
+from lead.load.errors import HashMismatch
+from lead.load.frame import (
+    already_loaded,
+    ensure_log,
+    latest,
+    qualify,
+    record,
+    require_hash,
+)
+from lead.load.irs import choose_irs_members
+from lead.load.public import LoadResult, _resolved, _Results, load_file
+
+_SERIES_TABLES = {
+    "bls_ppi": "bls.ppi",
+    "bls_jolts": "bls.jolts",
+}
+_ABSENT = {"http_404": "http_404", "unreachable": "unreachable"}
+
+
+@dataclass(frozen=True)
+class _Item:
+    source: str
+    url: str
+    path: str
+    sha256: str
+    status: str
+    note: str
+
+
+def _inventory(root: Path) -> list[_Item]:
+    lines = (root / "files.tsv").read_text().splitlines()
+    header = lines[0].split("\t")
+    index = {name: pos for pos, name in enumerate(header)}
+
+    def column(parts: list[str], name: str) -> str:
+        pos = index[name]
+        return parts[pos] if pos < len(parts) else ""
+
+    items: list[_Item] = []
+    for line in lines[1:]:
+        if not line.strip():
+            continue
+        parts = line.split("\t")
+        items.append(
+            _Item(
+                source=column(parts, "source").strip(),
+                url=column(parts, "url").strip(),
+                path=column(parts, "path").strip(),
+                sha256=column(parts, "sha256").strip().lower(),
+                status=column(parts, "status").strip(),
+                note=column(parts, "note").strip(),
+            )
+        )
+    return items
+
+
+def _locate(root: Path, raw: str) -> Path:
+    path = Path(raw)
+    if path.is_absolute():
+        return path
+    return root / raw
+
+
+def _period(period: str, *, annual_month: bool) -> tuple[int | None, int | None]:
+    if re.fullmatch(r"M(0[1-9]|1[0-2])", period):
+        month = int(period[1:])
+        return month, 1 + (month - 1) // 3
+    if annual_month and period == "M13":
+        return 13, None
+    if re.fullmatch(r"Q0[1-4]", period):
+        return None, int(period[2:])
+    return None, None
+
+
+def _time_key(year: int | None, quarter: int | None) -> str | None:
+    if year is None or quarter is None:
+        return None
+    return f"{year}Q{quarter}"
+
+
+def _ensure_series(con, table: str) -> None:
+    schema, name = qualify(table).split(".")
+    con.execute(f"CREATE SCHEMA IF NOT EXISTS {schema}")
+    con.execute(
+        f"""
+        CREATE TABLE IF NOT EXISTS {schema}.{name} (
+            series_id VARCHAR,
+            spine_year INTEGER,
+            period VARCHAR,
+            value DOUBLE,
+            spine_month INTEGER,
+            spine_quarter INTEGER,
+            time_key VARCHAR,
+            content_sha256 VARCHAR,
+            source_path VARCHAR,
+            release VARCHAR
+        )
+        """
+    )
+
+
+def _load_series(
+    con, path: Path, *, table: str, source_id: str, sha256: str, actor: str
+) -> int:
+    digest = require_hash(path, sha256)
+    stored = _resolved(path)
+    rows = parse_series(path.read_text())
+    qualified = qualify(table)
+    _ensure_series(con, qualified)
+    con.execute(f"DELETE FROM {qualified} WHERE source_path = ?", [stored])
+    payload = []
+    for row in rows:
+        month, quarter = _period(row.period, annual_month=False)
+        payload.append(
+            (
+                row.series_id,
+                row.year,
+                row.period,
+                row.value,
+                month,
+                quarter,
+                _time_key(row.year, quarter),
+                digest,
+                stored,
+                path.name,
+            )
+        )
+    if payload:
+        con.executemany(
+            f"""
+            INSERT INTO {qualified} (
+                series_id, spine_year, period, value, spine_month, spine_quarter,
+                time_key, content_sha256, source_path, release
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            payload,
+        )
+    record(
+        con,
+        source_id=source_id,
+        release=path.name,
+        sha256=digest,
+        path=stored,
+        table=qualified,
+        status="loaded",
+        rows=len(payload),
+        reason="",
+        actor=actor,
+    )
+    return len(payload)
+
+
+def _load_ces(con, path: Path, *, sha256: str, actor: str) -> int:
+    digest = require_hash(path, sha256)
+    stored = _resolved(path)
+    built: list[CesRow] = []
+    for row in parse_series(path.read_text()):
+        month, _quarter = _period(row.period, annual_month=True)
+        if month is None or row.year is None:
+            continue
+        meta = SERIES.get(row.series_id)
+        built.append(
+            CesRow(
+                series_id=row.series_id,
+                industry_code=meta.industry_code if meta else "",
+                industry_name=meta.industry_name if meta else "",
+                data_type=meta.data_type if meta else "",
+                seasonal=meta.seasonal if meta else "",
+                year=row.year,
+                month=month,
+                value=row.value,
+                item_key="",
+            )
+        )
+    count = write_ces(con, built)
+    record(
+        con,
+        source_id="bls_ces",
+        release=path.name,
+        sha256=digest,
+        path=stored,
+        table="bls.ces",
+        status="loaded",
+        rows=count,
+        reason="",
+        actor=actor,
+    )
+    return count
+
+
+def _oews_year(path: Path) -> int:
+    with zipfile.ZipFile(path) as archive:
+        for name in archive.namelist():
+            matched = re.search(r"national_M(\d{4})_dl", name, re.I)
+            if matched:
+                return int(matched.group(1))
+    matched = re.search(r"oesm(\d{2})nat", path.name, re.I)
+    if matched is None:
+        raise ValueError(f"no OEWS year in {path}")
+    year = int(matched.group(1))
+    return 2000 + year if year < 80 else 1900 + year
+
+
+def _load_oews(con, path: Path, *, sha256: str, actor: str) -> int:
+    digest = require_hash(path, sha256)
+    year = _oews_year(path)
+    count = write_oews(
+        con, year, oews_rows(year, read_national(path, year), item_key="")
+    )
+    record(
+        con,
+        source_id="bls_oews",
+        release=str(year),
+        sha256=digest,
+        path=_resolved(path),
+        table="bls.oews",
+        status="loaded",
+        rows=count,
+        reason="",
+        actor=actor,
+    )
+    return count
+
+
+def _when(path: Path) -> tuple[int, int] | None:
+    matched = re.fullmatch(r"(\d{4})-(\d{2})", path.parent.name)
+    if matched is None:
+        return None
+    month = int(matched.group(2))
+    if month < 1 or month > 12:
+        return None
+    return int(matched.group(1)), month
+
+
+def _load_irs(
+    con,
+    path: Path,
+    *,
+    sha256: str,
+    actor: str,
+    standalone: set[str],
+) -> int:
+    if path.suffix.casefold() != ".zip":
+        return load_file(
+            con,
+            path,
+            table="econ.irs_soi",
+            source_id="irs_soi",
+            release=path.name,
+            sha256=sha256,
+            actor=actor,
+        )
+    with zipfile.ZipFile(path) as archive:
+        names = archive.namelist()
+    chosen = [
+        name
+        for name in choose_irs_members(names)
+        if Path(name).name.casefold() not in standalone
+    ]
+    if not chosen:
+        record(
+            con,
+            source_id="irs_soi",
+            release=path.name,
+            sha256=sha256.strip().lower(),
+            path=_resolved(path),
+            table="econ.irs_soi",
+            status="skipped",
+            rows=0,
+            reason="duplicate",
+            actor=actor,
+        )
+        return 0
+    return load_file(
+        con,
+        path,
+        table="econ.irs_soi",
+        source_id="irs_soi",
+        release=path.name,
+        sha256=sha256,
+        actor=actor,
+        members=chosen,
+    )
+
+
+def _dispatch(con, item: _Item, path: Path, *, actor: str, standalone: set[str]) -> int:
+    source = item.source
+    if source in _SERIES_TABLES:
+        return _load_series(
+            con,
+            path,
+            table=_SERIES_TABLES[source],
+            source_id=source,
+            sha256=item.sha256,
+            actor=actor,
+        )
+    if source == "bls_ces":
+        return _load_ces(con, path, sha256=item.sha256, actor=actor)
+    if source == "bls_oews":
+        return _load_oews(con, path, sha256=item.sha256, actor=actor)
+    if source == "qcew":
+        return load_file(
+            con,
+            path,
+            table="bls.qcew",
+            source_id="qcew",
+            release=path.name,
+            sha256=item.sha256,
+            actor=actor,
+        )
+    if source == "irs_soi":
+        return _load_irs(
+            con, path, sha256=item.sha256, actor=actor, standalone=standalone
+        )
+    if source == "ma_penetration":
+        return load_file(
+            con,
+            path,
+            table="lead_public.ma_county_penetration",
+            source_id="ma_penetration",
+            release=path.parent.name,
+            sha256=item.sha256,
+            actor=actor,
+            when=_when(path),
+        )
+    return load_file(
+        con,
+        path,
+        table=qualify(source),
+        source_id=source,
+        release=path.name,
+        sha256=item.sha256,
+        actor=actor,
+    )
+
+
+def _identity(item: _Item) -> str:
+    """Path when the file is on disk, otherwise the URL that was not fetched."""
+    return item.path or item.url
+
+
+def _remember(
+    con, item: _Item, *, status: str, reason: str, table: str, actor: str
+) -> None:
+    """Record a file that was not read, once per path.
+
+    A later run sees the same status and does not append another row.
+    An empty path keeps the URL, so thirty QCEW 404s do not collapse
+    into one row.
+    """
+    stored = _identity(item)
+    prior, why = latest(con, source_id=item.source, path=stored)
+    if prior == status and why == reason:
+        return
+    record(
+        con,
+        source_id=item.source,
+        release="",
+        sha256=item.sha256,
+        path=stored,
+        table=table,
+        status=status,
+        rows=0,
+        reason=reason,
+        actor=actor,
+    )
+
+
+def load_strand5(con, root: Path | str, *, actor: str) -> tuple[LoadResult, ...]:
+    """Load ``files.tsv`` under *root*. Return one result per inventory row."""
+    ensure_log(con)
+    base = Path(root)
+    items = _inventory(base)
+    standalone = {
+        Path(item.path).name.casefold()
+        for item in items
+        if item.source == "irs_soi"
+        and item.status == "ok"
+        and item.path
+        and Path(item.path).suffix.casefold() != ".zip"
+    }
+    results: list[LoadResult] = _Results()
+    for item in items:
+        if item.source == "cms_mds_frequency":
+            _remember(
+                con,
+                item,
+                status="skipped",
+                reason="covered_by_manifest",
+                table="lead_public.cms_facility_level_minimum_data_set_frequency",
+                actor=actor,
+            )
+            results.append(
+                LoadResult(
+                    item.source,
+                    "",
+                    _identity(item),
+                    item.sha256,
+                    "skipped",
+                    0,
+                    "lead_public.cms_facility_level_minimum_data_set_frequency",
+                    "covered_by_manifest",
+                )
+            )
+            continue
+        if item.status in _ABSENT:
+            _remember(
+                con,
+                item,
+                status="absent",
+                reason=_ABSENT[item.status],
+                table="",
+                actor=actor,
+            )
+            results.append(
+                LoadResult(
+                    item.source,
+                    "",
+                    _identity(item),
+                    item.sha256,
+                    "absent",
+                    0,
+                    "",
+                    _ABSENT[item.status],
+                )
+            )
+            continue
+        if item.status != "ok":
+            _remember(
+                con,
+                item,
+                status="skipped",
+                reason=item.status,
+                table="",
+                actor=actor,
+            )
+            results.append(
+                LoadResult(
+                    item.source,
+                    "",
+                    _identity(item),
+                    item.sha256,
+                    "skipped",
+                    0,
+                    "",
+                    item.status,
+                )
+            )
+            continue
+        if not item.path:
+            _remember(
+                con,
+                item,
+                status="missing",
+                reason="file is missing",
+                table="",
+                actor=actor,
+            )
+            results.append(
+                LoadResult(
+                    item.source,
+                    "",
+                    "",
+                    item.sha256,
+                    "missing",
+                    0,
+                    "",
+                    "file is missing",
+                )
+            )
+            continue
+        path = _locate(base, item.path)
+        stored = _resolved(path) if path.exists() else str(path)
+        if item.sha256 and already_loaded(
+            con, source_id=item.source, path=stored, sha256=item.sha256
+        ):
+            results.append(
+                LoadResult(
+                    item.source,
+                    path.name,
+                    item.path,
+                    item.sha256,
+                    "skipped",
+                    0,
+                    "",
+                    "already_loaded",
+                )
+            )
+            continue
+        if not path.exists():
+            results.append(
+                LoadResult(
+                    item.source,
+                    "",
+                    item.path,
+                    item.sha256,
+                    "missing",
+                    0,
+                    "",
+                    "file is missing",
+                )
+            )
+            continue
+        try:
+            inserted = _dispatch(con, item, path, actor=actor, standalone=standalone)
+        except HashMismatch:
+            results.append(
+                LoadResult(
+                    item.source,
+                    path.name,
+                    item.path,
+                    item.sha256,
+                    "mismatch",
+                    0,
+                    "",
+                    "sha256 mismatch",
+                )
+            )
+            continue
+        except Exception as exc:
+            results.append(
+                LoadResult(
+                    item.source,
+                    path.name,
+                    item.path,
+                    item.sha256,
+                    "unreadable",
+                    0,
+                    "",
+                    str(exc).split("\n", 1)[0][:400],
+                )
+            )
+            continue
+        status, reason = "loaded", ""
+        if inserted == 0:
+            row = con.execute(
+                """
+                SELECT status, reason FROM lead_public.load_log
+                WHERE source_id = ? AND path = ?
+                ORDER BY rowid DESC LIMIT 1
+                """,
+                [item.source, stored],
+            ).fetchone()
+            if row is not None and row[0] != "loaded":
+                status, reason = str(row[0]), str(row[1] or "")
+        results.append(
+            LoadResult(
+                item.source,
+                path.name,
+                item.path,
+                item.sha256,
+                status,
+                inserted,
+                "",
+                reason,
+            )
+        )
+    return tuple(results)
diff --git a/src/lead/review/__init__.py b/src/lead/review/__init__.py
new file mode 100644
index 0000000..20745ea
--- /dev/null
+++ b/src/lead/review/__init__.py
@@ -0,0 +1,21 @@
+"""Human gold labels for the LEAD screening calibration.
+
+The labels live in their own sqlite file. They are not ``screen:`` tags
+and they are not ``stage:`` tags, so a label does not move an item
+through the PRISMA flow and does not start a production screen.
+"""
+
+from __future__ import annotations
+
+from lead.review.codes import DECISIONS, GATE, PASSES, PROJECTS, STAGES
+from lead.review.store import Label, Store
+
+__all__ = [
+    "DECISIONS",
+    "GATE",
+    "PASSES",
+    "PROJECTS",
+    "STAGES",
+    "Label",
+    "Store",
+]
diff --git a/src/lead/review/codes.py b/src/lead/review/codes.py
new file mode 100644
index 0000000..d38552c
--- /dev/null
+++ b/src/lead/review/codes.py
@@ -0,0 +1,49 @@
+"""The five LEAD strands and the decision vocabulary the gate already uses."""
+
+from __future__ import annotations
+
+import re
+
+PROJECTS = (
+    "lead-history",
+    "lead-limits",
+    "lead-scan",
+    "lead-precursors",
+    "lead-external",
+)
+DECISIONS = ("include", "exclude", "uncertain")
+STAGES = ("title_abstract", "full_text")
+PASSES = (1, 2)
+
+# Same sizes prisma.calibrate.sized enforces. A test pins the two.
+GATE = {"title_abstract": 200, "full_text": 50}
+
+_CODE = re.compile(r"`([a-z0-9-]+)`")
+
+
+def reason_codes(project: str) -> tuple[str, ...]:
+    """Exclusion codes from that strand's codebook, in codebook order."""
+    from prisma.project import STARTERS
+
+    if project not in PROJECTS:
+        raise ValueError(project)
+    text = STARTERS[project][2]
+    return tuple(dict.fromkeys(_CODE.findall(text)))
+
+
+def criteria(project: str) -> str:
+    """The strand's inclusion criteria, as the anchor note would show them."""
+    from prisma.project import STARTERS
+
+    if project not in PROJECTS:
+        raise ValueError(project)
+    return STARTERS[project][0]
+
+
+def reasons(project: str) -> str:
+    """The strand's exclusion codebook."""
+    from prisma.project import STARTERS
+
+    if project not in PROJECTS:
+        raise ValueError(project)
+    return STARTERS[project][2]
diff --git a/src/lead/review/library.py b/src/lead/review/library.py
new file mode 100644
index 0000000..7e5e8bb
--- /dev/null
+++ b/src/lead/review/library.py
@@ -0,0 +1,146 @@
+"""Read bibliographic items out of a Zotero sqlite. Never writes it."""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from pathlib import Path
+
+from zot.db import FIELD_MAP
+
+_SKIP_TYPES = ("attachment", "note", "annotation")
+
+
+@dataclass(frozen=True)
+class Item:
+    key: str
+    title: str
+    authors: tuple[str, ...]
+    year: str
+    doi: str
+    abstract: str
+    attachment_paths: tuple[Path, ...]
+
+
+def connect(path: str | Path) -> sqlite3.Connection:
+    """Open the library read-only. A write on this connection fails."""
+    con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    con.row_factory = sqlite3.Row
+    return con
+
+
+def project_keys(con: sqlite3.Connection, project: str) -> tuple[str, ...]:
+    """Item keys tagged ``project:``, in key order.
+
+    Attachments, notes, and annotations are not records to label.
+    """
+    rows = con.execute(
+        """SELECT i.key
+           FROM items i
+           JOIN itemTypes ty ON i.itemTypeID = ty.itemTypeID
+           JOIN itemTags it ON i.itemID = it.itemID
+           JOIN tags t ON it.tagID = t.tagID
+           WHERE t.name = ?
+             AND ty.typeName NOT IN (?, ?, ?)
+           ORDER BY i.key""",
+        (f"project:{project}", *_SKIP_TYPES),
+    ).fetchall()
+    return tuple(row["key"] for row in rows)
+
+
+def next_unlabeled(keys: tuple[str, ...] | list[str], labeled: set[str]) -> str | None:
+    """First key that has no current label. Order is the library order."""
+    for key in keys:
+        if key not in labeled:
+            return key
+    return None
+
+
+def pass_two_keys(
+    keys: tuple[str, ...] | list[str], pass_one: set[str], pass_two: set[str]
+) -> tuple[str, ...]:
+    """Pass 2 sees only items pass 1 already labeled, still in key order."""
+    return tuple(key for key in keys if key in pass_one and key not in pass_two)
+
+
+def load_item(con: sqlite3.Connection, key: str, storage_dir: Path) -> Item:
+    """One item's title, authors, year, DOI, abstract, and attachment paths."""
+    row = con.execute("SELECT itemID FROM items WHERE key = ?", (key,)).fetchone()
+    if row is None:
+        raise KeyError(key)
+    item_id = row["itemID"]
+    fields = _fields(con, item_id)
+    authors = _authors(con, item_id)
+    paths = _attachments(con, item_id, storage_dir)
+    return Item(
+        key=key,
+        title=fields.get("title", ""),
+        authors=authors,
+        year=_year(fields.get("date", "")),
+        doi=fields.get("DOI", ""),
+        abstract=fields.get("abstractNote", ""),
+        attachment_paths=paths,
+    )
+
+
+def _fields(con: sqlite3.Connection, item_id: int) -> dict[str, str]:
+    wanted = {
+        FIELD_MAP["title"]: "title",
+        FIELD_MAP["abstractNote"]: "abstractNote",
+        FIELD_MAP["date"]: "date",
+        FIELD_MAP["DOI"]: "DOI",
+    }
+    rows = con.execute(
+        """SELECT id.fieldID, idv.value
+           FROM itemData id
+           JOIN itemDataValues idv ON id.valueID = idv.valueID
+           WHERE id.itemID = ?""",
+        (item_id,),
+    ).fetchall()
+    return {
+        wanted[row["fieldID"]]: row["value"] for row in rows if row["fieldID"] in wanted
+    }
+
+
+def _authors(con: sqlite3.Connection, item_id: int) -> tuple[str, ...]:
+    rows = con.execute(
+        """SELECT c.firstName, c.lastName
+           FROM itemCreators ic
+           JOIN creators c ON ic.creatorID = c.creatorID
+           WHERE ic.itemID = ?
+           ORDER BY ic.orderIndex""",
+        (item_id,),
+    ).fetchall()
+    names = []
+    for row in rows:
+        name = " ".join(part for part in (row["firstName"], row["lastName"]) if part)
+        if name:
+            names.append(name)
+    return tuple(names)
+
+
+def _attachments(
+    con: sqlite3.Connection, item_id: int, storage_dir: Path
+) -> tuple[Path, ...]:
+    from prisma.export import _resolve_attachment
+
+    rows = con.execute(
+        """SELECT i.key, ia.path
+           FROM itemAttachments ia
+           JOIN items i ON ia.itemID = i.itemID
+           WHERE ia.parentItemID = ?""",
+        (item_id,),
+    ).fetchall()
+    found: list[Path] = []
+    for row in rows:
+        path = _resolve_attachment(storage_dir, row["key"], row["path"])
+        if path is not None:
+            found.append(path)
+    return tuple(found)
+
+
+def _year(date: str) -> str:
+    for token in date.replace("-", " ").split():
+        if len(token) == 4 and token.isdigit():
+            return token
+    return ""
diff --git a/src/lead/review/store.py b/src/lead/review/store.py
new file mode 100644
index 0000000..7e70c32
--- /dev/null
+++ b/src/lead/review/store.py
@@ -0,0 +1,154 @@
+"""Append-only gold labels. The latest row for a key is the current one."""
+
+from __future__ import annotations
+
+import sqlite3
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+
+from lead.review.codes import DECISIONS, PASSES, PROJECTS, STAGES, reason_codes
+
+_DDL = """
+CREATE TABLE IF NOT EXISTS label (
+    id INTEGER PRIMARY KEY,
+    zotero_key TEXT NOT NULL,
+    project TEXT NOT NULL,
+    stage TEXT NOT NULL,
+    pass INTEGER NOT NULL,
+    decision TEXT NOT NULL,
+    reason TEXT NOT NULL,
+    rationale TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    created_at TEXT NOT NULL
+);
+"""
+
+
+@dataclass(frozen=True)
+class Label:
+    zotero_key: str
+    project: str
+    stage: str
+    pass_n: int
+    decision: str
+    reason: str
+    rationale: str
+    actor: str
+    created_at: str
+
+
+class Store:
+    """One sqlite file of human labels. Opens per call and commits."""
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        con = self._open()
+        try:
+            con.execute(_DDL)
+            con.commit()
+        finally:
+            con.close()
+
+    def _open(self) -> sqlite3.Connection:
+        con = sqlite3.connect(self.path)
+        con.row_factory = sqlite3.Row
+        return con
+
+    def record(
+        self,
+        *,
+        zotero_key: str,
+        project: str,
+        stage: str,
+        pass_n: int,
+        decision: str,
+        reason: str = "",
+        rationale: str,
+        actor: str,
+    ) -> None:
+        """Append one label. A second label for the same key supersedes it."""
+        key = zotero_key.strip()
+        if not key:
+            raise ValueError("zotero key is empty")
+        if project not in PROJECTS:
+            raise ValueError(project)
+        if stage not in STAGES:
+            raise ValueError(stage)
+        if pass_n not in PASSES:
+            raise ValueError(pass_n)
+        if decision not in DECISIONS:
+            raise ValueError(decision)
+        why = reason.strip()
+        if decision == "exclude":
+            if why not in reason_codes(project):
+                raise ValueError(f"{why or '(blank)'} is not in the {project} codebook")
+        elif why:
+            raise ValueError("a reason is recorded only on exclude")
+        text = rationale.strip()
+        if not text:
+            raise ValueError("rationale is empty")
+        who = actor.strip() or "reviewer"
+        stamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+        con = self._open()
+        try:
+            con.execute(
+                """INSERT INTO label (
+                       zotero_key, project, stage, pass, decision, reason,
+                       rationale, actor, created_at
+                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                (key, project, stage, pass_n, decision, why, text, who, stamp),
+            )
+            con.commit()
+        finally:
+            con.close()
+
+    def latest(self, project: str, stage: str, pass_n: int) -> dict[str, str]:
+        """Current decision per zotero key. A later row replaces an earlier one."""
+        con = self._open()
+        try:
+            rows = con.execute(
+                """SELECT zotero_key, decision FROM label
+                   WHERE project = ? AND stage = ? AND pass = ?
+                   ORDER BY id""",
+                (project, stage, pass_n),
+            ).fetchall()
+        finally:
+            con.close()
+        out: dict[str, str] = {}
+        for row in rows:
+            out[row["zotero_key"]] = row["decision"]
+        return out
+
+    def count(self, project: str, stage: str, pass_n: int) -> int:
+        return len(self.latest(project, stage, pass_n))
+
+    def history(self, project: str, stage: str, pass_n: int) -> tuple[Label, ...]:
+        """Every row, oldest first. The page shows the tail."""
+        con = self._open()
+        try:
+            rows = con.execute(
+                """SELECT zotero_key, project, stage, pass, decision, reason,
+                          rationale, actor, created_at
+                   FROM label
+                   WHERE project = ? AND stage = ? AND pass = ?
+                   ORDER BY id""",
+                (project, stage, pass_n),
+            ).fetchall()
+        finally:
+            con.close()
+        return tuple(
+            Label(
+                zotero_key=row["zotero_key"],
+                project=row["project"],
+                stage=row["stage"],
+                pass_n=row["pass"],
+                decision=row["decision"],
+                reason=row["reason"],
+                rationale=row["rationale"],
+                actor=row["actor"],
+                created_at=row["created_at"],
+            )
+            for row in rows
+        )
diff --git a/stack.toml b/stack.toml
index 0747941..4caf6d7 100644
--- a/stack.toml
+++ b/stack.toml
@@ -24,7 +24,7 @@ subdomains = [
     "dashboard", "docs", "git", "ci", "notebooks", "zotero",
     "webdav", "api", "nessie", "trino", "polaris",
     "grafana", "prometheus", "tempo", "loki",
-    "s3", "s3console", "llm", "bcda",
+    "s3", "s3console", "llm", "bcda", "lead",
 ]
 
 [services]
diff --git a/tests/bls/test_oews.py b/tests/bls/test_oews.py
index 8f9dd2e..b352875 100644
--- a/tests/bls/test_oews.py
+++ b/tests/bls/test_oews.py
@@ -247,7 +247,7 @@ NEW_ROWS = [
 
 class TestYearsAndIds:
     def test_years_and_zip_urls(self):
-        assert list(YEARS) == list(range(2013, 2025))
+        assert list(YEARS) == list(range(2013, 2026))
         assert zip_url(2023) == "https://www.bls.gov/oes/special-requests/oesm23nat.zip"
         assert zip_url(2013).endswith("oesm13nat.zip")
 
diff --git a/tests/lead/test_explore.py b/tests/lead/test_explore.py
new file mode 100644
index 0000000..10b50f6
--- /dev/null
+++ b/tests/lead/test_explore.py
@@ -0,0 +1,359 @@
+"""The public-data explorer reads a mirror and does not write one."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import duckdb
+import pytest
+
+from lead.explore import catalog, guard, writeup
+from lead.explore.catalog import HEAVY_ROWS, NeedsFilter, QueryTimeout
+from lead.load.frame import ensure_log, record
+
+
+def _mirror(tmp_path: Path) -> duckdb.DuckDBPyConnection:
+    con = duckdb.connect(str(tmp_path / "public.duckdb"))
+    ensure_log(con)
+    record(
+        con,
+        source_id="cms_demo",
+        release="2024",
+        sha256="a" * 64,
+        path="/data/cms_demo.csv",
+        table="lead_public.cms_demo",
+        status="loaded",
+        rows=2,
+        reason="",
+        actor="stack",
+    )
+    record(
+        con,
+        source_id="qcew",
+        release="2020",
+        sha256="b" * 64,
+        path="/data/2020_qcew.zip",
+        table="bls.qcew",
+        status="loaded",
+        rows=HEAVY_ROWS,
+        reason="",
+        actor="stack",
+    )
+    record(
+        con,
+        source_id="bls_ces",
+        release="ces.csv",
+        sha256="c" * 64,
+        path="/data/ces.csv",
+        table="bls.ces",
+        status="loaded",
+        rows=2,
+        reason="",
+        actor="stack",
+    )
+    record(
+        con,
+        source_id="fred",
+        release="",
+        sha256="",
+        path="/data/fred.csv",
+        table="",
+        status="absent",
+        rows=0,
+        reason="unreachable",
+        actor="stack",
+    )
+    record(
+        con,
+        source_id="cms_note",
+        release="note",
+        sha256="d" * 64,
+        path="/data/note.txt",
+        table="lead_public.cms_note",
+        status="loaded",
+        rows=99,
+        reason="",
+        actor="stack",
+    )
+    record(
+        con,
+        source_id="cms_note",
+        release="note",
+        sha256="d" * 64,
+        path="/data/note.txt",
+        table="lead_public.cms_note",
+        status="skipped",
+        rows=0,
+        reason="kind:note",
+        actor="stack",
+    )
+    con.execute("CREATE SCHEMA IF NOT EXISTS bls")
+    con.execute(
+        """
+        CREATE TABLE lead_public.cms_demo (
+            geo_fips VARCHAR, spine_year INTEGER, agi VARCHAR, geo_zip VARCHAR
+        )
+        """
+    )
+    con.execute(
+        """
+        INSERT INTO lead_public.cms_demo VALUES
+            ('01001', 2010, '5', NULL),
+            ('01003', 2011, '9', '30301')
+        """
+    )
+    con.execute(
+        """
+        CREATE TABLE bls.qcew (
+            geo_fips VARCHAR, spine_year INTEGER, series_id VARCHAR, month3_emplvl VARCHAR
+        )
+        """
+    )
+    con.execute(
+        """
+        INSERT INTO bls.qcew VALUES
+            ('01001', 2020, 'X', '10'),
+            ('01001', 2021, 'X', '11'),
+            ('01003', 2020, 'Y', '9')
+        """
+    )
+    con.execute(
+        """
+        CREATE TABLE bls.ces (
+            series_id VARCHAR, year INTEGER, month INTEGER, value DOUBLE
+        )
+        """
+    )
+    con.execute(
+        """
+        INSERT INTO bls.ces VALUES
+            ('CES1', 2020, 1, 10),
+            ('CES1', 2022, 1, 14)
+        """
+    )
+    return con
+
+
+def _cards(con):
+    return catalog.tables(con)
+
+
+def _card(con, name):
+    return catalog.card_for(_cards(con), name)
+
+
+def test_latest_file_wins_and_its_rows_are_not_double_counted(tmp_path):
+    con = _mirror(tmp_path)
+    files = catalog.latest_files(con)
+    note = next(row for row in files if row.source_id == "cms_note")
+    assert note.status == "skipped"
+    assert note.row_count == 0
+    loaded = [row for row in files if row.status == "loaded"]
+    assert {row.source_id for row in loaded} == {"cms_demo", "qcew", "bls_ces"}
+    assert sum(row.row_count for row in loaded) == 2 + HEAVY_ROWS + 2
+
+
+def test_a_heavy_table_is_not_aggregated_without_a_year(tmp_path):
+    con = _mirror(tmp_path)
+    qcew = _card(con, "bls.qcew")
+    assert qcew.heavy
+    with pytest.raises(NeedsFilter):
+        catalog.count_rows(con, qcew, geo=("geo_fips", "01001"))
+    with pytest.raises(NeedsFilter):
+        catalog.sample(con, qcew, geo=("geo_fips", "01001"))
+    with pytest.raises(NeedsFilter):
+        catalog.profile(con, qcew)
+    got = catalog.sample(con, qcew, geo=("geo_fips", "01001"), year=2020)
+    assert [row["month3_emplvl"] for row in got] == ["10"]
+    assert catalog.count_rows(con, qcew, geo=("geo_fips", "01001"), year=2020) == 1
+
+
+def test_a_quoted_value_is_a_parameter(tmp_path):
+    con = _mirror(tmp_path)
+    demo = _card(con, "lead_public.cms_demo")
+    got = catalog.sample(con, demo, geo=("geo_fips", "01001' OR '1'='1"))
+    assert got == []
+    plain = catalog.sample(con, demo, geo=("geo_fips", "01001"))
+    assert [row["agi"] for row in plain] == ["5"]
+
+
+def test_an_unfiltered_sample_of_a_heavy_table_is_capped(tmp_path):
+    con = _mirror(tmp_path)
+    got = catalog.sample(con, _card(con, "bls.qcew"), limit=2)
+    assert len(got) == 2
+
+
+def test_series_span_is_last_minus_first_in_year_order(tmp_path):
+    con = _mirror(tmp_path)
+    ces = _card(con, "bls.ces")
+    points = catalog.series_points(
+        con, ces, series_column="series_id", series_id="CES1", value_column="value"
+    )
+    got = catalog.span(points, year_key="year", value_key="value")
+    assert (got.first_year, got.last_year, got.change) == (2020, 2022, 4)
+
+
+def test_place_summary_skips_a_heavy_scan_until_a_year_is_set(tmp_path):
+    con = _mirror(tmp_path)
+    early = catalog.place_summary(
+        con, _cards(con), geo_column="geo_fips", geo_value="01001"
+    )
+    by_table = {row["table"]: row for row in early}
+    assert by_table["bls.qcew"]["rows"] is None
+    assert by_table["bls.qcew"]["note"] == "needs a year"
+    assert by_table["lead_public.cms_demo"]["rows"] == 1
+    later = catalog.place_summary(
+        con, _cards(con), geo_column="geo_fips", geo_value="01001", year=2020
+    )
+    assert {row["table"]: row["rows"] for row in later}["bls.qcew"] == 1
+
+
+def test_sql_is_one_capped_read():
+    wrapped = guard.prepare("SELECT * FROM bls.qcew")
+    assert wrapped.endswith("LIMIT 2000")
+    assert "AS _lead_q" in wrapped
+    assert (
+        guard.prepare("DESCRIBE lead_public.load_log")
+        == "DESCRIBE lead_public.load_log"
+    )
+    with pytest.raises(guard.SqlRejected):
+        guard.prepare("SELECT 1; SELECT 2")
+    with pytest.raises(guard.SqlRejected):
+        guard.prepare("DROP TABLE bls.qcew")
+    with pytest.raises(guard.SqlRejected):
+        guard.prepare("EXPLAIN ANALYZE SELECT 1")
+
+
+def test_a_heavy_table_without_where_is_refused(tmp_path):
+    con = _mirror(tmp_path)
+    cards = _cards(con)
+    with pytest.raises(NeedsFilter):
+        catalog.run_select(con, "SELECT * FROM bls.qcew", cards)
+    headers, rows = catalog.run_select(
+        con, "SELECT geo_fips FROM bls.qcew WHERE spine_year = 2020", cards, cap=10
+    )
+    assert headers == ["geo_fips"]
+    assert len(rows) == 2
+
+
+def test_a_long_scan_is_interrupted():
+    con = duckdb.connect()
+    with pytest.raises(QueryTimeout):
+        catalog.bounded_execute(
+            con, "SELECT count(*) FROM range(1000000000000)", seconds=0.3
+        )
+
+
+def test_open_mirror_does_not_write(tmp_path):
+    con = _mirror(tmp_path)
+    path = tmp_path / "public.duckdb"
+    con.close()
+    ro = catalog.open_mirror(path)
+    with pytest.raises(duckdb.Error, match="(?i)read-only"):
+        ro.execute("CREATE TABLE lead_public.nope (a VARCHAR)")
+    ro.close()
+
+
+def test_the_writeup_counts_the_log_and_states_the_contract(tmp_path):
+    con = _mirror(tmp_path)
+    sections = writeup.build(con)
+    text = "\n".join(
+        part for section in sections for part in (section.heading, *section.body)
+    )
+    headings = [section.heading for section in sections]
+    assert headings == [
+        "Registration",
+        "The mirror",
+        "What the load recorded",
+        "Largest tables",
+        "Families",
+        "How a row is keyed",
+        "Geography and time",
+        "Gaps",
+        "What this file is not",
+        "Exploring it",
+    ]
+    assert "10.17605/OSF.IO/8WQ3M" in text
+    assert "https://osf.io/8wq3m/" in text
+    assert "local DuckDB mirror" in text
+    assert "read-only" in text
+    assert "does not download" in text
+    assert "Unity Catalog" in text
+    assert "evidence ledger" in text
+    assert "geo_zip" in text and "geo_zcta" in text
+    assert "000" in text
+    assert "time_key" in text
+    assert "month 13" in text
+    assert "beneficiary" in text
+    assert "FRED is absent" in text
+    assert "econ.fred" in text
+    recorded = next(
+        section for section in sections if section.heading == "What the load recorded"
+    )
+    assert "3 files loaded" in recorded.body[0]
+    assert f"{2 + HEAVY_ROWS + 2:,} rows recorded" in recorded.body[0]
+    assert ("loaded", 3) in recorded.rows
+    assert ("absent", 1) in recorded.rows
+    assert ("skipped", 1) in recorded.rows
+    families = next(section for section in sections if section.heading == "Families")
+    by_name = {row[0]: row for row in families.rows}
+    assert by_name["Bureau of Labor Statistics"][1] == 2
+    assert "QCEW" in by_name["Bureau of Labor Statistics"][4]
+    assert by_name["CMS"][1] == 1
+    largest = next(
+        section for section in sections if section.heading == "Largest tables"
+    )
+    assert largest.rows[0][0] == "bls.qcew"
+    assert largest.rows[0][3] == "yes"
+    gaps = next(section for section in sections if section.heading == "Gaps")
+    assert ("fred", "fred.csv", "unreachable") in gaps.rows
+    spine = next(
+        section for section in sections if section.heading == "How a row is keyed"
+    )
+    fips = next(row for row in spine.rows if row[0] == "geo_fips")
+    assert "000" in fips[1]
+
+
+def test_writeup_page_renders_the_recorded_count(tmp_path, monkeypatch):
+    con = _mirror(tmp_path)
+    path = str(tmp_path / "public.duckdb")
+    con.close()
+    monkeypatch.setenv("LEAD_PUBLIC_DB", path)
+    from streamlit.testing.v1 import AppTest
+
+    at = AppTest.from_string(
+        "from lead.explore.views import writeup\nwriteup()\n",
+        default_timeout=60,
+    )
+    at.run()
+    assert not at.exception
+    text = "\n".join(item.value for item in at.markdown)
+    assert "3 files loaded" in text
+    assert "FRED is absent" in text
+
+
+def test_explore_command_points_streamlit_at_the_mirror(tmp_path, monkeypatch):
+    db = tmp_path / "public.duckdb"
+    db.write_bytes(b"")
+    seen: dict = {}
+
+    def fake_call(cmd, env=None, **_kwargs):
+        seen["cmd"] = cmd
+        seen["env"] = env
+        return 0
+
+    monkeypatch.setattr("subprocess.call", fake_call)
+    from typer.testing import CliRunner
+
+    from cli.lead import app
+
+    result = CliRunner().invoke(
+        app, ["explore", "--db", str(db), "--port", "8765", "--no-headless"]
+    )
+    assert result.exit_code == 0
+    assert "http://localhost:8765" in result.output
+    assert seen["env"]["LEAD_PUBLIC_DB"] == str(db)
+    cmd = seen["cmd"]
+    assert cmd[1:3] == ["-m", "lead.explore"]
+    assert cmd[cmd.index("--server.port") + 1] == "8765"
+    assert cmd[cmd.index("--server.headless") + 1] == "false"
diff --git a/tests/lead/test_load_public.py b/tests/lead/test_load_public.py
new file mode 100644
index 0000000..a1d249b
--- /dev/null
+++ b/tests/lead/test_load_public.py
@@ -0,0 +1,1591 @@
+"""Public files load into a local DuckDB mirror. Nothing is downloaded."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import zipfile
+from pathlib import Path
+
+import duckdb
+import openpyxl
+import pytest
+from typer.testing import CliRunner
+
+from cli import app
+from lead.load import public as public_load
+from lead.load.errors import HashMismatch
+from lead.load.frame import LOAD_LOG_DDL
+from lead.load.public import _NeedsSerial, load_file, load_manifest_files
+
+runner = CliRunner()
+
+_HEADER = (
+    "source_id,title,publisher,kind,url,api_endpoint,landing_page,release,"
+    "fetched_utc,status,path,sha256,rows,size_bytes,license_terms,lead_use"
+)
+
+
+def _sha(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _manifest(root: Path, lines: list[str]) -> Path:
+    path = root / "manifest.csv"
+    path.write_text(_HEADER + "\n" + "\n".join(lines) + "\n")
+    return path
+
+
+def _row(
+    source: str, release: str, rel: str, digest: str, *, kind: str = "data"
+) -> str:
+    return (
+        f"{source},Title,Pub,{kind},http://example.invalid/not-fetched,,,{release},"
+        f"2026-11-01,ok,{rel},{digest},1,1,public,benchmark"
+    )
+
+
+def test_a_csv_lands_on_the_spine_and_a_second_load_inserts_nothing(tmp_path):
+    src = tmp_path / "rows.csv"
+    src.write_text("FIPS,Year,Quarter,n\n01001,2024,1,4\n01000,2024,1,9\n")
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="lead_public.demo",
+                source_id="demo",
+                release="2024",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 2
+        )
+        assert (
+            load_file(
+                con,
+                src,
+                table="lead_public.demo",
+                source_id="demo",
+                release="2024",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 0
+        )
+        rows = con.execute(
+            """
+            SELECT geo_fips, spine_year, spine_quarter, time_key, n
+            FROM lead_public.demo
+            ORDER BY geo_fips NULLS FIRST
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert rows == [
+        (None, 2024, 1, "2024Q1", "9"),
+        ("01001", 2024, 1, "2024Q1", "4"),
+    ]
+
+
+def test_a_month_fills_the_quarter(tmp_path):
+    src = tmp_path / "m.csv"
+    src.write_text("fips,year,month,n\n01001,2024,2,1\n")
+    con = duckdb.connect()
+    try:
+        load_file(
+            con,
+            src,
+            table="lead_public.months",
+            source_id="months",
+            release="2024",
+            sha256=_sha(src),
+            actor="tester",
+        )
+        row = con.execute(
+            "SELECT spine_month, spine_quarter, time_key FROM lead_public.months"
+        ).fetchone()
+    finally:
+        con.close()
+    assert row == (2, 1, "2024Q1")
+
+
+def test_a_bls_period_fills_the_month_and_a_blank_value_stays_text(tmp_path):
+    src = tmp_path / "p.csv"
+    src.write_text(
+        "series_id,year,period,value\nWPS512,2024,M02,1.5\nWPS512,2024,M13,.\n"
+    )
+    con = duckdb.connect()
+    try:
+        load_file(
+            con,
+            src,
+            table="lead_public.periods",
+            source_id="periods",
+            release="2024",
+            sha256=_sha(src),
+            actor="tester",
+        )
+        rows = con.execute(
+            """
+            SELECT period, spine_month, spine_quarter, time_key, value
+            FROM lead_public.periods
+            ORDER BY period
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert rows == [
+        ("M02", 2, 1, "2024Q1", "1.5"),
+        ("M13", None, None, None, "."),
+    ]
+
+
+def test_a_hash_mismatch_inserts_nothing(tmp_path):
+    src = tmp_path / "rows.csv"
+    src.write_text("fips,year,n\n01001,2024,1\n")
+    con = duckdb.connect()
+    try:
+        with pytest.raises(HashMismatch):
+            load_file(
+                con,
+                src,
+                table="lead_public.demo",
+                source_id="demo",
+                release="2024",
+                sha256="0" * 64,
+                actor="tester",
+            )
+        found = con.execute(
+            """
+            SELECT COUNT(*) FROM information_schema.tables
+            WHERE table_schema = 'lead_public' AND table_name = 'demo'
+            """
+        ).fetchone()[0]
+        logged = con.execute("SELECT COUNT(*) FROM lead_public.load_log").fetchone()[0]
+    finally:
+        con.close()
+    assert found == 0
+    assert logged == 0
+
+
+def test_html_and_a_dictionary_are_not_observation_rows(tmp_path):
+    html = tmp_path / "spec.html"
+    html.write_text("not a table")
+    missing = tmp_path / "no-dictionary.pdf"
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                html,
+                table="lead_public.spec",
+                source_id="spec",
+                release="v1",
+                sha256=_sha(html),
+                actor="tester",
+            )
+            == 0
+        )
+        manifest = _manifest(
+            tmp_path,
+            [
+                _row("spec", "v1", "spec.html", _sha(html)),
+                _row("dict_src", "v1", "no-dictionary.pdf", "abc", kind="dictionary"),
+            ],
+        )
+        results = load_manifest_files(con, manifest, root=tmp_path, actor="tester")
+    finally:
+        con.close()
+    by_source = {row.source_id: row for row in results}
+    assert by_source["spec"].status == "skipped"
+    assert by_source["spec"].reason == "not_tabular"
+    assert by_source["dict_src"].status == "skipped"
+    assert by_source["dict_src"].reason == "kind:dictionary"
+    assert not missing.exists()
+
+
+def test_a_zip_csv_an_xlsx_and_a_bls_payload_load(tmp_path):
+    zpath = tmp_path / "pack.zip"
+    with zipfile.ZipFile(zpath, "w") as zf:
+        zf.writestr(
+            "inner/places.csv",
+            "LocationName,FIPS,Year,Data_Value\nAutauga,01001,2024,3\n",
+        )
+        zf.writestr("inner/readme.txt", "This file explains the extract.\n")
+    book = tmp_path / "book.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "county"
+    sheet.append(["ZCTA", "Year", "agi"])
+    sheet.append(["02138", 2023, 10])
+    wb.save(book)
+    payload = [
+        {
+            "status": "REQUEST_SUCCEEDED",
+            "message": [],
+            "Results": {
+                "series": [
+                    {
+                        "seriesID": "CUUR0000SAM",
+                        "data": [
+                            {"year": "2024", "period": "M01", "value": "500.1"},
+                            {"year": "2024", "period": "M13", "value": "."},
+                        ],
+                    }
+                ]
+            },
+        }
+    ]
+    blob = tmp_path / "cpi.json"
+    blob.write_text(json.dumps(payload))
+    pipe = tmp_path / "age.dat"
+    pipe.write_text("GEO_ID|B01001_E001\n0500000US01001|100\n")
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="lead_public.pack",
+                source_id="pack",
+                release="2024",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 1
+        )
+        zip_row = con.execute(
+            "SELECT geo_fips, spine_year, member, data_value FROM lead_public.pack"
+        ).fetchone()
+        assert (
+            load_file(
+                con,
+                book,
+                table="lead_public.book",
+                source_id="book",
+                release="2023",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        xlsx_row = con.execute(
+            "SELECT geo_zcta, spine_year, member, agi FROM lead_public.book"
+        ).fetchone()
+        assert (
+            load_file(
+                con,
+                blob,
+                table="lead_public.bls_cpi_medical",
+                source_id="bls_cpi_medical",
+                release="2026",
+                sha256=_sha(blob),
+                actor="tester",
+            )
+            == 2
+        )
+        cpi = con.execute(
+            """
+            SELECT series_id, spine_year, period, value, time_key
+            FROM lead_public.bls_cpi_medical
+            ORDER BY period
+            """
+        ).fetchall()
+        assert (
+            load_file(
+                con,
+                pipe,
+                table="lead_public.acs",
+                source_id="acs",
+                release="2024",
+                sha256=_sha(pipe),
+                actor="tester",
+            )
+            == 1
+        )
+        acs = con.execute("SELECT geo_id, b01001_e001 FROM lead_public.acs").fetchone()
+    finally:
+        con.close()
+    assert zip_row == ("01001", 2024, "inner/places.csv", "3")
+    assert xlsx_row == ("02138", 2023, "county", "10")
+    assert cpi == [
+        ("CUUR0000SAM", 2024, "M01", "500.1", "2024Q1"),
+        ("CUUR0000SAM", 2024, "M13", ".", None),
+    ]
+    assert acs == ("0500000US01001", "100")
+
+
+def test_awkward_public_csvs_stay_whole_rows(tmp_path):
+    """CMS and HCUP files that DuckDB's sniffer misreads still load whole.
+
+    A Windows-1252 apostrophe is not a reason to drop the row. A suppressed
+    ``'-`` cell is text, not a quote. An HCUP export wraps codes in single
+    quotes and descriptions that contain commas in double quotes.
+    """
+    coded = tmp_path / "aip.csv"
+    coded.write_bytes(
+        b'aco_id,aco_name\nA5655,"St. David\x92s Care Partners ACO, LLC"\n'
+    )
+    suppressed = tmp_path / "reach.csv"
+    suppressed.write_text(
+        "aco_id,county_name,elig_months_ad,aligned\nD0004,Danville city,'-,'-\n",
+        encoding="utf-8",
+    )
+    sas = tmp_path / "prccsr.csv"
+    sas.write_text(
+        "'ICD-10-PCS','ICD-10-PCS DESCRIPTION','PRCCSR'\n"
+        "'0016070',\"Bypass Cerebral Ventricle to Nasopharynx with "
+        "Autologous Tissue Substitute, Open Approach\",'CNS010'\n",
+        encoding="utf-8",
+    )
+    con = duckdb.connect()
+    try:
+        for path, table in (
+            (coded, "aip"),
+            (suppressed, "reach"),
+            (sas, "prccsr"),
+        ):
+            load_file(
+                con,
+                path,
+                table=table,
+                source_id=table,
+                release="2026",
+                sha256=_sha(path),
+                actor="tester",
+            )
+        name = con.execute("SELECT aco_name FROM lead_public.aip").fetchone()[0]
+        reach = con.execute(
+            "SELECT elig_months_ad, aligned FROM lead_public.reach"
+        ).fetchone()
+        procedure = con.execute(
+            """
+            SELECT icd_10_pcs, icd_10_pcs_description, prccsr
+            FROM lead_public.prccsr
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert name == "St. David\u2019s Care Partners ACO, LLC"
+    assert reach == ("'-", "'-")
+    assert procedure == (
+        "0016070",
+        "Bypass Cerebral Ventricle to Nasopharynx with Autologous Tissue "
+        "Substitute, Open Approach",
+        "CNS010",
+    )
+
+
+def test_a_quoted_newline_and_a_title_banner_stay_one_table(tmp_path):
+    report = tmp_path / "report.csv"
+    report.write_text(
+        'Title,Abstract\nOne,"line one\nline two"\nTwo,plain\n',
+        encoding="utf-8",
+    )
+    mapping = tmp_path / "mapping.csv"
+    mapping.write_bytes(
+        b'"Model title",,,\n'
+        b"note,,,\n"
+        b",,,\n"
+        b'"Diagnosis\nCode",Description,Flag,Extra\n'
+        b"A0103,Typhoid pneumonia,115,1\n"
+    )
+    con = duckdb.connect()
+    con.execute("SET preserve_insertion_order = false")
+    try:
+        load_file(
+            con,
+            report,
+            table="report",
+            source_id="report",
+            release="2026",
+            sha256=_sha(report),
+            actor="tester",
+        )
+        load_file(
+            con,
+            mapping,
+            table="mapping",
+            source_id="mapping",
+            release="2026",
+            sha256=_sha(mapping),
+            actor="tester",
+        )
+        abstracts = con.execute(
+            "SELECT title, abstract FROM lead_public.report ORDER BY title"
+        ).fetchall()
+        mapped = con.execute(
+            """
+            SELECT diagnosis_code, description, flag
+            FROM lead_public.mapping
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert abstracts == [("One", "line one\nline two"), ("Two", "plain")]
+    assert mapped == [("A0103", "Typhoid pneumonia", "115")]
+
+
+def test_a_sas_macro_beside_a_coefficient_table_is_skipped(tmp_path):
+    zpath = tmp_path / "soft.zip"
+    with zipfile.ZipFile(zpath, "w") as archive:
+        archive.writestr(
+            "V22I0ED4.TXT",
+            " %MACRO V22I0ED4(AGE=, SEX=);\n %LET X=1;\n %MEND;\n",
+        )
+        archive.writestr(
+            "C2214O5P.csv",
+            "Name,Coeff,Label\nHCC1,1.25,Diabetes\n",
+        )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="soft",
+                source_id="soft",
+                release="2026",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            "SELECT name, coeff, label, member FROM lead_public.soft"
+        ).fetchone()
+    finally:
+        con.close()
+    assert row == ("HCC1", "1.25", "Diabetes", "C2214O5P.csv")
+
+
+def test_a_parallel_reader_refusal_is_loaded_on_one_thread(tmp_path, monkeypatch):
+    src = tmp_path / "rows.csv"
+    src.write_text("fips,year,n\n01001,2024,4\n")
+    real = public_load._read_delimited
+    seen: list[bool] = []
+
+    def wrapped(con, path, *, serial=False):
+        seen.append(serial)
+        if len(seen) == 1:
+            raise _NeedsSerial()
+        return real(con, path, serial=serial)
+
+    monkeypatch.setattr(public_load, "_read_delimited", wrapped)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="demo",
+                source_id="demo",
+                release="2024",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            "SELECT geo_fips, spine_year, n FROM lead_public.demo"
+        ).fetchone()
+    finally:
+        con.close()
+    assert seen == [False, True]
+    assert row == ("01001", 2024, "4")
+
+
+def test_an_empty_sheet_does_not_drop_the_workbook(tmp_path):
+    book = tmp_path / "states.xlsx"
+    wb = openpyxl.Workbook()
+    blank = wb.active
+    blank.title = "doc"
+    sheet = wb.create_sheet("zip")
+    sheet.append(["zip", "agi"])
+    sheet.append(["30301", "7"])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="2005",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute("SELECT zip, agi FROM lead_public.irs").fetchone()
+    finally:
+        con.close()
+    assert row == ("30301", "7")
+
+
+def test_an_rxnorm_rrf_keeps_its_first_record(tmp_path):
+    zpath = tmp_path / "rxnorm.zip"
+    with zipfile.ZipFile(zpath, "w") as archive:
+        archive.writestr(
+            "Readme.txt",
+            "October 05, 2026\nThis file explains the release.\n",
+        )
+        archive.writestr(
+            "rrf/RXNCONSO.RRF",
+            "38|ENG||||||829|829|38||RXNORM|BN|38|Parlodel||N|4096|\n",
+        )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="rxnorm",
+                source_id="rxnorm",
+                release="current",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            "SELECT rxcui, tty, code, str_ FROM lead_public.rxnorm"
+        ).fetchone()
+    finally:
+        con.close()
+    assert row == ("38", "BN", "38", "Parlodel")
+
+
+def test_two_rxnorm_members_keep_published_names(tmp_path):
+    zpath = tmp_path / "rxnorm.zip"
+    rel = "|".join(
+        [
+            "",
+            "a1",
+            "b1",
+            "c1",
+            "RO",
+            "a2",
+            "b2",
+            "c2",
+            "tradename_of",
+            "r1",
+            "s1",
+            "RXNORM",
+            "l",
+            "g",
+            "d",
+            "N",
+        ]
+    )
+    with zipfile.ZipFile(zpath, "w") as archive:
+        archive.writestr(
+            "rrf/RXNCONSO.RRF",
+            "38|ENG||||||829|829|38||RXNORM|BN|38|Parlodel||N|4096|\n",
+        )
+        archive.writestr("rrf/RXNREL.RRF", rel + "|\n")
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="rxnorm",
+                source_id="rxnorm",
+                release="current",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 2
+        )
+        names = {
+            row[0] for row in con.execute("DESCRIBE lead_public.rxnorm").fetchall()
+        }
+        conso = con.execute(
+            """
+            SELECT rxcui, tty, str_
+            FROM lead_public.rxnorm
+            WHERE member = 'rrf/RXNCONSO.RRF'
+            """
+        ).fetchone()
+        related = con.execute(
+            """
+            SELECT rxcui1, rel, rxcui2
+            FROM lead_public.rxnorm
+            WHERE member = 'rrf/RXNREL.RRF'
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert "parlodel" not in names
+    assert conso == ("38", "BN", "Parlodel")
+    assert related == (None, "c1", "RO")
+
+
+def test_manifest_stacks_releases_and_keeps_going_after_a_bad_hash(tmp_path):
+    good = tmp_path / "cms_demo" / "2024"
+    good.mkdir(parents=True)
+    first = good / "a.csv"
+    first.write_text("fips,year,quarter,n\n01001,2024,1,4\n")
+    second_dir = tmp_path / "cms_demo" / "2025"
+    second_dir.mkdir()
+    second = second_dir / "b.csv"
+    second.write_text("fips,year,quarter,n\n01003,2025,2,7\n")
+    bad = tmp_path / "cms_demo" / "bad.csv"
+    bad.write_text("fips,year,n\n01001,2024,1\n")
+    manifest = _manifest(
+        tmp_path,
+        [
+            _row("cms_demo", "2024", "cms_demo/2024/a.csv", _sha(first)),
+            _row("cms_demo", "2025", "cms_demo/2025/b.csv", _sha(second)),
+            _row("cms_demo", "bad", "cms_demo/bad.csv", "f" * 64),
+            _row("cms_gap", "2024", "cms_gap/missing.csv", ""),
+        ],
+    )
+    con = duckdb.connect()
+    try:
+        results = load_manifest_files(con, manifest, root=tmp_path, actor="tester")
+        stacked = con.execute(
+            """
+            SELECT release, geo_fips, time_key
+            FROM lead_public.cms_demo
+            ORDER BY release
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    by_key = {(row.source_id, row.release): row.status for row in results}
+    assert by_key == {
+        ("cms_demo", "2024"): "loaded",
+        ("cms_demo", "2025"): "loaded",
+        ("cms_demo", "bad"): "mismatch",
+        ("cms_gap", "2024"): "gap",
+    }
+    assert stacked == [("2024", "01001", "2024Q1"), ("2025", "01003", "2025Q2")]
+
+
+def test_a_wrapped_label_row_is_not_the_pfs_header(tmp_path):
+    """PPRRVU puts a dense row of wrapped labels one line above HCPCS."""
+    src = tmp_path / "PPRRVU.csv"
+    src.write_text(
+        ",,2025 National Physician Fee Schedule Relative Value File,,,,,,,,\n"
+        ",,CPT codes and descriptions only are copyright 2024,,,,,,,,\n"
+        ",,,,,,,,,,,\n"
+        ",,,,NOT USED,,,,,PHYSICIAN,,,\n"
+        ",,,STATUS,MEDICARE,WORK,NON-FAC,NA,FACILITY,NA,MP,TOTAL\n"
+        "HCPCS,MOD,DESCRIPTION,CODE,PAYMENT,RVU,PE RVU,INDICATOR,PE RVU,INDICATOR,RVU\n"
+        "0001F,,Heart failure composite,I,,0.00,0.00,,0.00,,0.00\n",
+        encoding="utf-8",
+    )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="pfs",
+                source_id="pfs",
+                release="2025",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            "SELECT hcpcs, mod, description, code FROM lead_public.pfs"
+        ).fetchone()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'pfs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert row == ("0001F", None, "Heart failure composite", "I")
+    assert "status" not in columns
+    assert "medicare" not in columns
+
+
+def test_a_year_header_below_a_title_is_kept(tmp_path):
+    src = tmp_path / "NHE60-34.csv"
+    src.write_text(
+        "NATIONAL HEALTH EXPENDITURES BY TYPE OF SERVICE,,,,,,,,\n"
+        ",Expenditure Amount (Millions),,,,,,,\n"
+        "Type of Service,Source of Funds,1960,1961,1962,1963,1964,1965\n"
+        ",,Historical,Historical,Historical,Historical,Historical,Historical\n"
+        ",Amount in Billions,,,,,,\n"
+        "National Health Expenditures,Total Expenditures,27122,29060,31765,34558,38245,41627\n",
+        encoding="utf-8",
+    )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="nhe",
+                source_id="nhe",
+                release="2026",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            """
+            SELECT type_of_service, source_of_funds, _1960
+            FROM lead_public.nhe
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert row == ("National Health Expenditures", "Total Expenditures", "27122")
+
+
+def test_an_excel_title_row_is_not_the_header(tmp_path):
+    book = tmp_path / "ccsr.xlsx"
+    wb = openpyxl.Workbook()
+    prose = wb.active
+    prose.title = "Table_of_Contents"
+    prose.append(["This Excel file provides searchable lists of the CCSR categories"])
+    prose.append(["Table of Contents"])
+    mapping = wb.create_sheet("DX_to_CCSR_Mapping")
+    mapping.append(["Clinical Classifications Software Refined, version 2026.1"])
+    mapping.append(
+        [
+            "ICD-10-CM Code",
+            "ICD-10-CM Code Description",
+            "CCSR Category",
+            "CCSR Category Description",
+        ]
+    )
+    mapping.append(
+        [
+            "A000",
+            "Cholera due to Vibrio cholerae 01, biovar cholerae",
+            "DIG001",
+            "Intestinal infection",
+        ]
+    )
+    names = wb.create_sheet("Naming_Conventions")
+    names.append(["Naming conventions for the body-system abbreviations"])
+    names.append(["Body System", "Abbreviation"])
+    names.append(["Diseases of the blood", "BLD"])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="ccsr",
+                source_id="ccsr",
+                release="2026",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 2
+        )
+        coded = con.execute(
+            """
+            SELECT icd_10_cm_code, icd_10_cm_code_description, ccsr_category
+            FROM lead_public.ccsr
+            WHERE icd_10_cm_code = 'A000'
+            """
+        ).fetchone()
+        named = con.execute(
+            """
+            SELECT body_system, abbreviation
+            FROM lead_public.ccsr
+            WHERE abbreviation = 'BLD'
+            """
+        ).fetchone()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'ccsr'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert coded == (
+        "A000",
+        "Cholera due to Vibrio cholerae 01, biovar cholerae",
+        "DIG001",
+    )
+    assert named == ("Diseases of the blood", "BLD")
+    assert not any("searchable" in name or "provides" in name for name in columns)
+
+
+def test_an_irs_title_block_drops_the_column_index_row(tmp_path):
+    book = tmp_path / "county.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "county income"
+    sheet.append(["ALASKA"])
+    sheet.append(
+        [
+            "Individual Income Tax Returns: Selected Income Items by State and County, Tax Year 2009"
+        ]
+    )
+    sheet.append(["(Money amounts are in thousands of dollars)"])
+    sheet.append(["Source: IRS Individual Master File"])
+    sheet.append([None])
+    sheet.append(
+        [
+            "State Code",
+            "County Code",
+            "County Name",
+            "Number of returns",
+            "Adjusted gross income (AGI)",
+        ]
+    )
+    sheet.append([-1, -2, -3, -4, -5])
+    sheet.append([2, 0, "ALASKA", 293702, 17595406])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="2009",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            """
+            SELECT state_code, county_code, county_name, adjusted_gross_income_agi
+            FROM lead_public.irs
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert row == ("2", "0", "ALASKA", "17595406")
+
+
+def test_a_zip_keeps_each_members_own_header(tmp_path):
+    zpath = tmp_path / "rvu.zip"
+    with zipfile.ZipFile(zpath, "w") as archive:
+        archive.writestr(
+            "GPCI2025.csv",
+            "ADDENDUM E. GEOGRAPHIC PRACTICE COST INDICES,,,,,,\n"
+            ",,,,,,\n"
+            "Medicare Administrative Contractor (MAC),State,Locality Number,"
+            "Locality Name,Work GPCI,PE GPCI,MP GPCI\n"
+            "10112,AL,00,ALABAMA,1,0.869,0.575\n",
+        )
+        archive.writestr(
+            "OPPSCAP.csv",
+            "HCPCS,MOD,PROCSTAT,CARRIER,LOCALITY,FACILITY PRICE,NON-FACILTY PRICE\n"
+            "0633T,TC,C,01112,05,150.69,150.69\n",
+        )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="pfs",
+                source_id="pfs",
+                release="2025",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 2
+        )
+        gpci = con.execute(
+            """
+            SELECT medicare_administrative_contractor_mac, state, locality_name
+            FROM lead_public.pfs
+            WHERE member = 'GPCI2025.csv'
+            """
+        ).fetchone()
+        opps = con.execute(
+            """
+            SELECT hcpcs, mod, procstat, non_facilty_price
+            FROM lead_public.pfs
+            WHERE member = 'OPPSCAP.csv'
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert gpci == ("10112", "AL", "ALABAMA")
+    assert opps == ("0633T", "TC", "C", "150.69")
+
+
+def test_icd_fixed_width_files_keep_commas_in_the_description(tmp_path):
+    zpath = tmp_path / "icd.zip"
+    short = "Cholera due to Vibrio cholerae 01, biovar cholerae".ljust(60)
+    long = "Cholera due to Vibrio cholerae 01, biovar cholerae"
+    order = (
+        f"00002 {'A000'.ljust(7)} 1 {short}{long}\n"
+        f"00001 {'A00'.ljust(7)} 0 {'Cholera'.ljust(60)}Cholera\n"
+    )
+    with zipfile.ZipFile(zpath, "w") as archive:
+        archive.writestr(
+            "icd10cm_codes_2026.txt",
+            "A000    Cholera due to Vibrio cholerae 01, biovar cholerae\n"
+            "C441021 Unspecified malignant neoplasm of skin\n",
+        )
+        archive.writestr("icd10cm_order_2026.txt", order)
+        archive.writestr(
+            "icd10cm_codes_addenda_2026.txt",
+            "Delete:      B880    Other acariasis\n"
+            "Add:         B8801   Infestation by Demodex mites\n"
+            "Add:         C50A0   Malignant inflammatory neoplasm of unspecified breast\n"
+            "\n"
+            "    28 deletions\n",
+        )
+        archive.writestr(
+            "icd10cm_order_addenda_2026.txt",
+            "Add:         1 B8801   Infestation by Demodex mites"
+            "                                 Infestation by Demodex mites\n"
+            "Delete:      0 B880    Other acariasis"
+            "                                              Other acariasis\n",
+        )
+        archive.writestr("icd10cmCodesFile.pdf", b"%PDF-1.4 not a table")
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                zpath,
+                table="icd",
+                source_id="icd",
+                release="2026",
+                sha256=_sha(zpath),
+                actor="tester",
+            )
+            == 9
+        )
+        coded = con.execute(
+            """
+            SELECT code, description FROM lead_public.icd
+            WHERE member = 'icd10cm_codes_2026.txt'
+            ORDER BY code
+            """
+        ).fetchall()
+        ordered = con.execute(
+            """
+            SELECT order_number, code, header_flag, short_description
+            FROM lead_public.icd
+            WHERE member = 'icd10cm_order_2026.txt' AND code = 'A000'
+            """
+        ).fetchone()
+        added = con.execute(
+            """
+            SELECT action, code, description FROM lead_public.icd
+            WHERE member = 'icd10cm_codes_addenda_2026.txt'
+            ORDER BY code
+            """
+        ).fetchall()
+        revised = con.execute(
+            """
+            SELECT action, header_flag, code, short_description, long_description
+            FROM lead_public.icd
+            WHERE member = 'icd10cm_order_addenda_2026.txt' AND action = 'Add'
+            """
+        ).fetchone()
+    finally:
+        con.close()
+    assert coded == [
+        ("A000", "Cholera due to Vibrio cholerae 01, biovar cholerae"),
+        ("C441021", "Unspecified malignant neoplasm of skin"),
+    ]
+    assert ordered == (
+        "00002",
+        "A000",
+        "1",
+        "Cholera due to Vibrio cholerae 01, biovar cholerae",
+    )
+    assert added == [
+        ("Delete", "B880", "Other acariasis"),
+        ("Add", "B8801", "Infestation by Demodex mites"),
+        ("Add", "C50A0", "Malignant inflammatory neoplasm of unspecified breast"),
+    ]
+    assert revised == (
+        "Add",
+        "1",
+        "B8801",
+        "Infestation by Demodex mites",
+        "Infestation by Demodex mites",
+    )
+
+
+def test_a_group_label_stays_on_the_following_years(tmp_path):
+    """CMS program statistics put Aged above a run of year rows."""
+    book = tmp_path / "inpatient.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "MDCR INPT HOSP 1"
+    sheet.append([None, "All Medicare Inpatient Hospitals"])
+    sheet.append([None, "Calendar Years 2019-2024"])
+    sheet.append(
+        [
+            "Type of Entitlement and Calendar Year",
+            "Total Enrollees",
+            "Discharges",
+            "Payments",
+        ]
+    )
+    sheet.append(["BLANK"])
+    sheet.append(["All Beneficiaries"])
+    sheet.append(["2019", 100, 10, 5])
+    sheet.append(["2020", 110, 11, 6])
+    sheet.append(["Aged Beneficiaries"])
+    sheet.append(["2019", 80, 8, 4])
+    sheet.append(["NOTE: Persons with coinsurance are counted once."])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="cps",
+                source_id="cps",
+                release="2024",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 3
+        )
+        rows = con.execute(
+            """
+            SELECT section, type_of_entitlement_and_calendar_year, total_enrollees
+            FROM lead_public.cps
+            ORDER BY section, type_of_entitlement_and_calendar_year
+            """
+        ).fetchall()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'cps'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert rows == [
+        ("Aged Beneficiaries", "2019", "80"),
+        ("All Beneficiaries", "2019", "100"),
+        ("All Beneficiaries", "2020", "110"),
+    ]
+    assert not any(name.startswith("column") or "unnamed" in name for name in columns)
+
+
+def test_a_group_label_after_the_first_row_keeps_one_width(tmp_path):
+    """Some IRS sheets print the total, then a one-cell group, then the counties."""
+    book = tmp_path / "county.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "county"
+    sheet.append(["State", "Returns", "Income"])
+    sheet.append(["AL", 10, 20])
+    sheet.append(["Aged"])
+    sheet.append(["AL", 4, 8])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="2002",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 2
+        )
+        rows = con.execute(
+            """
+            SELECT section, state, returns
+            FROM lead_public.irs
+            ORDER BY returns
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert rows == [("", "AL", "10"), ("Aged", "AL", "4")]
+
+
+def test_a_blank_header_cell_keeps_the_code_under_it(tmp_path):
+    """1990 IRS prints CODES once and leaves the county-code cell blank."""
+    book = tmp_path / "ALABAMA90ci.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "A"
+    sheet.append(["TAX YEAR 1990"])
+    sheet.append([None, None, None, None, "NUMBER OF", "NUMBER OF", "ADJUSTED"])
+    sheet.append(
+        [None, "CODES", None, "COUNTY", "RETURNS", "EXEMPTIONS", "GROSS INCOME"]
+    )
+    sheet.append([None, 1, 1, "AUTAUGA", 12883, 31491, 348492])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="1990",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            """
+            SELECT codes, codes_2, county, number_of_returns
+            FROM lead_public.irs
+            """
+        ).fetchone()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'irs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert row == ("1", "1", "AUTAUGA", "12883")
+    assert not any(name.startswith("column") for name in columns)
+
+
+def test_a_headerless_icd_map_keeps_its_first_code(tmp_path):
+    src = tmp_path / "F2825T1N.TXT"
+    src.write_text(
+        "A0104\t92\t\nA0105\t92\t\nB377\t2\t\nB377\t6\tD\n",
+        encoding="utf-8",
+    )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="hcc",
+                source_id="hcc",
+                release="2026",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 4
+        )
+        rows = con.execute(
+            """
+            SELECT icd10, cc, duplicate_cc
+            FROM lead_public.hcc
+            ORDER BY icd10, cc
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert rows == [
+        ("A0104", "92", None),
+        ("A0105", "92", None),
+        ("B377", "2", None),
+        ("B377", "6", "D"),
+    ]
+
+
+def test_a_backtick_in_a_blank_header_is_not_a_column(tmp_path):
+    """1998 Georgia has one stray backtick past the last named field."""
+    book = tmp_path / "georgia.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "Sheet 1"
+    sheet.append(["State", "Returns", None])
+    sheet.append(["AL", 10, "`"])
+    sheet.append(["GA", 12, None])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="1998",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 2
+        )
+        rows = con.execute(
+            "SELECT state, returns FROM lead_public.irs ORDER BY state"
+        ).fetchall()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'irs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert rows == [("AL", "10"), ("GA", "12")]
+    assert not any(name.startswith("column") or name == "col" for name in columns)
+
+
+def test_an_empty_trailing_header_is_not_a_column(tmp_path):
+    src = tmp_path / "sites.csv"
+    src.write_text("site_name,site_id,\nAlpha,1,\nBeta,2,\n", encoding="utf-8")
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="sites",
+                source_id="sites",
+                release="2026",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 2
+        )
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'sites'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert "site_name" in columns
+    assert not any(name.startswith("column") or name == "col" for name in columns)
+
+
+def test_a_dropdown_sheet_is_not_a_table(tmp_path):
+    book = tmp_path / "CMS165.xlsx"
+    wb = openpyxl.Workbook()
+    dropdown = wb.active
+    dropdown.title = "DropdownOptions"
+    dropdown.append(
+        [
+            "Types of TRNs (please do not delete this sheet -- it is used to define the dropdown options.)",
+            "Measure Section",
+            "Source of Change",
+        ]
+    )
+    dropdown.append(["Header", "eCQM Title", "Annual Update"])
+    dropdown.append(["Logic", "CMS ID", "Measure Lead"])
+    notes = wb.create_sheet("TRNs")
+    notes.append(["CMS165v15 - Controlling High Blood Pressure"])
+    notes.append(
+        [
+            "Technical Release Notes",
+            "Type of TRN",
+            "Measure Section",
+            "Source of Change",
+        ]
+    )
+    notes.append(
+        ["Updated the version.", "Header", "eCQM Version Number", "Annual Update"]
+    )
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="ecqi",
+                source_id="ecqi",
+                release="v15",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            """
+            SELECT technical_release_notes, type_of_trn
+            FROM lead_public.ecqi
+            """
+        ).fetchone()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'ecqi'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert row == ("Updated the version.", "Header")
+    assert not any(
+        "do_not_delete" in name or "types_of_trns" in name for name in columns
+    )
+
+
+def test_a_suppressed_zip_row_does_not_become_the_header(tmp_path):
+    """IRS 2007 prints ``.`` for a withheld count. That row is data."""
+    src = tmp_path / "zipcode07.csv"
+    src.write_text(
+        '"state","ZIPCODE","agi_class","n1","mars2"\n'
+        '"AL",35001,1,116,.\n'
+        '"AL",35004,1,916,122\n',
+        encoding="utf-8",
+    )
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                src,
+                table="irs",
+                source_id="irs",
+                release="2007",
+                sha256=_sha(src),
+                actor="tester",
+            )
+            == 2
+        )
+        rows = con.execute(
+            """
+            SELECT state, zipcode, agi_class, n1, mars2
+            FROM lead_public.irs
+            ORDER BY zipcode
+            """
+        ).fetchall()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'irs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert rows == [
+        ("AL", "35001", "1", "116", "."),
+        ("AL", "35004", "1", "916", "122"),
+    ]
+    assert not any(
+        name in {"al", "col"} or name.startswith("column") or name.startswith("_")
+        for name in columns
+    )
+
+
+def test_a_parenthesized_index_row_is_not_data(tmp_path):
+    """1998 IRS puts ``(1), (2)`` under the names, and ``C`` means Schedule C."""
+    book = tmp_path / "98zp.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "Sheet 1"
+    sheet.append(["ALABAMA"])
+    sheet.append(["", "Schedule C", "", "Schedule F", ""])
+    sheet.append(["Income Size", "Number of", "Schedules", "Number of", "Schedules"])
+    sheet.append(["Income Size", "Returns", "C", "Returns", "F"])
+    sheet.append(["", "(1)", "(2)", "(3)", "(4)"])
+    sheet.append(["ALABAMA", 10, 30, 40, 50])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="irs",
+                source_id="irs",
+                release="1998",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 1
+        )
+        row = con.execute(
+            """
+            SELECT income_size, schedules_c, schedules_f
+            FROM lead_public.irs
+            """
+        ).fetchone()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'irs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert row == ("ALABAMA", "30", "50")
+    assert "c" not in columns
+    assert "f" not in columns
+    assert not any(name.startswith("column") for name in columns)
+
+
+def test_an_action_code_key_keeps_the_first_letter(tmp_path):
+    """The HCPCS key is ``A / Add`` with a title and no header row."""
+    book = tmp_path / "hcpcs.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "Action Code Key"
+    sheet.append(["Action Codes"])
+    sheet.append(["A", "Add"])
+    sheet.append(["B", "Coverage Change"])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="hcpcs",
+                source_id="hcpcs",
+                release="2026",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 2
+        )
+        rows = con.execute(
+            """
+            SELECT action_code, action_label
+            FROM lead_public.hcpcs
+            ORDER BY action_code
+            """
+        ).fetchall()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'hcpcs'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert rows == [("A", "Add"), ("B", "Coverage Change")]
+    assert "a" not in columns
+    assert "add" not in columns
+
+
+def test_a_blank_leading_header_is_the_row_stub(tmp_path):
+    """CMS home health leaves the stub cell blank and puts Total under it."""
+    book = tmp_path / "hha.xlsx"
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.title = "MDCR HHA 6"
+    sheet.append([None, "Medicare Home Health Agency"])
+    sheet.append([None, "Persons", "Visits"])
+    sheet.append(["BLANK"])
+    sheet.append(["Number of Service Visits"])
+    sheet.append(["Total", 10, 20])
+    sheet.append(["1-9", 4, 8])
+    wb.save(book)
+    con = duckdb.connect()
+    try:
+        assert (
+            load_file(
+                con,
+                book,
+                table="hha",
+                source_id="hha",
+                release="2024",
+                sha256=_sha(book),
+                actor="tester",
+            )
+            == 2
+        )
+        rows = con.execute(
+            """
+            SELECT section, category, persons, visits
+            FROM lead_public.hha
+            ORDER BY category
+            """
+        ).fetchall()
+        columns = [
+            name
+            for (name,) in con.execute(
+                """
+                SELECT column_name FROM information_schema.columns
+                WHERE table_name = 'hha'
+                """
+            ).fetchall()
+        ]
+    finally:
+        con.close()
+    assert rows == [
+        ("Number of Service Visits", "1-9", "4", "8"),
+        ("Number of Service Visits", "Total", "10", "20"),
+    ]
+    assert not any(name.startswith("column") or name == "col" for name in columns)
+
+
+def test_the_log_ddl_has_no_catalog_token():
+    assert "aco" not in LOAD_LOG_DDL.casefold()
+
+
+def test_cli_loads_a_manifest_and_does_not_fetch(tmp_path):
+    src_dir = tmp_path / "demo"
+    src_dir.mkdir()
+    src = src_dir / "rows.csv"
+    src.write_text("hsa,year,quarter,n\n12,2024,4,1\n")
+    manifest = _manifest(
+        tmp_path,
+        [_row("demo", "2024", "demo/rows.csv", _sha(src))],
+    )
+    db = tmp_path / "public.duckdb"
+    result = runner.invoke(
+        app,
+        [
+            "lead",
+            "load",
+            "--db",
+            str(db),
+            "--manifest",
+            str(manifest),
+            "--root",
+            str(tmp_path),
+            "--actor",
+            "tester",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert "loaded\t1" in result.output
+    con = duckdb.connect(str(db), read_only=True)
+    try:
+        row = con.execute("SELECT geo_hsa, time_key FROM lead_public.demo").fetchone()
+    finally:
+        con.close()
+    assert row == ("12", "2024Q4")
diff --git a/tests/lead/test_load_spine.py b/tests/lead/test_load_spine.py
new file mode 100644
index 0000000..3226066
--- /dev/null
+++ b/tests/lead/test_load_spine.py
@@ -0,0 +1,54 @@
+"""Column and filename cues for the public-data spine. No crosswalk."""
+
+from __future__ import annotations
+
+from lead.load.spine_map import file_quarter, file_year, spine_columns
+
+
+def test_county_fips_year_and_quarter_are_named():
+    got = spine_columns(["area_fips", "year", "qtr", "employment"])
+    assert got.fips == "area_fips"
+    assert got.year == "year"
+    assert got.quarter == "qtr"
+
+
+def test_a_literal_fips_wins_over_area_fips():
+    assert spine_columns(["area_fips", "fips"]).fips == "fips"
+
+
+def test_state_and_county_parts_stay_uncomposed():
+    got = spine_columns(["statefips", "countyfips", "a00100"])
+    assert got.state_fips == "statefips"
+    assert got.county_fips == "countyfips"
+    assert got.fips == ""
+
+
+def test_hsa_hrr_cbsa_zcta_and_zip_are_separate():
+    got = spine_columns(["hsa_num", "hrrnum", "cbsa_code", "zcta5", "zipcode"])
+    assert (got.hsa, got.hrr, got.cbsa, got.zcta, got.zip) == (
+        "hsa_num",
+        "hrrnum",
+        "cbsa_code",
+        "zcta5",
+        "zipcode",
+    )
+
+
+def test_irs_national_filenames_carry_one_year():
+    assert file_year("23incyallagi.csv") == 2023
+    assert file_year("12cyallagi.csv") == 2012
+    assert file_year("23zpallnoagi.csv") == 2023
+    assert file_year("1989countyincome.zip") == 1989
+    assert file_year("1998zipcode.zip") == 1998
+    assert file_year("county2013.zip") == 2013
+
+
+def test_two_years_in_one_name_are_not_a_year():
+    assert file_year("cpi_medical_2007_2026.json") is None
+
+
+def test_one_quarter_token_is_the_file_quarter():
+    assert file_year("MDS_Facility_Level_2026_Q2.csv") == 2026
+    assert file_quarter("MDS_Facility_Level_2026_Q2.csv") == 2
+    assert file_quarter("2024_qtrly_by_industry.zip") is None
+    assert file_quarter("2024.q1-q4.by_industry.csv") is None
diff --git a/tests/lead/test_load_strand5.py b/tests/lead/test_load_strand5.py
new file mode 100644
index 0000000..781e69e
--- /dev/null
+++ b/tests/lead/test_load_strand5.py
@@ -0,0 +1,279 @@
+"""Strand-5 files load into queryable tables. Absent series stay absent."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import zipfile
+from pathlib import Path
+
+import duckdb
+import openpyxl
+
+from bls.ces import read_series
+from bls.oews import read_occupations
+from lead.load.irs import choose_irs_members
+from lead.load.strand5 import load_strand5
+
+_TSV = "source\turl\tpath\tbytes\tsha256\tstatus\tnote\n"
+
+
+def _sha(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _line(source: str, path: Path | str, status: str, *, note: str = "") -> str:
+    if isinstance(path, Path):
+        digest = _sha(path) if path.exists() else ""
+        size = path.stat().st_size if path.exists() else ""
+        shown = str(path)
+    else:
+        digest = ""
+        size = ""
+        shown = path
+    return f"{source}\thttp://example.invalid\t{shown}\t{size}\t{digest}\t{status}\t{note}\n"
+
+
+def _oews_zip(path: Path) -> None:
+    wb = openpyxl.Workbook()
+    sheet = wb.active
+    sheet.append(
+        ["AREA", "OCC_CODE", "OCC_TITLE", "O_GROUP", "TOT_EMP", "H_MEAN", "A_MEAN"]
+    )
+    sheet.append(["99", "29-1141", "Registered Nurses", "detailed", 100, 40, 80000])
+    buf = io.BytesIO()
+    wb.save(buf)
+    with zipfile.ZipFile(path, "w") as zf:
+        zf.writestr("oesm25nat/national_M2025_dl.xlsx", buf.getvalue())
+
+
+def test_absent_rows_keep_distinct_urls(tmp_path):
+    root = tmp_path
+    (root / "files.tsv").write_text(
+        _TSV
+        + "qcew\thttps://data.bls.gov/cew/data/files/1975/csv/1975_annual_by_industry.zip\t\t\t\thttp_404\tabsent\n"
+        + "qcew\thttps://data.bls.gov/cew/data/files/1976/csv/1976_annual_by_industry.zip\t\t\t\thttp_404\tabsent\n"
+        + "fred\thttps://fred.stlouisfed.org/graph/fredgraph.csv?id=UNRATE\t\t\t\tunreachable\tno key\n"
+    )
+    con = duckdb.connect()
+    try:
+        load_strand5(con, root, actor="tester")
+        load_strand5(con, root, actor="tester")
+        rows = con.execute(
+            """
+            SELECT source_id, path, status
+            FROM lead_public.load_log
+            ORDER BY source_id, path
+            """
+        ).fetchall()
+    finally:
+        con.close()
+    assert len(rows) == 3
+    assert [row[1].rsplit("/", 1)[-1] for row in rows if row[0] == "qcew"] == [
+        "1975_annual_by_industry.zip",
+        "1976_annual_by_industry.zip",
+    ]
+    assert rows[0][0] == "fred" and rows[0][2] == "absent"
+
+
+def test_choose_irs_members_keeps_national_and_cbsa_files():
+    members = [
+        "13incyallagi.csv",
+        "13incyallnoagi.csv",
+        "13incyall.xls",
+        "13incyal.xls",
+        "13incbsa.csv",
+        "11incydocguide.doc",
+    ]
+    assert choose_irs_members(members) == [
+        "13incyallagi.csv",
+        "13incyallnoagi.csv",
+        "13incbsa.csv",
+    ]
+
+
+def test_a_zipcode_csv_is_the_national_extract():
+    assert choose_irs_members(
+        [
+            "zipcode07.csv",
+            "ZIP Code 2007 TX.xls",
+            "ZIP Code 2007 CA.xls",
+        ]
+    ) == ["zipcode07.csv"]
+
+
+def test_a_state_only_zip_keeps_the_state_workbooks():
+    assert choose_irs_members(
+        ["1998ZIPCode/", "1998ZIPCode/98zp01al.xls", "1998ZIPCode/98zp02ak.xls"]
+    ) == ["1998ZIPCode/98zp01al.xls", "1998ZIPCode/98zp02ak.xls"]
+
+
+def test_strand5_loads_each_family_and_skips_what_is_not_a_file(tmp_path):
+    root = tmp_path / "strand5"
+    (root / "bls").mkdir(parents=True)
+    (root / "qcew").mkdir()
+    (root / "irs_soi" / "county").mkdir(parents=True)
+    (root / "irs_soi" / "zip").mkdir()
+    (root / "ma_penetration" / "2024-06").mkdir(parents=True)
+    (root / "fred").mkdir()
+
+    ppi = root / "bls" / "ppi.csv"
+    ppi.write_text(
+        "series_id,year,period,value\nWPS512,2024,M01,100.5\nWPS512,2024,M02,\n"
+    )
+    jolts = root / "bls" / "jolts.csv"
+    jolts.write_text(
+        "series_id,year,period,value\nJTS620000000000000JOL,2024,M12,1410\n"
+    )
+    ces = root / "bls" / "ces.csv"
+    ces.write_text(
+        "series_id,year,period,value\nCES6562000001,2024,M01,17000\nCES6562000001,2024,M13,16900\n"
+    )
+    oews = tmp_path / "oesm25nat.zip"
+    _oews_zip(oews)
+
+    qcew = root / "qcew" / "2024_annual_by_industry.zip"
+    with zipfile.ZipFile(qcew, "w") as zf:
+        zf.writestr(
+            "2024.annual.by_industry/2024.annual 10 Total, all industries.csv",
+            "area_fips,year,qtr,annual_avg_emplvl\n01000,2024,A,10\n01001,2024,A,3\n",
+        )
+
+    county = root / "irs_soi" / "county" / "23incyallagi.csv"
+    county.write_text("STATEFIPS,COUNTYFIPS,A00100\n01,001,100\n01,000,50\n")
+    zcta = root / "irs_soi" / "zip" / "23zpallagi.csv"
+    zcta.write_text("STATEFIPS,ZIPCODE,A00100\n01,30301,7\n")
+    bundled = root / "irs_soi" / "county" / "county2023.zip"
+    with zipfile.ZipFile(bundled, "w") as zf:
+        zf.writestr("23incyallagi.csv", county.read_text())
+        zf.writestr("al23.xls", b"not-read")
+
+    pen = root / "ma_penetration" / "2024-06" / "county-penetration-june-2024.zip"
+    with zipfile.ZipFile(pen, "w") as zf:
+        zf.writestr("readme.txt", "notes\n")
+        zf.writestr(
+            "State_County_Penetration_MA_2024_06.csv",
+            "FIPS,Eligibles,Enrolled,Penetration\n01001,100,40,0.4\n",
+        )
+
+    tsv = root / "files.tsv"
+    tsv.write_text(
+        _TSV
+        + _line("bls_ppi", ppi, "ok")
+        + _line("bls_jolts", jolts, "ok")
+        + _line("bls_ces", ces, "ok")
+        + _line("bls_oews", oews, "ok", note="May 2025 national")
+        + _line("qcew", qcew, "ok")
+        + _line("qcew", "1975_annual_by_industry.zip", "http_404", note="absent")
+        + _line("irs_soi", county, "ok", note="county")
+        + _line("irs_soi", zcta, "ok", note="zip")
+        + _line("irs_soi", bundled, "ok", note="county")
+        + _line("ma_penetration", pen, "ok")
+        + _line("ma_penetration", pen, "http_22", note="curl 22")
+        + _line("fred", "", "unreachable", note="no file")
+        + _line(
+            "cms_mds_frequency",
+            "/no/such/mds.csv",
+            "ok",
+            note="covered by the enrichment manifest",
+        )
+    )
+    # The unreachable and 404 rows have an empty path in the real inventory.
+    text = tsv.read_text()
+    text = text.replace(
+        _line("qcew", "1975_annual_by_industry.zip", "http_404", note="absent"),
+        "qcew\thttp://example.invalid\t\t\t\thttp_404\tabsent\n",
+    )
+    text = text.replace(
+        _line("fred", "", "unreachable", note="no file"),
+        "fred\thttp://example.invalid\t\t\t\tunreachable\tno file\n",
+    )
+    # The duplicate-hash http_22 row must not carry the ok row's digest as a
+    # second load of the same bytes. Record it as a failed attempt with no hash.
+    text = text.replace(
+        _line("ma_penetration", pen, "http_22", note="curl 22"),
+        f"ma_penetration\thttp://example.invalid\t{pen}\t\t\thttp_22\tcurl 22\n",
+    )
+    tsv.write_text(text)
+
+    con = duckdb.connect()
+    try:
+        first = load_strand5(con, root, actor="tester")
+        second = load_strand5(con, root, actor="tester")
+        ppi_row = con.execute(
+            "SELECT series_id, spine_year, spine_month, value FROM bls.ppi ORDER BY spine_month"
+        ).fetchall()
+        jolts_row = con.execute(
+            "SELECT series_id, value, time_key FROM bls.jolts"
+        ).fetchone()
+        ces_rows = read_series(con, ["CES6562000001"])
+        oews_rows = read_occupations(con, ["29-1141"])
+        qcew_rows = con.execute(
+            """
+            SELECT geo_fips, spine_year, file_year, annual_avg_emplvl
+            FROM bls.qcew
+            ORDER BY geo_fips NULLS FIRST
+            """
+        ).fetchall()
+        irs_rows = con.execute(
+            """
+            SELECT geo_fips, geo_zip, file_year, a00100
+            FROM econ.irs_soi
+            ORDER BY geo_fips NULLS LAST, geo_zip NULLS LAST
+            """
+        ).fetchall()
+        pen_row = con.execute(
+            """
+            SELECT geo_fips, spine_year, spine_month, spine_quarter, time_key, penetration
+            FROM lead_public.ma_county_penetration
+            """
+        ).fetchone()
+        fred = con.execute(
+            """
+            SELECT COUNT(*) FROM information_schema.tables
+            WHERE table_schema = 'econ' AND table_name = 'fred'
+            """
+        ).fetchone()[0]
+        fred_log = con.execute(
+            """
+            SELECT status, reason, count(*)
+            FROM lead_public.load_log
+            WHERE source_id = 'fred'
+            GROUP BY 1, 2
+            """
+        ).fetchall()
+    finally:
+        con.close()
+
+    assert ppi_row == [("WPS512", 2024, 1, 100.5), ("WPS512", 2024, 2, None)]
+    assert jolts_row == ("JTS620000000000000JOL", 1410.0, "2024Q4")
+    assert [(r.year, r.month, r.value) for r in ces_rows] == [
+        (2024, 1, 17000.0),
+        (2024, 13, 16900.0),
+    ]
+    assert len(oews_rows) == 1 and oews_rows[0].year == 2025
+    assert qcew_rows == [(None, 2024, 2024, "10"), ("01001", 2024, 2024, "3")]
+    assert irs_rows == [
+        ("01001", None, 2023, "100"),
+        (None, "30301", 2023, "7"),
+        (None, None, 2023, "50"),
+    ]
+    assert pen_row == ("01001", 2024, 6, 2, "2024Q2", "0.4")
+    assert fred == 0
+    assert fred_log == [("absent", "unreachable", 1)]
+    by_status = {}
+    for row in first:
+        by_status.setdefault(row.status, []).append(row.source_id)
+    assert "fred" in by_status["absent"] or "fred" in by_status.get("unreachable", [])
+    assert "qcew" in by_status["absent"] or any(
+        row.source_id == "qcew" and row.status == "absent" for row in first
+    )
+    assert any(
+        row.source_id == "cms_mds_frequency" and row.status == "skipped"
+        for row in first
+    )
+    assert any(
+        row.source_id == "ma_penetration" and row.status == "skipped" for row in first
+    )
+    assert sum(row.rows for row in second) == 0
+    assert all(row.status in {"loaded", "skipped", "absent"} for row in second)
diff --git a/tests/lead/test_review.py b/tests/lead/test_review.py
new file mode 100644
index 0000000..cb54708
--- /dev/null
+++ b/tests/lead/test_review.py
@@ -0,0 +1,188 @@
+"""Gold labels are a separate file. They do not tag the library."""
+
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+import pytest
+
+from lead.review.codes import GATE, PROJECTS, reason_codes
+from lead.review.library import (
+    connect,
+    load_item,
+    next_unlabeled,
+    pass_two_keys,
+    project_keys,
+)
+from lead.review.store import Store
+from prisma.audit import disagreements
+from prisma.calibrate import sized
+from zot.db import TYPE_MAP, Db
+from zot.schema import create_db
+
+
+def test_gate_sizes_match_the_calibration_module():
+    for stage, n in GATE.items():
+        assert sized(n, stage)
+        assert not sized(n - 1, stage)
+
+
+def test_every_strand_has_an_exclude_codebook():
+    for project in PROJECTS:
+        codes = reason_codes(project)
+        assert "off-topic" in codes
+        assert "unavailable-fulltext" in codes
+
+
+def test_exclude_needs_a_codebook_reason_and_a_rationale(tmp_path: Path):
+    store = Store(tmp_path / "labels.sqlite")
+    store.record(
+        zotero_key="AAAAAAAA",
+        project="lead-history",
+        stage="title_abstract",
+        pass_n=1,
+        decision="include",
+        rationale="Names a predecessor of a registry target.",
+        actor="kert",
+    )
+    with pytest.raises(ValueError, match="codebook"):
+        store.record(
+            zotero_key="BBBBBBBB",
+            project="lead-history",
+            stage="title_abstract",
+            pass_n=1,
+            decision="exclude",
+            rationale="No.",
+            actor="kert",
+        )
+    with pytest.raises(ValueError, match="only on exclude"):
+        store.record(
+            zotero_key="CCCCCCCC",
+            project="lead-history",
+            stage="title_abstract",
+            pass_n=1,
+            decision="uncertain",
+            reason="off-topic",
+            rationale="The abstract does not say.",
+            actor="kert",
+        )
+    with pytest.raises(ValueError, match="rationale"):
+        store.record(
+            zotero_key="DDDDDDDD",
+            project="lead-history",
+            stage="title_abstract",
+            pass_n=1,
+            decision="include",
+            rationale="  ",
+            actor="kert",
+        )
+
+
+def test_a_later_label_is_the_current_one(tmp_path: Path):
+    store = Store(tmp_path / "labels.sqlite")
+    kwargs = {
+        "zotero_key": "AAAAAAAA",
+        "project": "lead-limits",
+        "stage": "title_abstract",
+        "pass_n": 1,
+        "actor": "kert",
+    }
+    store.record(**kwargs, decision="exclude", reason="no-limit", rationale="No bound.")
+    store.record(
+        **kwargs, decision="uncertain", rationale="The formula might be there."
+    )
+    assert store.latest("lead-limits", "title_abstract", 1) == {"AAAAAAAA": "uncertain"}
+    assert store.count("lead-limits", "title_abstract", 1) == 1
+    assert len(store.history("lead-limits", "title_abstract", 1)) == 2
+
+
+def test_pass_two_is_the_pass_one_keys_still_open():
+    keys = ("AAAAAAAA", "BBBBBBBB", "CCCCCCCC")
+    assert next_unlabeled(keys, {"AAAAAAAA"}) == "BBBBBBBB"
+    assert next_unlabeled(keys, set(keys)) is None
+    assert pass_two_keys(keys, {"BBBBBBBB", "CCCCCCCC"}, {"CCCCCCCC"}) == ("BBBBBBBB",)
+
+
+def test_two_passes_can_disagree_without_a_sensitivity(tmp_path: Path):
+    store = Store(tmp_path / "labels.sqlite")
+    for key, first, second in (
+        ("AAAAAAAA", "include", "exclude"),
+        ("BBBBBBBB", "exclude", "exclude"),
+    ):
+        store.record(
+            zotero_key=key,
+            project="lead-scan",
+            stage="title_abstract",
+            pass_n=1,
+            decision=first,
+            reason="off-topic" if first == "exclude" else "",
+            rationale="Pass 1.",
+            actor="kert",
+        )
+        store.record(
+            zotero_key=key,
+            project="lead-scan",
+            stage="title_abstract",
+            pass_n=2,
+            decision=second,
+            reason="off-topic" if second == "exclude" else "",
+            rationale="Pass 2.",
+            actor="kert",
+        )
+    one = store.latest("lead-scan", "title_abstract", 1)
+    two = store.latest("lead-scan", "title_abstract", 2)
+    assert disagreements(one, two) == ("AAAAAAAA",)
+
+
+def test_the_library_is_read_and_labels_do_not_tag_it(tmp_path: Path):
+    library = tmp_path / "zotero.sqlite"
+    create_db(str(library)).close()
+    with Db(str(library)) as db:
+        kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
+        db.set_field(kept, "title", "A predecessor of the frailty index")
+        db.set_field(kept, "abstractNote", "Medicare cohort, factor before outcome.")
+        db.set_field(kept, "date", "2020")
+        db.set_field(kept, "DOI", "10.0000/example")
+        db.add_creators(kept, [("Ada", "Lovelace")])
+        db.sync_tags(kept, ["project:lead-precursors"])
+        note = db.create_item(TYPE_MAP["note"], key="BBBBBBBB")
+        db.sync_tags(note, ["project:lead-precursors"])
+        db.commit()
+
+    con = connect(library)
+    try:
+        with pytest.raises(sqlite3.OperationalError):
+            con.execute("CREATE TABLE leaked (id INTEGER)")
+        assert project_keys(con, "lead-precursors") == ("AAAAAAAA",)
+        item = load_item(con, "AAAAAAAA", tmp_path / "storage")
+    finally:
+        con.close()
+    assert item.title.startswith("A predecessor")
+    assert item.authors == ("Ada Lovelace",)
+    assert item.year == "2020"
+    assert item.doi == "10.0000/example"
+    assert "Medicare" in item.abstract
+    assert item.attachment_paths == ()
+
+    Store(tmp_path / "labels.sqlite").record(
+        zotero_key="AAAAAAAA",
+        project="lead-precursors",
+        stage="full_text",
+        pass_n=1,
+        decision="exclude",
+        reason="cross-sectional-no-time-order",
+        rationale="No temporal order.",
+        actor="kert",
+    )
+    with Db(str(library)) as db:
+        tags = [
+            row[0]
+            for row in db.con.execute(
+                """SELECT t.name FROM itemTags it
+                   JOIN tags t ON it.tagID = t.tagID
+                   WHERE it.itemID = ?""",
+                (kept,),
+            ).fetchall()
+        ]
+    assert tags == ["project:lead-precursors"]
-- 
2.49.1


From ed2e21ce8d6a610a059c474b7d605485e2385908 Mon Sep 17 00:00:00 2001
From: kert 
Date: Mon, 2 Nov 2026 00:48:20 -0500
Subject: [PATCH 30/30] fix(lead): open the Zotero library on each labels run

The cached connection was created on one script thread and reused on the next, so the labels page died in project_keys. The file is mounted read-only, so the open is immutable and does not create a -shm beside it. The explorer pages are covered for the 99% gate.
---
 src/lead/explore/review_page.py  |  21 +-
 src/lead/review/library.py       |   9 +-
 tests/lead/test_explore.py       | 173 ++++++++++++-
 tests/lead/test_explore_pages.py | 431 +++++++++++++++++++++++++++++++
 tests/lead/test_review.py        |  87 ++++++-
 5 files changed, 711 insertions(+), 10 deletions(-)
 create mode 100644 tests/lead/test_explore_pages.py

diff --git a/src/lead/explore/review_page.py b/src/lead/explore/review_page.py
index c81978a..75ec5ec 100644
--- a/src/lead/explore/review_page.py
+++ b/src/lead/explore/review_page.py
@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import os
+import sqlite3
 from pathlib import Path
 
 import streamlit as st
@@ -60,11 +61,24 @@ def review() -> None:
     pass_n = st.radio("Pass", (1, 2), horizontal=True)
     try:
         store = _store(_LABELS)
-        lib = _library(_ZOTERO)
+        lib = connect(_ZOTERO)
     except Exception as exc:
         st.error(str(exc))
         return
 
+    try:
+        _review_body(store, lib, project, stage, pass_n)
+    finally:
+        lib.close()
+
+
+def _review_body(
+    store: Store,
+    lib: sqlite3.Connection,
+    project: str,
+    stage: str,
+    pass_n: int,
+) -> None:
     labeled = store.latest(project, stage, pass_n)
     first = store.latest(project, stage, 1)
     target = GATE[stage]
@@ -171,11 +185,6 @@ def _actor() -> str:
     return "reviewer"
 
 
-@st.cache_resource(show_spinner=False)
-def _library(path: str):
-    return connect(path)
-
-
 @st.cache_resource(show_spinner=False)
 def _store(path: str) -> Store:
     return Store(path)
diff --git a/src/lead/review/library.py b/src/lead/review/library.py
index 7e5e8bb..48ef159 100644
--- a/src/lead/review/library.py
+++ b/src/lead/review/library.py
@@ -23,8 +23,13 @@ class Item:
 
 
 def connect(path: str | Path) -> sqlite3.Connection:
-    """Open the library read-only. A write on this connection fails."""
-    con = sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+    """Open the library read-only. A write on this connection fails.
+
+    ``immutable=1`` keeps SQLite from creating a ``-shm`` file next to the
+    database. The explorer mounts that file read-only, and without this a
+    read fails before the first query.
+    """
+    con = sqlite3.connect(f"file:{path}?mode=ro&immutable=1", uri=True)
     con.row_factory = sqlite3.Row
     return con
 
diff --git a/tests/lead/test_explore.py b/tests/lead/test_explore.py
index 10b50f6..86b6e52 100644
--- a/tests/lead/test_explore.py
+++ b/tests/lead/test_explore.py
@@ -8,7 +8,7 @@ import duckdb
 import pytest
 
 from lead.explore import catalog, guard, writeup
-from lead.explore.catalog import HEAVY_ROWS, NeedsFilter, QueryTimeout
+from lead.explore.catalog import HEAVY_ROWS, NeedsFilter, QueryTimeout, TableCard
 from lead.load.frame import ensure_log, record
 
 
@@ -357,3 +357,174 @@ def test_explore_command_points_streamlit_at_the_mirror(tmp_path, monkeypatch):
     assert cmd[1:3] == ["-m", "lead.explore"]
     assert cmd[cmd.index("--server.port") + 1] == "8765"
     assert cmd[cmd.index("--server.headless") + 1] == "false"
+
+
+def test_a_missing_mirror_and_a_bad_identifier_are_refused(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        catalog.open_mirror(tmp_path / "missing.duckdb")
+    with pytest.raises(ValueError):
+        catalog.quote("1nope")
+    assert catalog.quote("geo_fips") == '"geo_fips"'
+    bare = duckdb.connect()
+    assert catalog.latest_files(bare) == ()
+    with pytest.raises(KeyError):
+        catalog.card_for((), "lead_public.missing")
+
+
+def test_filters_series_and_a_profile_stay_inside_the_cap(tmp_path):
+    con = _mirror(tmp_path)
+    ces = _card(con, "bls.ces")
+    qcew = _card(con, "bls.qcew")
+    demo = _card(con, "lead_public.cms_demo")
+    assert ces.year_column() == "year"
+    assert "value" in ces.measure_columns()
+    plain = TableCard("lead_public", "note", (("body", "VARCHAR"),), 1, 1, ())
+    assert plain.year_column() == ""
+    measured = TableCard(
+        "lead_public",
+        "note",
+        (("amount", "DECIMAL(10,2)"), ("year", "INTEGER"), ("spine_month", "INTEGER")),
+        1,
+        1,
+        (),
+    )
+    assert measured.measure_columns() == ("amount",)
+    got = catalog.sample(con, ces, series=("series_id", "CES1"))
+    assert {row["year"] for row in got} == {2020, 2022}
+    with pytest.raises(ValueError, match="nope"):
+        catalog.sample(con, ces, series=("nope", "CES1"))
+    with pytest.raises(ValueError, match="geo_hsa"):
+        catalog.sample(con, ces, geo=("geo_hsa", "1"))
+    with pytest.raises(ValueError, match="limit"):
+        catalog.sample(con, ces, limit=0)
+    with pytest.raises(ValueError, match="limit"):
+        catalog.sample(con, ces, limit=catalog.SAMPLE_CAP + 1)
+    assert catalog.series_ids(con, ces) == ("CES1",)
+    with pytest.raises(NeedsFilter):
+        catalog.series_ids(con, qcew)
+    with pytest.raises(ValueError, match="nope"):
+        catalog.series_ids(con, ces, column="nope")
+    with pytest.raises(ValueError, match="column"):
+        catalog.series_points(
+            con, ces, series_column="nope", series_id="CES1", value_column="value"
+        )
+    with pytest.raises(NeedsFilter):
+        catalog.series_points(
+            con, ces, series_column="series_id", series_id="", value_column="value"
+        )
+    empty = catalog.span(
+        [{"year": None, "value": 1}, {"year": "x", "value": "no"}],
+        year_key="year",
+        value_key="value",
+    )
+    assert empty.points == 0
+    one = catalog.span([{"year": 2020, "value": 3}], year_key="year", value_key="value")
+    assert one.change is None
+    profiled = catalog.profile(con, demo)
+    assert profiled["rows"] == 2
+    assert profiled["year_column"] == "spine_year"
+    assert profiled["geo_fips"] == 2
+    con.execute("CREATE TABLE lead_public.note (body VARCHAR)")
+    with pytest.raises(NeedsFilter):
+        catalog.profile(con, _card(con, "lead_public.note"))
+    con.execute("INSERT INTO lead_public.note VALUES ('a')")
+    record(
+        con,
+        source_id="cms_note_body",
+        release="1",
+        sha256="e" * 64,
+        path="/data/note-body.csv",
+        table="lead_public.note",
+        status="loaded",
+        rows=1,
+        reason="",
+        actor="stack",
+    )
+    note = _card(con, "lead_public.note")
+    assert catalog.profile(con, note)["rows"] == 1
+    with pytest.raises(ValueError, match="year"):
+        catalog.profile(con, note, year=2020)
+    with pytest.raises(ValueError, match="year"):
+        catalog.sample(con, note, year=2020)
+    con.execute("CREATE TABLE lead_public.county (geo_fips VARCHAR)")
+    con.execute("INSERT INTO lead_public.county VALUES ('01001')")
+    looked = catalog.place_summary(
+        con,
+        (_card(con, "lead_public.county"), demo),
+        geo_column="geo_fips",
+        geo_value="01001",
+        year=2010,
+    )
+    by_table = {row["table"]: row for row in looked}
+    assert by_table["lead_public.county"]["note"] == "no year column"
+    assert by_table["lead_public.county"]["rows"] == 1
+    assert by_table["lead_public.cms_demo"]["rows"] == 1
+
+
+def test_a_timed_out_place_is_recorded_and_the_guard_rejects_the_rest(
+    tmp_path, monkeypatch
+):
+    con = _mirror(tmp_path)
+
+    def _timeout(*_args, **_kwargs):
+        raise QueryTimeout(1)
+
+    monkeypatch.setattr(catalog, "count_rows", _timeout)
+    rows = catalog.place_summary(
+        con, _cards(con), geo_column="geo_fips", geo_value="01001", year=2020
+    )
+    assert {row["note"] for row in rows} == {"timed out"}
+    with pytest.raises(ValueError):
+        catalog.place_summary(con, (), geo_column="zip", geo_value="1")
+    with pytest.raises(guard.SqlRejected, match="empty"):
+        guard.prepare("  ")
+    with pytest.raises(guard.SqlRejected, match="start with"):
+        guard.prepare("hello")
+    with pytest.raises(guard.SqlRejected, match="cap"):
+        guard.prepare("SELECT 1", cap=0)
+
+
+def test_the_writeup_names_every_family_and_an_empty_file():
+    assert writeup.family_of("irs_soi") == "irs"
+    assert writeup.family_of("census_acs") == "census"
+    assert writeup.family_of("cdc_places") == "cdc"
+    assert writeup.family_of("ahrq_ccsr") == "ahrq"
+    assert writeup.family_of("nlm_rxnorm") == "nlm"
+    assert writeup.family_of("hrsa_hpsa") == "hrsa"
+    assert writeup.family_of("ecqi_measures") == "ecqi"
+    assert writeup.family_of("ma_penetration") == "ma"
+    assert writeup.family_of("ma") == "ma"
+    assert writeup.family_of("fred") == "fred"
+    assert writeup.family_of("noaa") == "other"
+    recorded = writeup._recorded({"loaded": 1, "partial": 2}, 5, 1)
+    assert ("partial", 2) in recorded.rows
+    bare = duckdb.connect()
+    mirror = next(
+        section for section in writeup.build(bare) if section.heading == "The mirror"
+    )
+    assert "no load log and no observation tables" in mirror.body[1]
+    bare.execute("CREATE TABLE notes (body VARCHAR)")
+    mirror = next(
+        section for section in writeup.build(bare) if section.heading == "The mirror"
+    )
+    assert "no load log" in mirror.body[1]
+    present = writeup._gaps(
+        (),
+        (TableCard("econ", "fred", (), 1, 1, ()),),
+    )
+    assert present.body[0] == "`econ.fred` is present in this file."
+    other = writeup._gaps(
+        (
+            catalog.FileRow(
+                "fred", "fred.csv", "/data/fred.csv", "", "skipped", 0, "later"
+            ),
+        ),
+        (),
+    )
+    assert "skipped" in other.body[0]
+    assert "reason later" in other.body[0]
+    none = writeup._gaps((), ())
+    assert none.body[0] == (
+        "The load log has no FRED row, and `econ.fred` is not in this file."
+    )
+    assert "No file's latest status is skipped." in none.body
diff --git a/tests/lead/test_explore_pages.py b/tests/lead/test_explore_pages.py
new file mode 100644
index 0000000..168f0c1
--- /dev/null
+++ b/tests/lead/test_explore_pages.py
@@ -0,0 +1,431 @@
+"""The explorer pages render against a small mirror and do not write it."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+from tests.lead.test_explore import _mirror
+from zot.db import TYPE_MAP, Db
+from zot.schema import create_db
+
+
+def _run(source: str):
+    from streamlit.testing.v1 import AppTest
+
+    at = AppTest.from_string(source, default_timeout=90)
+    at.run()
+    assert not at.exception, at.exception
+    return at
+
+
+def _blob(at) -> str:
+    chunks: list[str] = []
+    for name in (
+        "markdown",
+        "caption",
+        "warning",
+        "error",
+        "info",
+        "success",
+        "text",
+        "title",
+        "header",
+        "subheader",
+        "metric",
+        "text_area",
+    ):
+        for item in getattr(at, name):
+            chunks.append(str(getattr(item, "value", "") or ""))
+            chunks.append(str(getattr(item, "label", "") or ""))
+    return "\n".join(part for part in chunks if part)
+
+
+def _widget(at, kind: str, label: str):
+    kinds = ("button", "form_submit_button") if kind == "button" else (kind,)
+    found = [
+        item for name in kinds for item in getattr(at, name) if item.label == label
+    ]
+    assert found, f"no {kind} labeled {label}"
+    return found[0]
+
+
+def _mirror_path(tmp_path: Path) -> str:
+    con = _mirror(tmp_path)
+    path = str(tmp_path / "public.duckdb")
+    con.close()
+    return path
+
+
+def test_coverage_filters_the_log_and_a_missing_file_is_an_error(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+    at = _run("from lead.explore.views import coverage\ncoverage()\n")
+    assert "loaded" in _blob(at)
+    assert at.dataframe
+    picked = _widget(at, "selectbox", "Status")
+    picked.select("loaded").run()
+    assert not at.exception, at.exception
+    _widget(at, "text_input", "Source or path contains").input("qcew").run()
+    assert not at.exception, at.exception
+    assert at.dataframe
+
+    monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "absent.duckdb"))
+    missing = _run("from lead.explore.views import writeup\nwriteup()\n")
+    assert "No mirror" in _blob(missing)
+
+
+def test_a_locked_mirror_and_any_other_open_error_are_shown(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "locked.duckdb"))
+
+    def locked(_path):
+        raise RuntimeError("Could not set lock on file")
+
+    monkeypatch.setattr("lead.explore.views.open_mirror", locked)
+    at = _run("from lead.explore.views import coverage\ncoverage()\n")
+    assert "locked by a writer" in _blob(at)
+
+    def broken(_path):
+        raise RuntimeError("header said no")
+
+    monkeypatch.setattr("lead.explore.views.open_mirror", broken)
+    at = _run("from lead.explore.views import coverage\ncoverage()\n")
+    assert "header said no" in _blob(at)
+
+
+def test_tables_place_series_and_sql_read_the_mirror(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    assert browse.dataframe
+    _widget(browse, "text_input", "Filter names").input("zzzz").run()
+    assert not browse.exception, browse.exception
+    assert "No table name matches." in _blob(browse)
+
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    _widget(browse, "selectbox", "Table").select("bls.qcew").run()
+    assert not browse.exception, browse.exception
+    _widget(browse, "text_input", "Year").input("nope")
+    _widget(browse, "button", "Show 100 rows").click().run()
+    assert "Year must be a number." in _blob(browse)
+
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    _widget(browse, "selectbox", "Table").select("bls.qcew").run()
+    _widget(browse, "selectbox", "Geography column").select("geo_fips")
+    _widget(browse, "text_input", "Geography value").input("01001")
+    _widget(browse, "button", "Show 100 rows").click().run()
+    assert "needs a year" in _blob(browse)
+
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    _widget(browse, "selectbox", "Table").select("bls.qcew").run()
+    _widget(browse, "text_input", "Year").input("2020")
+    _widget(browse, "button", "Profile").click().run()
+    assert not browse.exception, browse.exception
+    assert browse.dataframe
+
+    place = _run("from lead.explore.views import place\nplace()\n")
+    _widget(place, "button", "Look up").click().run()
+    assert "Enter a value." in _blob(place)
+    _widget(place, "text_input", "Value").input("0100")
+    _widget(place, "text_input", "Year (required for a heavy table)").input("year")
+    _widget(place, "button", "Look up").click().run()
+    assert "Year must be a number." in _blob(place)
+
+    place = _run("from lead.explore.views import place\nplace()\n")
+    _widget(place, "text_input", "Value").input("01001")
+    _widget(place, "text_input", "Year (required for a heavy table)").input("2020")
+    _widget(place, "button", "Look up").click().run()
+    assert not place.exception, place.exception
+    assert place.dataframe
+    assert "tables have this value" in _blob(place)
+
+    series = _run("from lead.explore.views import series\nseries()\n")
+    _widget(series, "button", "Plot").click().run()
+    assert not series.exception, series.exception
+    assert series.dataframe
+    assert "Last minus first" in _blob(series)
+
+    reader = _run("from lead.explore.views import reader\nreader()\n")
+    _widget(reader, "button", "Run").click().run()
+    assert not reader.exception, reader.exception
+    assert reader.dataframe
+    _widget(reader, "text_area", "Statement").input("DROP TABLE bls.qcew")
+    _widget(reader, "button", "Run").click().run()
+    assert "only a read is allowed" in _blob(reader)
+
+
+def test_each_page_says_when_the_mirror_is_missing(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", str(tmp_path / "absent.duckdb"))
+    at = _run(
+        "from lead.explore.views import browse, place, series, reader\n"
+        "browse()\nplace()\nseries()\nreader()\n"
+    )
+    assert _blob(at).count("No mirror") == 4
+
+
+def test_show_rows_and_a_heavy_profile_without_a_year(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    _widget(browse, "selectbox", "Table").select("lead_public.cms_demo").run()
+    _widget(browse, "button", "Show 100 rows").click().run()
+    assert not browse.exception, browse.exception
+    assert browse.dataframe
+
+    browse = _run("from lead.explore.views import browse\nbrowse()\n")
+    _widget(browse, "selectbox", "Table").select("bls.qcew").run()
+    _widget(browse, "button", "Profile").click().run()
+    assert "needs a year" in _blob(browse)
+
+
+def test_place_and_series_surface_a_query_error(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+
+    def place_refused(*_args, **_kwargs):
+        raise ValueError("place refused")
+
+    monkeypatch.setattr("lead.explore.views.place_summary", place_refused)
+    place = _run("from lead.explore.views import place\nplace()\n")
+    _widget(place, "text_input", "Value").input("01001")
+    _widget(place, "button", "Look up").click().run()
+    assert "place refused" in _blob(place)
+
+    def series_refused(*_args, **_kwargs):
+        raise ValueError("series refused")
+
+    monkeypatch.setattr("lead.explore.views.series_points", series_refused)
+    series = _run("from lead.explore.views import series\nseries()\n")
+    _widget(series, "button", "Plot").click().run()
+    assert "series refused" in _blob(series)
+
+
+def test_an_empty_mirror_has_nothing_to_browse(tmp_path, monkeypatch):
+    path = tmp_path / "empty.duckdb"
+    import duckdb
+
+    duckdb.connect(str(path)).close()
+    monkeypatch.setenv("LEAD_PUBLIC_DB", str(path))
+    source = (
+        "from lead.explore.views import browse, place, series\n"
+        "browse()\nplace()\nseries()\n"
+    )
+    at = _run(source)
+    text = _blob(at)
+    assert "no tables" in text
+    assert "No table carries a geography column." in text
+    assert "No table has both series_id" in text
+
+
+def test_a_heavy_series_is_typed_and_an_unknown_id_is_empty(tmp_path, monkeypatch):
+    con = _mirror(tmp_path)
+    con.execute(
+        """
+        UPDATE lead_public.load_log
+        SET row_count = 2000000
+        WHERE table_name = 'bls.ces' AND status = 'loaded'
+        """
+    )
+    path = str(tmp_path / "public.duckdb")
+    con.close()
+    monkeypatch.setenv("LEAD_PUBLIC_DB", path)
+    at = _run("from lead.explore.views import series\nseries()\n")
+    assert "not listed from a scan" in _blob(at)
+    _widget(at, "text_input", "series_id").input("NOPE")
+    _widget(at, "button", "Plot").click().run()
+    assert "No rows for that id." in _blob(at)
+    _widget(at, "text_input", "series_id").input("CES1")
+    _widget(at, "button", "Plot").click().run()
+    assert not at.exception, at.exception
+    assert "Last minus first" in _blob(at)
+
+
+def test_a_series_list_that_fails_asks_for_the_id(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+
+    def fail(*_args, **_kwargs):
+        from lead.explore.catalog import NeedsFilter
+
+        raise NeedsFilter("bls.ces")
+
+    monkeypatch.setattr("lead.explore.views.series_ids", fail)
+    at = _run("from lead.explore.views import series\nseries()\n")
+    assert "needs a year or a series" in _blob(at)
+    assert _widget(at, "text_input", "series_id")
+
+
+def test_the_shell_links_osf_and_reconnects(tmp_path, monkeypatch):
+    monkeypatch.setenv("LEAD_PUBLIC_DB", _mirror_path(tmp_path))
+    at = _run("from lead.explore.app import main\nmain()\n")
+    urls = [str(getattr(item, "url", "")) for item in at.link_button]
+    assert any("osf.io/8wq3m" in url for url in urls) or "osf.io/8wq3m" in _blob(at)
+    _widget(at, "button", "Reconnect").click().run()
+    assert not at.exception, at.exception
+
+
+def test_module_main_forwards_streamlit_and_keeps_a_chosen_theme(monkeypatch):
+    from bcda.ui import brand
+    from lead.explore import __main__ as entry
+
+    seen: dict = {}
+
+    def exit_three():
+        seen["argv"] = list(sys.argv)
+        raise SystemExit(3)
+
+    monkeypatch.setattr("streamlit.web.cli.main", exit_three)
+    assert entry.main(["--server.port", "1"]) == 3
+    argv = seen["argv"]
+    assert argv[:2] == ["streamlit", "run"]
+    assert argv[3:5] == ["--theme.base", str(brand.THEME_PATH)]
+
+    def already():
+        seen["argv"] = list(sys.argv)
+
+    monkeypatch.setattr("streamlit.web.cli.main", already)
+    assert entry.main(["--theme.base", "plain"]) == 0
+    assert seen["argv"].count("--theme.base") == 1
+    assert entry.main(["--theme.base=plain"]) == 0
+
+    def exit_none():
+        raise SystemExit(None)
+
+    monkeypatch.setattr("streamlit.web.cli.main", exit_none)
+    assert entry.main([]) == 0
+
+
+def _zotero(tmp_path: Path, *, attachment: Path | None) -> Path:
+    library = tmp_path / "zotero.sqlite"
+    create_db(str(library)).close()
+    with Db(str(library)) as db:
+        kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
+        db.set_field(kept, "title", "A predecessor of the frailty index")
+        db.set_field(kept, "abstractNote", "Medicare cohort, factor before outcome.")
+        db.set_field(kept, "date", "2020")
+        db.set_field(kept, "DOI", "10.0000/example")
+        db.add_creators(kept, [("Ada", "Lovelace")])
+        db.sync_tags(kept, ["project:lead-history"])
+        if attachment is not None:
+            db.add_attachment(kept, path=str(attachment), key="BBBBBBBB")
+        db.commit()
+    return library
+
+
+def _labels(monkeypatch, tmp_path: Path, library: Path) -> None:
+    from lead.explore import review_page
+
+    monkeypatch.setattr(review_page, "_ZOTERO", str(library))
+    monkeypatch.setattr(review_page, "_STORAGE", str(tmp_path / "storage"))
+    monkeypatch.setattr(review_page, "_LABELS", str(tmp_path / "labels.sqlite"))
+
+
+def test_the_label_page_saves_a_decision_and_shows_a_full_text(tmp_path, monkeypatch):
+    html = tmp_path / "paper.html"
+    html.write_text(
+        "
    The exposure precedes the outcome.
    "
+    )
+    _labels(monkeypatch, tmp_path, _zotero(tmp_path, attachment=html))
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    assert "Ada Lovelace" in _blob(at)
+    assert "Nothing unlabeled" not in _blob(at)
+    _widget(at, "radio", "Decision").set_value("include")
+    _widget(at, "text_area", "Why, in a sentence that names the criterion").input(
+        "Names a predecessor of a registry target."
+    )
+    _widget(at, "button", "Save label").click().run()
+    assert not at.exception, at.exception
+    assert "Nothing unlabeled on this pass." in _blob(at)
+
+    from lead.review.store import Store
+
+    saved = Store(tmp_path / "labels.sqlite")
+    assert saved.latest("lead-history", "title_abstract", 1) == {"AAAAAAAA": "include"}
+    _widget(at, "radio", "Pass").set_value(2).run()
+    assert not at.exception, at.exception
+    assert "Ada Lovelace" in _blob(at)
+
+    fresh = _run("from lead.explore.review_page import review\nreview()\n")
+    _widget(fresh, "selectbox", "Stage").select("full_text").run()
+    assert not fresh.exception, fresh.exception
+    assert "exposure precedes" in _blob(fresh).lower()
+
+
+def test_pass_two_waits_and_a_bad_exclude_is_not_saved(tmp_path, monkeypatch):
+    _labels(monkeypatch, tmp_path, _zotero(tmp_path, attachment=None))
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    _widget(at, "radio", "Pass").set_value(2).run()
+    assert "Pass 2 waits until pass 1 has a label." in _blob(at)
+
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    _widget(at, "selectbox", "Stage").select("full_text").run()
+    assert "No full text is attached." in _blob(at)
+    _widget(at, "radio", "Decision").set_value("exclude")
+    _widget(at, "text_area", "Why, in a sentence that names the criterion").input(
+        "No temporal order."
+    )
+    _widget(at, "button", "Save label").click().run()
+    assert "codebook" in _blob(at)
+
+
+def test_a_missing_library_is_an_error_and_the_actor_is_the_sso_header(
+    tmp_path, monkeypatch
+):
+    from lead.explore import review_page
+
+    monkeypatch.setattr(review_page, "_ZOTERO", str(tmp_path / "missing.sqlite"))
+    monkeypatch.setattr(review_page, "_LABELS", str(tmp_path / "labels.sqlite"))
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    assert at.error
+
+    class _Context:
+        def __init__(self, headers):
+            self.headers = headers
+
+    monkeypatch.setattr(
+        review_page.st,
+        "context",
+        _Context({"X-Auth-Request-Email": "kert@example"}),
+        raising=False,
+    )
+    assert review_page._actor() == "kert@example"
+    monkeypatch.setattr(
+        review_page.st,
+        "context",
+        _Context({"X-Auth-Request-User": "kert"}),
+        raising=False,
+    )
+    assert review_page._actor() == "kert"
+    monkeypatch.setattr(review_page.st, "context", _Context({}), raising=False)
+    assert review_page._actor() == "reviewer"
+
+
+def test_an_unreadable_attachment_is_a_warning(tmp_path, monkeypatch):
+    blob = tmp_path / "scan.txt"
+    blob.write_text("not html")
+    _labels(monkeypatch, tmp_path, _zotero(tmp_path, attachment=blob))
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    _widget(at, "selectbox", "Stage").select("full_text").run()
+    assert "no text came out of it" in _blob(at)
+
+
+def test_two_passes_that_disagree_are_counted(tmp_path, monkeypatch):
+    from lead.review.store import Store
+
+    library = _zotero(tmp_path, attachment=None)
+    labels = tmp_path / "labels.sqlite"
+    store = Store(labels)
+    for pass_n, decision in ((1, "include"), (2, "exclude")):
+        store.record(
+            zotero_key="AAAAAAAA",
+            project="lead-history",
+            stage="title_abstract",
+            pass_n=pass_n,
+            decision=decision,
+            reason="off-topic" if decision == "exclude" else "",
+            rationale=f"Pass {pass_n}.",
+            actor="kert",
+        )
+    from lead.explore import review_page
+
+    monkeypatch.setattr(review_page, "_ZOTERO", str(library))
+    monkeypatch.setattr(review_page, "_STORAGE", str(tmp_path / "storage"))
+    monkeypatch.setattr(review_page, "_LABELS", str(labels))
+    at = _run("from lead.explore.review_page import review\nreview()\n")
+    assert "disagree" in _blob(at)
+    assert "Nothing unlabeled on this pass." in _blob(at)
diff --git a/tests/lead/test_review.py b/tests/lead/test_review.py
index cb54708..44ce30a 100644
--- a/tests/lead/test_review.py
+++ b/tests/lead/test_review.py
@@ -3,11 +3,12 @@
 from __future__ import annotations
 
 import sqlite3
+import threading
 from pathlib import Path
 
 import pytest
 
-from lead.review.codes import GATE, PROJECTS, reason_codes
+from lead.review.codes import GATE, PROJECTS, criteria, reason_codes, reasons
 from lead.review.library import (
     connect,
     load_item,
@@ -186,3 +187,87 @@ def test_the_library_is_read_and_labels_do_not_tag_it(tmp_path: Path):
             ).fetchall()
         ]
     assert tags == ["project:lead-precursors"]
+
+
+def test_two_threads_each_open_the_library(tmp_path: Path):
+    library = tmp_path / "zotero.sqlite"
+    create_db(str(library)).close()
+    with Db(str(library)) as db:
+        kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
+        db.sync_tags(kept, ["project:lead-history"])
+        db.commit()
+    errors: list[BaseException] = []
+
+    def read() -> None:
+        con = connect(library)
+        try:
+            project_keys(con, "lead-history")
+        except BaseException as exc:
+            errors.append(exc)
+        finally:
+            con.close()
+
+    threads = [threading.Thread(target=read) for _ in range(4)]
+    for thread in threads:
+        thread.start()
+    for thread in threads:
+        thread.join()
+    assert errors == []
+
+
+def test_unknown_projects_and_blank_labels_are_refused(tmp_path: Path):
+    with pytest.raises(ValueError):
+        reason_codes("not-a-strand")
+    with pytest.raises(ValueError):
+        criteria("not-a-strand")
+    with pytest.raises(ValueError):
+        reasons("not-a-strand")
+    store = Store(tmp_path / "labels.sqlite")
+    kwargs = {
+        "zotero_key": "AAAAAAAA",
+        "project": "lead-history",
+        "stage": "title_abstract",
+        "pass_n": 1,
+        "decision": "include",
+        "rationale": "Names the predecessor.",
+        "actor": "kert",
+    }
+
+    def _bad(**overrides):
+        fields = dict(kwargs)
+        fields.update(overrides)
+        store.record(**fields)
+
+    with pytest.raises(ValueError, match="zotero key"):
+        _bad(zotero_key="  ")
+    with pytest.raises(ValueError, match="not-a-strand"):
+        _bad(project="not-a-strand")
+    with pytest.raises(ValueError, match="screen"):
+        _bad(stage="screen")
+    with pytest.raises(ValueError, match="3"):
+        _bad(pass_n=3)
+    with pytest.raises(ValueError, match="maybe"):
+        _bad(decision="maybe")
+
+
+def test_a_missing_item_and_a_dateless_attachment_stay_out_of_the_label(
+    tmp_path: Path,
+):
+    library = tmp_path / "zotero.sqlite"
+    create_db(str(library)).close()
+    paper = tmp_path / "paper.html"
+    paper.write_text("
    Exposure first.
    ")
+    with Db(str(library)) as db:
+        kept = db.create_item(TYPE_MAP["journalArticle"], key="AAAAAAAA")
+        db.set_field(kept, "date", "n.d.")
+        db.add_attachment(kept, path=str(paper), key="BBBBBBBB")
+        db.commit()
+    con = connect(library)
+    try:
+        with pytest.raises(KeyError):
+            load_item(con, "ZZZZZZZZ", tmp_path)
+        item = load_item(con, "AAAAAAAA", tmp_path)
+    finally:
+        con.close()
+    assert item.year == ""
+    assert item.attachment_paths == (paper,)
-- 
2.49.1