homelab/stack
1
0
Fork 0
Code Issues 110 Pull Requests 1 Actions 4 Packages Projects 1 Releases Wiki Activity

feat(cli): stack comments dockets/seal/unseal; extract skips sealed dockets, lazy bib lookup, --reattach (refs #615)

Browse Source
This commit is contained in:
kert
2026-09-08 13:17:25 -04:00
parent e7287c827f
commit 8b81beda14
2 changed files with 283 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
src/cli/comments.py
View File

@@ -7,6 +7,9 @@ Phase 1 (this file):
extract — write combined.md per comment (PDF/DOCX → text)
extract-ocr — phase-2 stub (raises until tesseract integration lands)
stats — count comments by extraction status
dockets — list known dockets (optionally discover new ones)
seal — mark a docket complete so later stages skip it
unseal — reopen a sealed docket
"""
from __future__ import annotations
@@ -28,54 +31,55 @@ NOTE_TITLE = "Comment text"
def _build_bib_helpers(use_bib: bool, attach: bool):
"""Return ``(body_lookup, attach_callback)``.
"""Return ``(body_lookup, attach_callback, store_or_None)``.
``body_lookup(comment_id) -> str`` is called from worker threads and
returns the inline comment body from bib.sqlite (or "" if missing or
--no-bib). Backed by a comment_id → (item_key, body) cache built
once at startup — bib has 164k+ rows and the URL → comment_id parse
can't use any index, so we materialize the whole map up front.
``body_lookup`` queries bib per comment id (``items.url`` is indexed)
under a lock — worker threads share one connection. Nothing is
loaded up front, so a run that extracts nothing reads nothing.
``attach_callback(comment_id, comment_dir) -> None`` runs on the main
thread and registers a single ``"Comment text"`` note on the bib item
containing combined.md rendered to HTML. Skips items that already have
one. Returns None when --no-bib or --no-attach.
one. Both ``attach_callback`` and the store are ``None`` when
--no-bib or --no-attach (store is ``None`` only for --no-bib, so
``extract`` can still read seals with --no-attach).
"""
if not use_bib:
return lambda _cid: "", None
return (lambda _cid: ""), None, None
import sqlite3
from conf import path
db = str(path("db.bib"))
con = sqlite3.connect(db, check_same_thread=False)
try:
cache: dict[str, tuple[str, str]] = {}
for row in con.execute(
"SELECT key, url, abstract FROM items "
"WHERE url LIKE 'https://www.regulations.gov/comment/%'"
):
cid = (row[1] or "").rsplit("/", 1)[-1]
if cid:
cache[cid] = (row[0], row[2] or "")
finally:
con.close()
def body_lookup(comment_id: str) -> str:
return cache.get(comment_id, ("", ""))[1]
if not attach:
return body_lookup, None
import threading
from bib import connect
from rex.comments.render import render_combined_md
store = connect()
if store._db_path == ":memory:": # noqa: SLF001
con = store._con() # noqa: SLF001 — tests only: a fresh connection would be an empty DB
else:
con = sqlite3.connect(store._db_path, check_same_thread=False) # noqa: SLF001
con.row_factory = sqlite3.Row
lock = threading.Lock()
def _row(comment_id: str):
with lock:
return con.execute(
"SELECT key, abstract FROM items WHERE url = ?",
(f"https://www.regulations.gov/comment/{comment_id}",),
).fetchone()
def body_lookup(comment_id: str) -> str:
row = _row(comment_id)
return (row["abstract"] or "") if row else ""
if not attach:
return body_lookup, None, store
from rex.comments.render import render_combined_md
# Items that already have a NOTE_TITLE note — keeps re-runs idempotent.
items_with_note: set[str] = {
row[0]
for row in store._con().execute(
for row in con.execute(
"SELECT i.key FROM notes n "
"JOIN items i ON n.item_id = i.id "
"WHERE n.title = ?",
@@ -84,10 +88,10 @@ def _build_bib_helpers(use_bib: bool, attach: bool):
}
def attach_callback(comment_id: str, comment_dir: Path) -> None:
info = cache.get(comment_id)
if not info:
row = _row(comment_id)
if not row:
return # comment not in bib
item_key = info[0]
item_key = row["key"]
if item_key in items_with_note:
return
combined = comment_dir / "combined.md"
@@ -95,17 +99,18 @@ def _build_bib_helpers(use_bib: bool, attach: bool):
return # extraction must have failed; nothing to attach
try:
html = render_combined_md(combined.read_text(encoding="utf-8"))
store.attach_note(item_key, html, title=NOTE_TITLE)
with lock:
store.attach_note(item_key, html, title=NOTE_TITLE)
items_with_note.add(item_key)
except Exception as e: # noqa: BLE001
log.warning("attach_note failed for %s: %s", item_key, e)
return body_lookup, attach_callback
return body_lookup, attach_callback, store
# Back-compat alias — older code/tests may still import this name.
def _bib_lookup_factory(use_bib: bool):
body, _ = _build_bib_helpers(use_bib, attach=False)
body, _, _ = _build_bib_helpers(use_bib, attach=False)
return body
@@ -133,6 +138,12 @@ def extract(
help="Attach combined.md back to its bib item (so it flows to "
"Zotero via `bib sync`). Implies --bib; no-op when --no-bib.",
),
reattach: bool = typer.Option(
False,
"--reattach",
help="Also attach combined.md notes for dirs that were already "
"current (repair).",
),
) -> None:
"""Walk comment dirs and write combined.md (PDF/DOCX → text)."""
from rex.comments.walker import walk_and_extract
@@ -141,7 +152,8 @@ def extract(
typer.echo(f"root not found: {root}", err=True)
raise typer.Exit(1)
lookup, attach_cb = _build_bib_helpers(use_bib, attach=use_bib and attach)
lookup, attach_cb, store = _build_bib_helpers(use_bib, attach=use_bib and attach)
skip = set(store.sealed_dockets()) if (store is not None and not force) else set()
stats = walk_and_extract(
root,
inline_body_lookup=lookup,
@@ -150,7 +162,11 @@ def extract(
workers=workers,
force=force,
on_extracted=attach_cb,
skip_dockets=skip,
reattach=reattach,
)
if docket and docket in skip:
typer.echo(f" {docket}: sealed; skipped — use --force")
for k, v in stats.items():
typer.echo(f" {k}: {v}")
@@ -254,3 +270,86 @@ def stats(
cols = ["comments", "attachments", "ok", "ocr_needed", "failed"]
for k, v in zip(cols, rows, strict=True):
typer.echo(f" {k}: {v or 0}")
@app.command()
def dockets(
discover: bool = typer.Option(
False,
"--discover",
help="Populate rows for every reg-docket: tag in bib that has no "
"dockets row (one reg.gov listing call each).",
),
) -> None:
"""List dockets: id, rule, close date, watermark, counts, seal."""
from bib import connect
from bib.regulations_gov import docket_counts
store = connect()
if discover:
from bib.regulations_gov import Client, _rule_tag_for_docket, discover_docket
tagged = [
r[0].split(":", 1)[1]
for r in store._con().execute( # noqa: SLF001
"SELECT name FROM tags WHERE name LIKE 'reg-docket:%' ORDER BY name"
)
]
missing = [d for d in tagged if store.docket_get(d) is None]
with Client(sleep=1.3) as api:
for d in missing:
row = discover_docket(
api, d, rule_cms_id=_rule_tag_for_docket(store, d)
)
store.docket_upsert(row)
typer.echo(
f" discovered {d}: closes {row.comment_end_date or '?'} "
f"rule {row.rule_cms_id or '?'}"
)
rows = store.dockets()
if not rows:
typer.echo("no dockets recorded — run `stack comments dockets --discover`")
return
typer.echo(
f"{'docket':<16}{'rule':<12}{'closes':<12}{'comments':>9}{'enriched':>9} "
f"{'watermark':<20} state"
)
for d in rows:
c = docket_counts(store, d.id)
state = f"sealed {d.sealed_at[:10]} ({d.seal_reason})" if d.sealed else "open"
typer.echo(
f"{d.id:<16}{d.rule_cms_id:<12}{d.comment_end_date:<12}"
f"{c['comments']:>9}{c['enriched']:>9} {d.pull_watermark[:19]:<20} {state}"
)
@app.command()
def seal(
docket: str = typer.Argument(..., help="reg.gov docket id"),
reason: str = typer.Option("manual", "--reason"),
) -> None:
"""Mark a docket complete: every stage skips it until unsealed or --force."""
from bib import connect
from bib.regulations_gov import docket_counts
store = connect()
if store.docket_get(docket) is None:
typer.echo(
f"unknown docket {docket} — run `stack comments dockets --discover` first"
)
raise typer.Exit(1)
store.docket_seal(docket, reason=reason, counts=docket_counts(store, docket))
typer.echo(f"sealed {docket} ({reason})")
@app.command()
def unseal(docket: str = typer.Argument(..., help="reg.gov docket id")) -> None:
"""Reopen a sealed docket."""
from bib import connect
store = connect()
if store.docket_get(docket) is None:
typer.echo(f"unknown docket {docket}")
raise typer.Exit(1)
store.docket_unseal(docket)
typer.echo(f"unsealed {docket}")

145
tests/cli/test_comments_dockets.py Normal file
View File

@@ -0,0 +1,145 @@
from __future__ import annotations
from pathlib import Path
from unittest.mock import MagicMock, patch
import fitz
from typer.testing import CliRunner
from bib.dockets import Docket
from bib.store import Store
from cli.comments import app
runner = CliRunner()
def _store() -> Store:
return Store(":memory:", storage_dir="/tmp/nope")
def _seed_dir(root: Path, docket: str) -> Path:
cdir = root / docket / f"{docket}-0001"
cdir.mkdir(parents=True)
doc = fitz.open()
doc.new_page().insert_text((50, 72), "Long body content " * 10)
doc.save(str(cdir / "attachment_1.pdf"))
doc.close()
return cdir
@patch("bib.connect")
def test_dockets_table_lists_rows(mc_connect):
s = _store()
s.docket_upsert(
Docket(
id="CMS-2019-0111",
rule_cms_id="CMS-1693-P",
comment_end_date="2019-09-27",
sealed_at="2020-01-01T00:00:00Z",
seal_reason="manual",
)
)
s.docket_upsert(
Docket(
id="CMS-2026-2377", rule_cms_id="CMS-1848-P", comment_end_date="2026-09-14"
)
)
mc_connect.return_value = s
result = runner.invoke(app, ["dockets"])
assert result.exit_code == 0, result.output
assert "CMS-2019-0111" in result.output and "sealed" in result.output
assert "CMS-2026-2377" in result.output and "open" in result.output
@patch("bib.connect")
def test_seal_and_unseal(mc_connect):
s = _store()
s.docket_upsert(Docket(id="CMS-2019-0111"))
mc_connect.return_value = s
assert (
runner.invoke(
app, ["seal", "CMS-2019-0111", "--reason", "historical"]
).exit_code
== 0
)
assert s.docket_get("CMS-2019-0111").seal_reason == "historical"
assert runner.invoke(app, ["unseal", "CMS-2019-0111"]).exit_code == 0
assert not s.docket_get("CMS-2019-0111").sealed
@patch("bib.connect")
def test_seal_unknown_docket_fails(mc_connect):
mc_connect.return_value = _store()
result = runner.invoke(app, ["seal", "CMS-0000-0000"])
assert result.exit_code == 1
assert "unknown docket" in result.output
@patch("bib.regulations_gov.discover_docket")
@patch("bib.regulations_gov.Client")
@patch("bib.connect")
def test_dockets_discover_populates_from_tags(mc_connect, mc_client, mc_disc):
from bib.item import Source
s = _store()
it = Source(title="c", url="https://www.regulations.gov/comment/CMS-2019-0111-1")
it.add_tag("reg-docket:CMS-2019-0111")
it.add_tag("rule:CMS-1693-P")
s.create(it)
mc_connect.return_value = s
api = MagicMock()
api.__enter__ = MagicMock(return_value=api)
api.__exit__ = MagicMock(return_value=False)
mc_client.return_value = api
mc_disc.return_value = Docket(
id="CMS-2019-0111",
rule_cms_id="CMS-1693-P",
fr_object_id="o",
comment_end_date="2019-09-27",
)
result = runner.invoke(app, ["dockets", "--discover"])
assert result.exit_code == 0, result.output
mc_disc.assert_called_once_with(api, "CMS-2019-0111", rule_cms_id="CMS-1693-P")
assert s.docket_get("CMS-2019-0111").comment_end_date == "2019-09-27"
@patch("bib.connect")
def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path):
s = _store()
s.docket_upsert(
Docket(
id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z", seal_reason="manual"
)
)
mc_connect.return_value = s
cdir = _seed_dir(tmp_path, "CMS-2024-0001")
result = runner.invoke(
app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"]
)
assert result.exit_code == 0, result.output
assert "skipped_sealed: 1" in result.output
assert not (cdir / "combined.md").exists()
@patch("bib.connect")
def test_extract_force_runs_sealed_docket(mc_connect, tmp_path: Path):
s = _store()
s.docket_upsert(Docket(id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z"))
mc_connect.return_value = s
cdir = _seed_dir(tmp_path, "CMS-2024-0001")
result = runner.invoke(
app,
[
"extract",
"--root",
str(tmp_path),
"--workers",
"1",
"--no-attach",
"--force",
],
)
assert result.exit_code == 0, result.output
assert (cdir / "combined.md").is_file()