feat(cli): stack comments dockets/seal/unseal; extract skips sealed dockets, lazy bib lookup, --reattach (refs #615)

src/cli/comments.py

@@ -7,6 +7,9 @@ Phase 1 (this file):
     extract       — write combined.md per comment (PDF/DOCX → text)
     extract-ocr   — phase-2 stub (raises until tesseract integration lands)
     stats         — count comments by extraction status
+    dockets       — list known dockets (optionally discover new ones)
+    seal          — mark a docket complete so later stages skip it
+    unseal        — reopen a sealed docket
 """
 
 from __future__ import annotations
@@ -28,54 +31,55 @@ NOTE_TITLE = "Comment text"
 
 
 def _build_bib_helpers(use_bib: bool, attach: bool):
-    """Return ``(body_lookup, attach_callback)``.
+    """Return ``(body_lookup, attach_callback, store_or_None)``.
 
-    ``body_lookup(comment_id) -> str`` is called from worker threads and
+    ``body_lookup`` queries bib per comment id (``items.url`` is indexed)
-    returns the inline comment body from bib.sqlite (or "" if missing or
+    under a lock — worker threads share one connection. Nothing is
-    --no-bib). Backed by a comment_id → (item_key, body) cache built
+    loaded up front, so a run that extracts nothing reads nothing.
-    once at startup — bib has 164k+ rows and the URL → comment_id parse
-    can't use any index, so we materialize the whole map up front.
 
     ``attach_callback(comment_id, comment_dir) -> None`` runs on the main
     thread and registers a single ``"Comment text"`` note on the bib item
     containing combined.md rendered to HTML. Skips items that already have
-    one. Returns None when --no-bib or --no-attach.
+    one. Both ``attach_callback`` and the store are ``None`` when
+    --no-bib or --no-attach (store is ``None`` only for --no-bib, so
+    ``extract`` can still read seals with --no-attach).
     """
     if not use_bib:
-        return lambda _cid: "", None
+        return (lambda _cid: ""), None, None
 
     import sqlite3
-
+    import threading
-    from conf import path
-
-    db = str(path("db.bib"))
-    con = sqlite3.connect(db, check_same_thread=False)
-    try:
-        cache: dict[str, tuple[str, str]] = {}
-        for row in con.execute(
-            "SELECT key, url, abstract FROM items "
-            "WHERE url LIKE 'https://www.regulations.gov/comment/%'"
-        ):
-            cid = (row[1] or "").rsplit("/", 1)[-1]
-            if cid:
-                cache[cid] = (row[0], row[2] or "")
-    finally:
-        con.close()
-
-    def body_lookup(comment_id: str) -> str:
-        return cache.get(comment_id, ("", ""))[1]
-
-    if not attach:
-        return body_lookup, None
 
     from bib import connect
-    from rex.comments.render import render_combined_md
 
     store = connect()
+    if store._db_path == ":memory:":  # noqa: SLF001
+        con = store._con()  # noqa: SLF001 — tests only: a fresh connection would be an empty DB
+    else:
+        con = sqlite3.connect(store._db_path, check_same_thread=False)  # noqa: SLF001
+        con.row_factory = sqlite3.Row
+    lock = threading.Lock()
+
+    def _row(comment_id: str):
+        with lock:
+            return con.execute(
+                "SELECT key, abstract FROM items WHERE url = ?",
+                (f"https://www.regulations.gov/comment/{comment_id}",),
+            ).fetchone()
+
+    def body_lookup(comment_id: str) -> str:
+        row = _row(comment_id)
+        return (row["abstract"] or "") if row else ""
+
+    if not attach:
+        return body_lookup, None, store
+
+    from rex.comments.render import render_combined_md
+
     # Items that already have a NOTE_TITLE note — keeps re-runs idempotent.
     items_with_note: set[str] = {
         row[0]
-        for row in store._con().execute(
+        for row in con.execute(
             "SELECT i.key FROM notes n "
             "JOIN items i ON n.item_id = i.id "
             "WHERE n.title = ?",
@@ -84,10 +88,10 @@ def _build_bib_helpers(use_bib: bool, attach: bool):
     }
 
     def attach_callback(comment_id: str, comment_dir: Path) -> None:
-        info = cache.get(comment_id)
+        row = _row(comment_id)
-        if not info:
+        if not row:
             return  # comment not in bib
-        item_key = info[0]
+        item_key = row["key"]
         if item_key in items_with_note:
             return
         combined = comment_dir / "combined.md"
@@ -95,17 +99,18 @@ def _build_bib_helpers(use_bib: bool, attach: bool):
             return  # extraction must have failed; nothing to attach
         try:
             html = render_combined_md(combined.read_text(encoding="utf-8"))
+            with lock:
                 store.attach_note(item_key, html, title=NOTE_TITLE)
             items_with_note.add(item_key)
         except Exception as e:  # noqa: BLE001
             log.warning("attach_note failed for %s: %s", item_key, e)
 
-    return body_lookup, attach_callback
+    return body_lookup, attach_callback, store
 
 
 # Back-compat alias — older code/tests may still import this name.
 def _bib_lookup_factory(use_bib: bool):
-    body, _ = _build_bib_helpers(use_bib, attach=False)
+    body, _, _ = _build_bib_helpers(use_bib, attach=False)
     return body
 
 
@@ -133,6 +138,12 @@ def extract(
         help="Attach combined.md back to its bib item (so it flows to "
         "Zotero via `bib sync`). Implies --bib; no-op when --no-bib.",
     ),
+    reattach: bool = typer.Option(
+        False,
+        "--reattach",
+        help="Also attach combined.md notes for dirs that were already "
+        "current (repair).",
+    ),
 ) -> None:
     """Walk comment dirs and write combined.md (PDF/DOCX → text)."""
     from rex.comments.walker import walk_and_extract
@@ -141,7 +152,8 @@ def extract(
         typer.echo(f"root not found: {root}", err=True)
         raise typer.Exit(1)
 
-    lookup, attach_cb = _build_bib_helpers(use_bib, attach=use_bib and attach)
+    lookup, attach_cb, store = _build_bib_helpers(use_bib, attach=use_bib and attach)
+    skip = set(store.sealed_dockets()) if (store is not None and not force) else set()
     stats = walk_and_extract(
         root,
         inline_body_lookup=lookup,
@@ -150,7 +162,11 @@ def extract(
         workers=workers,
         force=force,
         on_extracted=attach_cb,
+        skip_dockets=skip,
+        reattach=reattach,
     )
+    if docket and docket in skip:
+        typer.echo(f"  {docket}: sealed; skipped — use --force")
     for k, v in stats.items():
         typer.echo(f"  {k}: {v}")
 
@@ -254,3 +270,86 @@ def stats(
     cols = ["comments", "attachments", "ok", "ocr_needed", "failed"]
     for k, v in zip(cols, rows, strict=True):
         typer.echo(f"  {k}: {v or 0}")
+
+
+@app.command()
+def dockets(
+    discover: bool = typer.Option(
+        False,
+        "--discover",
+        help="Populate rows for every reg-docket: tag in bib that has no "
+        "dockets row (one reg.gov listing call each).",
+    ),
+) -> None:
+    """List dockets: id, rule, close date, watermark, counts, seal."""
+    from bib import connect
+    from bib.regulations_gov import docket_counts
+
+    store = connect()
+    if discover:
+        from bib.regulations_gov import Client, _rule_tag_for_docket, discover_docket
+
+        tagged = [
+            r[0].split(":", 1)[1]
+            for r in store._con().execute(  # noqa: SLF001
+                "SELECT name FROM tags WHERE name LIKE 'reg-docket:%' ORDER BY name"
+            )
+        ]
+        missing = [d for d in tagged if store.docket_get(d) is None]
+        with Client(sleep=1.3) as api:
+            for d in missing:
+                row = discover_docket(
+                    api, d, rule_cms_id=_rule_tag_for_docket(store, d)
+                )
+                store.docket_upsert(row)
+                typer.echo(
+                    f"  discovered {d}: closes {row.comment_end_date or '?'} "
+                    f"rule {row.rule_cms_id or '?'}"
+                )
+    rows = store.dockets()
+    if not rows:
+        typer.echo("no dockets recorded — run `stack comments dockets --discover`")
+        return
+    typer.echo(
+        f"{'docket':<16}{'rule':<12}{'closes':<12}{'comments':>9}{'enriched':>9}  "
+        f"{'watermark':<20} state"
+    )
+    for d in rows:
+        c = docket_counts(store, d.id)
+        state = f"sealed {d.sealed_at[:10]} ({d.seal_reason})" if d.sealed else "open"
+        typer.echo(
+            f"{d.id:<16}{d.rule_cms_id:<12}{d.comment_end_date:<12}"
+            f"{c['comments']:>9}{c['enriched']:>9}  {d.pull_watermark[:19]:<20} {state}"
+        )
+
+
+@app.command()
+def seal(
+    docket: str = typer.Argument(..., help="reg.gov docket id"),
+    reason: str = typer.Option("manual", "--reason"),
+) -> None:
+    """Mark a docket complete: every stage skips it until unsealed or --force."""
+    from bib import connect
+    from bib.regulations_gov import docket_counts
+
+    store = connect()
+    if store.docket_get(docket) is None:
+        typer.echo(
+            f"unknown docket {docket} — run `stack comments dockets --discover` first"
+        )
+        raise typer.Exit(1)
+    store.docket_seal(docket, reason=reason, counts=docket_counts(store, docket))
+    typer.echo(f"sealed {docket} ({reason})")
+
+
+@app.command()
+def unseal(docket: str = typer.Argument(..., help="reg.gov docket id")) -> None:
+    """Reopen a sealed docket."""
+    from bib import connect
+
+    store = connect()
+    if store.docket_get(docket) is None:
+        typer.echo(f"unknown docket {docket}")
+        raise typer.Exit(1)
+    store.docket_unseal(docket)
+    typer.echo(f"unsealed {docket}")

tests/cli/test_comments_dockets.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import fitz
+from typer.testing import CliRunner
+
+from bib.dockets import Docket
+from bib.store import Store
+from cli.comments import app
+
+runner = CliRunner()
+
+
+def _store() -> Store:
+    return Store(":memory:", storage_dir="/tmp/nope")
+
+
+def _seed_dir(root: Path, docket: str) -> Path:
+    cdir = root / docket / f"{docket}-0001"
+    cdir.mkdir(parents=True)
+    doc = fitz.open()
+    doc.new_page().insert_text((50, 72), "Long body content " * 10)
+    doc.save(str(cdir / "attachment_1.pdf"))
+    doc.close()
+    return cdir
+
+
+@patch("bib.connect")
+def test_dockets_table_lists_rows(mc_connect):
+    s = _store()
+    s.docket_upsert(
+        Docket(
+            id="CMS-2019-0111",
+            rule_cms_id="CMS-1693-P",
+            comment_end_date="2019-09-27",
+            sealed_at="2020-01-01T00:00:00Z",
+            seal_reason="manual",
+        )
+    )
+    s.docket_upsert(
+        Docket(
+            id="CMS-2026-2377", rule_cms_id="CMS-1848-P", comment_end_date="2026-09-14"
+        )
+    )
+    mc_connect.return_value = s
+    result = runner.invoke(app, ["dockets"])
+    assert result.exit_code == 0, result.output
+    assert "CMS-2019-0111" in result.output and "sealed" in result.output
+    assert "CMS-2026-2377" in result.output and "open" in result.output
+
+
+@patch("bib.connect")
+def test_seal_and_unseal(mc_connect):
+    s = _store()
+    s.docket_upsert(Docket(id="CMS-2019-0111"))
+    mc_connect.return_value = s
+    assert (
+        runner.invoke(
+            app, ["seal", "CMS-2019-0111", "--reason", "historical"]
+        ).exit_code
+        == 0
+    )
+    assert s.docket_get("CMS-2019-0111").seal_reason == "historical"
+    assert runner.invoke(app, ["unseal", "CMS-2019-0111"]).exit_code == 0
+    assert not s.docket_get("CMS-2019-0111").sealed
+
+
+@patch("bib.connect")
+def test_seal_unknown_docket_fails(mc_connect):
+    mc_connect.return_value = _store()
+    result = runner.invoke(app, ["seal", "CMS-0000-0000"])
+    assert result.exit_code == 1
+    assert "unknown docket" in result.output
+
+
+@patch("bib.regulations_gov.discover_docket")
+@patch("bib.regulations_gov.Client")
+@patch("bib.connect")
+def test_dockets_discover_populates_from_tags(mc_connect, mc_client, mc_disc):
+    from bib.item import Source
+
+    s = _store()
+    it = Source(title="c", url="https://www.regulations.gov/comment/CMS-2019-0111-1")
+    it.add_tag("reg-docket:CMS-2019-0111")
+    it.add_tag("rule:CMS-1693-P")
+    s.create(it)
+    mc_connect.return_value = s
+    api = MagicMock()
+    api.__enter__ = MagicMock(return_value=api)
+    api.__exit__ = MagicMock(return_value=False)
+    mc_client.return_value = api
+    mc_disc.return_value = Docket(
+        id="CMS-2019-0111",
+        rule_cms_id="CMS-1693-P",
+        fr_object_id="o",
+        comment_end_date="2019-09-27",
+    )
+
+    result = runner.invoke(app, ["dockets", "--discover"])
+
+    assert result.exit_code == 0, result.output
+    mc_disc.assert_called_once_with(api, "CMS-2019-0111", rule_cms_id="CMS-1693-P")
+    assert s.docket_get("CMS-2019-0111").comment_end_date == "2019-09-27"
+
+
+@patch("bib.connect")
+def test_extract_skips_sealed_docket(mc_connect, tmp_path: Path):
+    s = _store()
+    s.docket_upsert(
+        Docket(
+            id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z", seal_reason="manual"
+        )
+    )
+    mc_connect.return_value = s
+    cdir = _seed_dir(tmp_path, "CMS-2024-0001")
+    result = runner.invoke(
+        app, ["extract", "--root", str(tmp_path), "--workers", "1", "--no-attach"]
+    )
+    assert result.exit_code == 0, result.output
+    assert "skipped_sealed: 1" in result.output
+    assert not (cdir / "combined.md").exists()
+
+
+@patch("bib.connect")
+def test_extract_force_runs_sealed_docket(mc_connect, tmp_path: Path):
+    s = _store()
+    s.docket_upsert(Docket(id="CMS-2024-0001", sealed_at="2025-01-01T00:00:00Z"))
+    mc_connect.return_value = s
+    cdir = _seed_dir(tmp_path, "CMS-2024-0001")
+    result = runner.invoke(
+        app,
+        [
+            "extract",
+            "--root",
+            str(tmp_path),
+            "--workers",
+            "1",
+            "--no-attach",
+            "--force",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert (cdir / "combined.md").is_file()