feat(comments): stack comments CLI + integration smoke test (refs #253)

- src/cli/comments.py exposes:
    stack comments extract     — walk + write combined.md (default
                                 root .state/comments/, parallel via
                                 walker, --docket/--limit/--workers
                                 /--force/--bib/--no-bib).
    stack comments extract-ocr — phase-2 stub (raises typer.Exit(2)
                                 with a clear message).
    stack comments stats       — rebuild index, print per-status counts.
- Inline body lookup hits bib.sqlite items.abstract by comment_id;
  --no-bib disables (useful in tests / when bib isn't seeded).
- Integration test exercises one real .state/comments/{docket}/{cid}/
  dir if present (skipped on CI without state).

Closes the v1 implementation surface for #253. Live smoke run + #253
issue update happen as the operational follow-up.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/cli/__init__.py

@@ -14,6 +14,7 @@ import typer
 
 from cli.api import app as api_app
 from cli.bib import app as bib_app
+from cli.comments import app as comments_app
 from cli.db import app as db_app
 from cli.docs import app as docs_app
 from cli.generate import app as generate_app
@@ -40,6 +41,11 @@ app.command()(health)
 app.add_typer(load_app, name="load", help="Ingest data (CCLF, BCDA, seeds).")
 app.add_typer(generate_app, name="generate", help="Regenerate code artefacts.")
 app.add_typer(bib_app, name="bib", help="Bibliography operations.")
+app.add_typer(
+    comments_app,
+    name="comments",
+    help="CMS rulemaking comment text extraction & analysis.",
+)
 app.add_typer(lake_app, name="lake", help="Lakehouse schema and data.")
 app.add_typer(db_app, name="db", help="DuckDB utilities.")
 app.add_typer(docs_app, name="docs", help="Documentation generation.")

src/cli/comments.py

@@ -0,0 +1,121 @@
+"""stack comments — text extraction + analysis for CMS rulemaking comments.
+
+Operates on per-comment dirs created by `bib backfill-comments`:
+    .state/comments/{docket_id}/{comment_id}/{attachment_*, combined.md}
+
+Phase 1 (this file):
+    extract       — write combined.md per comment (PDF/DOCX → text)
+    extract-ocr   — phase-2 stub (raises until tesseract integration lands)
+    stats         — count comments by extraction status
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+app = typer.Typer(no_args_is_help=True)
+
+_DEFAULT_ROOT = Path(".state/comments")
+
+
+def _bib_lookup_factory(use_bib: bool):
+    """Return a callable comment_id -> inline body. Empty if --no-bib."""
+    if not use_bib:
+        return lambda _cid: ""
+
+    from bib import connect
+
+    store = connect()
+    con = store._con()  # noqa: SLF001
+
+    def lookup(comment_id: str) -> str:
+        row = con.execute(
+            "SELECT abstract FROM items WHERE key = ?", (comment_id,)
+        ).fetchone()
+        return (row[0] if row and row[0] else "") or ""
+
+    return lookup
+
+
+@app.command()
+def extract(
+    root: Path = typer.Option(_DEFAULT_ROOT, "--root", help="Comments root dir."),
+    docket: str = typer.Option(
+        None, "--docket", help="Limit to one docket (e.g. CMS-2023-0121)."
+    ),
+    limit: int = typer.Option(
+        0, "--limit", help="Cap dirs processed this run. 0 = no cap."
+    ),
+    workers: int = typer.Option(8, "--workers", help="Concurrent extraction threads."),
+    force: bool = typer.Option(
+        False, "--force", help="Re-extract dirs that already have combined.md."
+    ),
+    use_bib: bool = typer.Option(
+        True,
+        "--bib/--no-bib",
+        help="Look up inline body from bib.sqlite items.abstract.",
+    ),
+) -> None:
+    """Walk comment dirs and write combined.md (PDF/DOCX → text)."""
+    from rex.comments.walker import walk_and_extract
+
+    if not root.is_dir():
+        typer.echo(f"root not found: {root}", err=True)
+        raise typer.Exit(1)
+
+    lookup = _bib_lookup_factory(use_bib)
+    stats = walk_and_extract(
+        root,
+        inline_body_lookup=lookup,
+        docket=docket,
+        limit=limit or None,
+        workers=workers,
+        force=force,
+    )
+    for k, v in stats.items():
+        typer.echo(f"  {k}: {v}")
+
+
+@app.command(name="extract-ocr")
+def extract_ocr(
+    root: Path = typer.Option(_DEFAULT_ROOT, "--root"),  # noqa: ARG001
+) -> None:
+    """OCR pass for ocr_needed attachments. Phase 2 — not yet implemented."""
+    typer.echo(
+        "extract-ocr is phase 2 — not yet implemented. "
+        "Tracks: see #253 follow-up. "
+        "Today, ocr_needed attachments stay flagged in combined.md frontmatter.",
+        err=True,
+    )
+    raise typer.Exit(2)
+
+
+@app.command()
+def stats(
+    root: Path = typer.Option(_DEFAULT_ROOT, "--root", help="Comments root dir."),
+) -> None:
+    """Rebuild the index and print per-status counts."""
+    import duckdb
+
+    from rex.comments.view import rebuild_index, register
+
+    if not root.is_dir():
+        typer.echo(f"root not found: {root}", err=True)
+        raise typer.Exit(1)
+
+    rebuild_index(root)
+    con = duckdb.connect(":memory:")
+    register(con, root=root)
+    rows = con.execute(
+        "SELECT COUNT(*) AS comments, "
+        "SUM(n_attachments) AS attachments, "
+        "SUM(n_ok) AS ok, "
+        "SUM(n_ocr_needed) AS ocr_needed, "
+        "SUM(n_failed) AS failed "
+        "FROM comments_index"
+    ).fetchone()
+    cols = ["comments", "attachments", "ok", "ocr_needed", "failed"]
+    for k, v in zip(cols, rows, strict=True):
+        typer.echo(f"  {k}: {v or 0}")

tests/cli/test_comments.py

@@ -0,0 +1,59 @@
+"""CLI: stack comments {extract, extract-ocr, stats}."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import fitz
+from typer.testing import CliRunner
+
+from cli.comments import app
+
+runner = CliRunner()
+
+
+def _seed(root: Path) -> Path:
+    cdir = root / "CMS-2024-0001" / "CMS-2024-0001-0001"
+    cdir.mkdir(parents=True)
+    doc = fitz.open()
+    doc.new_page().insert_text((50, 72), "Long body content " * 10)
+    doc.save(str(cdir / "attachment_1.pdf"))
+    doc.close()
+    return cdir
+
+
+def test_extract_writes_combined(tmp_path: Path):
+    cdir = _seed(tmp_path)
+    result = runner.invoke(
+        app,
+        [
+            "extract",
+            "--root",
+            str(tmp_path),
+            "--no-bib",  # skip bib.sqlite lookup in tests
+            "--workers",
+            "1",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert (cdir / "combined.md").is_file()
+    assert "written" in result.output.lower()
+
+
+def test_extract_ocr_is_stub(tmp_path: Path):
+    result = runner.invoke(app, ["extract-ocr", "--root", str(tmp_path)])
+    assert result.exit_code != 0
+    assert (
+        "phase 2" in result.output.lower() or "not implemented" in result.output.lower()
+    )
+
+
+def test_stats_after_extract(tmp_path: Path):
+    _seed(tmp_path)
+    runner.invoke(
+        app,
+        ["extract", "--root", str(tmp_path), "--no-bib", "--workers", "1"],
+    )
+    result = runner.invoke(app, ["stats", "--root", str(tmp_path)])
+    assert result.exit_code == 0, result.output
+    assert "1" in result.output  # at least one comment counted

tests/rex/comments/test_integration.py

@@ -0,0 +1,58 @@
+"""Smoke test against a real .state/comments/ dir if available.
+
+Skipped on CI where state isn't present. Exercises the PyMuPDF code path
+on real CMS comment PDFs (formatting, headers, multi-page) before
+running the full backfill.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import pytest
+
+from rex.comments import extract_comment, parse_combined
+
+_LIVE = Path(".state/comments")
+
+
+def _pick_real_comment_dir() -> Path | None:
+    """Find one .state/comments/{docket}/{cid}/ dir with at least one PDF."""
+    if not _LIVE.is_dir():
+        return None
+    for docket in sorted(_LIVE.iterdir()):
+        if not docket.is_dir() or not docket.name.startswith("CMS-"):
+            continue
+        for cdir in sorted(docket.iterdir()):
+            if not cdir.is_dir():
+                continue
+            if any(p.suffix.lower() == ".pdf" for p in cdir.iterdir()):
+                return cdir
+    return None
+
+
+def test_real_comment_extracts(tmp_path: Path):
+    src = _pick_real_comment_dir()
+    if src is None:
+        pytest.skip(".state/comments/ not present")
+
+    # Copy to tmp so we don't write combined.md into the live tree.
+    dst_docket = tmp_path / src.parent.name
+    dst_docket.mkdir(parents=True)
+    dst = dst_docket / src.name
+    shutil.copytree(src, dst)
+    # Strip any pre-existing combined.md so we re-run extraction.
+    pre = dst / "combined.md"
+    if pre.exists():
+        pre.unlink()
+
+    out = extract_comment(dst, inline_body="(no inline body in this fixture)")
+    fm, body = parse_combined(out.read_text())
+    assert fm["comment_id"] == src.name
+    assert fm["docket_id"] == src.parent.name
+    assert fm["attachments"], "expected at least one attachment"
+    # At least one of the attachments should be ok or ocr_needed (not failed).
+    statuses = {a["status"] for a in fm["attachments"]}
+    assert statuses & {"ok", "ocr_needed"}, f"all failed: {statuses}"
+    assert "## Inline comment" in body