d69d5fb3aa
- fetch-pfs-comments upserts the rule item before the per-docket resolve loop, so a CMS id that never resolves to a reg.gov docket no longer loses the rule entirely (the upsert used to live only inside the per-docket loop, guarded by a successful resolve). - --docket's help text and the command docstring now say precisely when the "no API calls" guarantee holds: only once a dockets row already exists for the CMS id. A CMS id with no row yet still costs one FR rule-metadata fetch plus one resolve_docket call. - extract_ocr closes the read connection _bib_lookup_factory opens (via the same `.close` attribute contract extract() already honours in its own finally), instead of leaking it. Adds coverage: the unresolvable-docket upsert (and rewires test_no_docket off a MagicMock rule, now that the rule is upserted unconditionally), --reattach's repair behavior at the CLI level, and the extract_ocr connection close.
223 lines
7.1 KiB
Python
223 lines
7.1 KiB
Python
"""CLI: stack comments {extract, extract-ocr, stats}."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import fitz
|
|
from typer.testing import CliRunner
|
|
|
|
from cli.comments import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def _seed(root: Path) -> Path:
|
|
cdir = root / "CMS-2024-0001" / "CMS-2024-0001-0001"
|
|
cdir.mkdir(parents=True)
|
|
doc = fitz.open()
|
|
doc.new_page().insert_text((50, 72), "Long body content " * 10)
|
|
doc.save(str(cdir / "attachment_1.pdf"))
|
|
doc.close()
|
|
return cdir
|
|
|
|
|
|
def test_extract_writes_combined(tmp_path: Path):
|
|
cdir = _seed(tmp_path)
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"extract",
|
|
"--root",
|
|
str(tmp_path),
|
|
"--no-bib", # skip bib.sqlite lookup in tests
|
|
"--workers",
|
|
"1",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert (cdir / "combined.md").is_file()
|
|
assert "written" in result.output.lower()
|
|
|
|
|
|
def test_extract_ocr_empty_queue(tmp_path: Path):
|
|
# Implemented in #663 — an empty root reports a zero queue and exits
|
|
# cleanly without loading the OCR engine or the bib lookup.
|
|
result = runner.invoke(app, ["extract-ocr", "--root", str(tmp_path)])
|
|
assert result.exit_code == 0
|
|
assert "ocr queue: 0" in result.output
|
|
|
|
|
|
def test_extract_ocr_closes_bib_lookup_connection(tmp_path: Path, monkeypatch):
|
|
"""refs #680: _bib_lookup_factory opens its own read connection to
|
|
bib.sqlite (via a `.close` attribute on the returned callable —
|
|
the same contract `extract()` honours in its `finally`); extract_ocr
|
|
used to never call it, leaking the connection."""
|
|
cdir = tmp_path / "CMS-2024-0001" / "CMS-2024-0001-0001"
|
|
cdir.mkdir(parents=True)
|
|
(cdir / "combined.md").write_text(
|
|
"---\nattachments:\n- status: ocr_needed\n---\nbody\n"
|
|
)
|
|
|
|
closed = {"called": False}
|
|
|
|
def fake_lookup(comment_id: str) -> str:
|
|
return ""
|
|
|
|
fake_lookup.close = lambda: closed.__setitem__("called", True)
|
|
|
|
monkeypatch.setattr("cli.comments._bib_lookup_factory", lambda use_bib: fake_lookup)
|
|
monkeypatch.setattr("rex.comments.combine.extract_comment", lambda *a, **k: None)
|
|
monkeypatch.setattr("rex.comments.ocr.RapidOcrEngine", lambda: lambda p: "")
|
|
|
|
result = runner.invoke(app, ["extract-ocr", "--root", str(tmp_path)])
|
|
assert result.exit_code == 0, result.output
|
|
assert closed["called"] is True
|
|
|
|
|
|
def test_stats_after_extract(tmp_path: Path):
|
|
_seed(tmp_path)
|
|
runner.invoke(
|
|
app,
|
|
["extract", "--root", str(tmp_path), "--no-bib", "--workers", "1"],
|
|
)
|
|
result = runner.invoke(app, ["stats", "--root", str(tmp_path)])
|
|
assert result.exit_code == 0, result.output
|
|
assert "1" in result.output # at least one comment counted
|
|
|
|
|
|
def test_extract_attaches_combined_md_as_note(tmp_path: Path, monkeypatch):
|
|
"""Extract writes combined.md and per-attachment sibling MDs to disk.
|
|
Only one Zotero-bound artifact: a single bib note titled 'Comment text'
|
|
whose content is the rendered-HTML of combined.md. No file attachments."""
|
|
from bib import connect
|
|
from bib.item import Source
|
|
|
|
bib_db = tmp_path / "bib.sqlite"
|
|
monkeypatch.setattr("conf.path", lambda _: bib_db)
|
|
|
|
cdir = _seed(tmp_path)
|
|
comment_id = cdir.name # CMS-2024-0001-0001
|
|
|
|
store = connect(str(bib_db))
|
|
item = Source(
|
|
title=f"Comment {comment_id}",
|
|
url=f"https://www.regulations.gov/comment/{comment_id}",
|
|
)
|
|
item_key = store.upsert(item)
|
|
store.close()
|
|
|
|
result = runner.invoke(app, ["extract", "--root", str(tmp_path), "--workers", "1"])
|
|
assert result.exit_code == 0, result.output
|
|
|
|
# Disk siblings still written (kept for non-Zotero corpus consumers)
|
|
assert (cdir / "combined.md").is_file()
|
|
assert (cdir / "attachment_1.pdf.md").is_file()
|
|
|
|
def md_attachments():
|
|
store = connect(str(bib_db))
|
|
rows = list(
|
|
store._con().execute(
|
|
"SELECT a.filename FROM attachments a "
|
|
"JOIN items i ON a.item_id=i.id "
|
|
"WHERE i.key=? AND a.filename LIKE '%.md'",
|
|
(item_key,),
|
|
)
|
|
)
|
|
store.close()
|
|
return rows
|
|
|
|
def notes():
|
|
store = connect(str(bib_db))
|
|
rows = list(
|
|
store._con().execute(
|
|
"SELECT n.title, n.content FROM notes n "
|
|
"JOIN items i ON n.item_id=i.id WHERE i.key=?",
|
|
(item_key,),
|
|
)
|
|
)
|
|
store.close()
|
|
return rows
|
|
|
|
# No file attachments — old behavior is gone.
|
|
assert md_attachments() == []
|
|
|
|
# Exactly one note, rendered HTML.
|
|
rows = notes()
|
|
assert len(rows) == 1
|
|
title, content = rows[0]
|
|
assert title == "Comment text"
|
|
assert "<h1>" in content or "<h2>" in content
|
|
assert "Long body content" in content # the seeded PDF text shows up
|
|
|
|
# Idempotent on re-run
|
|
runner.invoke(app, ["extract", "--root", str(tmp_path), "--workers", "1"])
|
|
assert len(notes()) == 1
|
|
assert md_attachments() == []
|
|
|
|
|
|
def test_extract_reattach_repairs_missing_note_for_current_dir(
|
|
tmp_path: Path, monkeypatch
|
|
):
|
|
"""refs #680: --reattach at the CLI level is the repair path — a dir
|
|
already "current" (combined.md up to date) is normally skipped
|
|
without touching bib at all, so a note lost from under it (e.g. a
|
|
bad Zotero sync) never comes back on a plain re-run. --reattach
|
|
re-fires the attach callback for every skipped dir."""
|
|
from bib import connect
|
|
from bib.item import Source
|
|
|
|
bib_db = tmp_path / "bib.sqlite"
|
|
monkeypatch.setattr("conf.path", lambda _: bib_db)
|
|
|
|
cdir = _seed(tmp_path)
|
|
comment_id = cdir.name
|
|
|
|
store = connect(str(bib_db))
|
|
item = Source(
|
|
title=f"Comment {comment_id}",
|
|
url=f"https://www.regulations.gov/comment/{comment_id}",
|
|
)
|
|
item_key = store.upsert(item)
|
|
store.close()
|
|
|
|
def note_count() -> int:
|
|
store = connect(str(bib_db))
|
|
n = (
|
|
store._con()
|
|
.execute(
|
|
"SELECT count(*) FROM notes n JOIN items i ON n.item_id = i.id "
|
|
"WHERE i.key = ?",
|
|
(item_key,),
|
|
)
|
|
.fetchone()[0]
|
|
)
|
|
store.close()
|
|
return n
|
|
|
|
result = runner.invoke(app, ["extract", "--root", str(tmp_path), "--workers", "1"])
|
|
assert result.exit_code == 0, result.output
|
|
assert note_count() == 1
|
|
|
|
# The note vanishes without combined.md changing at all — the dir
|
|
# stays "current" and a plain re-run must skip it untouched.
|
|
store = connect(str(bib_db))
|
|
store._con().execute(
|
|
"DELETE FROM notes WHERE item_id = (SELECT id FROM items WHERE key = ?)",
|
|
(item_key,),
|
|
)
|
|
store._con().commit()
|
|
store.close()
|
|
assert note_count() == 0
|
|
|
|
result = runner.invoke(app, ["extract", "--root", str(tmp_path), "--workers", "1"])
|
|
assert result.exit_code == 0, result.output
|
|
assert "skipped: 1" in result.output.lower()
|
|
assert note_count() == 0 # not repaired without --reattach
|
|
|
|
result = runner.invoke(
|
|
app, ["extract", "--root", str(tmp_path), "--workers", "1", "--reattach"]
|
|
)
|
|
assert result.exit_code == 0, result.output
|
|
assert note_count() == 1 # repaired
|