1f60f8c945
- walker.walk_and_extract iterates docket/cid dirs under root,
parallelizes per-comment extraction across ThreadPoolExecutor.
PyMuPDF releases the GIL during decode so threads scale.
Returns {written, skipped, failed} counts; honors --docket and
--limit; resume rule via existing combined.md.
- view.rebuild_index walks combined.md files and writes _index.csv;
view.register exposes it as a DuckDB view (comments_index) for
filtering/aggregation. Bodies stay on disk; view.body_path is the
canonical lookup helper for downstream consumers (#254/#255).
CLI + integration test land in the next batch.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
59 lines
1.7 KiB
Python
59 lines
1.7 KiB
Python
"""DuckDB view over combined.md index."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import duckdb
|
|
|
|
from rex.comments import extract_comment
|
|
from rex.comments.view import body_path, rebuild_index, register
|
|
|
|
|
|
def _seed_two(tmp_path: Path) -> None:
|
|
import fitz
|
|
|
|
for cid_suffix, text in (("0001", "Long body " * 20), ("0002", "x")):
|
|
cdir = tmp_path / "CMS-2024-0001" / f"CMS-2024-0001-{cid_suffix}"
|
|
cdir.mkdir(parents=True)
|
|
doc = fitz.open()
|
|
doc.new_page().insert_text((50, 72), text)
|
|
doc.save(str(cdir / "attachment_1.pdf"))
|
|
doc.close()
|
|
extract_comment(cdir, inline_body="inline body")
|
|
|
|
|
|
def test_rebuild_index_writes_csv(tmp_path: Path):
|
|
_seed_two(tmp_path)
|
|
csv = rebuild_index(tmp_path)
|
|
assert csv == tmp_path / "_index.csv"
|
|
rows = csv.read_text().splitlines()
|
|
assert rows[0].startswith("comment_id,docket_id,")
|
|
assert len(rows) == 3 # header + 2 comments
|
|
|
|
|
|
def test_register_view_queryable(tmp_path: Path):
|
|
_seed_two(tmp_path)
|
|
rebuild_index(tmp_path)
|
|
|
|
con = duckdb.connect(":memory:")
|
|
register(con, root=tmp_path)
|
|
rows = con.execute(
|
|
"SELECT comment_id, n_attachments, n_ok FROM comments_index ORDER BY comment_id"
|
|
).fetchall()
|
|
assert len(rows) == 2
|
|
assert rows[0][0] == "CMS-2024-0001-0001"
|
|
assert rows[0][1] == 1 # n_attachments
|
|
assert rows[0][2] == 1 # n_ok
|
|
# Second row: short text → ocr_needed
|
|
assert rows[1][2] == 0
|
|
|
|
|
|
def test_body_path():
|
|
p = body_path(
|
|
Path("/x/.state/comments"),
|
|
comment_id="CMS-2024-0001-0099",
|
|
docket_id="CMS-2024-0001",
|
|
)
|
|
assert p == Path("/x/.state/comments/CMS-2024-0001/CMS-2024-0001-0099/combined.md")
|