b2ad400764
- extract_attachment now dispatches PDF/DOCX/TXT/HTML; DOCX via python-docx, text-like via plain read with cheap HTML strip. - combine.py writes one combined.md per comment dir with YAML frontmatter (per-attachment status + chars) and per-section body. - extract_comment is idempotent (existing combined.md is a no-op unless force=True). - parse_combined() splits frontmatter ↔ body for downstream consumers. Walker + DuckDB view land in the next batch. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""Per-comment combine — write combined.md with YAML frontmatter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import fitz
|
|
|
|
from rex.comments import extract_comment
|
|
from rex.comments.combine import parse_combined
|
|
|
|
|
|
def _make_pdf(path: Path, text: str) -> None:
|
|
doc = fitz.open()
|
|
doc.new_page().insert_text((50, 72), text)
|
|
doc.save(str(path))
|
|
doc.close()
|
|
|
|
|
|
def _setup_comment_dir(tmp_path: Path, cid: str = "CMS-2024-0001-0042") -> Path:
|
|
"""Layout matches .state/comments/{docket}/{cid}/."""
|
|
docket = "-".join(cid.split("-")[:3])
|
|
d = tmp_path / docket / cid
|
|
d.mkdir(parents=True)
|
|
_make_pdf(
|
|
d / "attachment_1.pdf",
|
|
"Position: oppose the proposed rule. This is a substantive comment.",
|
|
)
|
|
return d
|
|
|
|
|
|
def test_extract_comment_writes_combined_md(tmp_path: Path):
|
|
d = _setup_comment_dir(tmp_path)
|
|
out = extract_comment(d, inline_body="Inline body text from items.abstract.")
|
|
assert out == d / "combined.md"
|
|
assert out.is_file()
|
|
|
|
|
|
def test_combined_md_has_frontmatter_and_body(tmp_path: Path):
|
|
d = _setup_comment_dir(tmp_path)
|
|
extract_comment(d, inline_body="Inline body.")
|
|
text = (d / "combined.md").read_text()
|
|
fm, body = parse_combined(text)
|
|
assert fm["comment_id"] == "CMS-2024-0001-0042"
|
|
assert fm["docket_id"] == "CMS-2024-0001"
|
|
assert fm["chars"] > 0
|
|
assert len(fm["attachments"]) == 1
|
|
assert fm["attachments"][0]["file"] == "attachment_1.pdf"
|
|
assert fm["attachments"][0]["status"] == "ok"
|
|
assert "## Inline comment" in body
|
|
assert "Inline body." in body
|
|
assert "## attachment_1.pdf" in body
|
|
assert "Position: oppose" in body
|
|
|
|
|
|
def test_extract_comment_is_idempotent(tmp_path: Path):
|
|
d = _setup_comment_dir(tmp_path)
|
|
first = extract_comment(d, inline_body="x")
|
|
mtime1 = first.stat().st_mtime
|
|
second = extract_comment(d, inline_body="x")
|
|
assert second == first
|
|
assert second.stat().st_mtime == mtime1 # no rewrite on second call
|
|
|
|
|
|
def test_extract_comment_force_rewrites(tmp_path: Path):
|
|
d = _setup_comment_dir(tmp_path)
|
|
extract_comment(d, inline_body="first")
|
|
extract_comment(d, inline_body="second", force=True)
|
|
body = (d / "combined.md").read_text()
|
|
assert "second" in body
|
|
assert "first" not in body
|
|
|
|
|
|
def test_extract_comment_no_attachments(tmp_path: Path):
|
|
d = tmp_path / "CMS-2024-0001" / "CMS-2024-0001-0099"
|
|
d.mkdir(parents=True)
|
|
out = extract_comment(d, inline_body="Inline only.")
|
|
fm, body = parse_combined(out.read_text())
|
|
assert fm["attachments"] == []
|
|
assert "Inline only." in body
|
|
|
|
|
|
def test_extract_comment_handles_failed_attachment(tmp_path: Path):
|
|
d = tmp_path / "CMS-2024-0001" / "CMS-2024-0001-0100"
|
|
d.mkdir(parents=True)
|
|
(d / "attachment_1.pdf").write_bytes(b"not a real pdf")
|
|
out = extract_comment(d, inline_body="x")
|
|
fm, _body = parse_combined(out.read_text())
|
|
assert fm["attachments"][0]["status"] == "failed"
|