77fd48eb31
extract_attachment returned "unsupported" for .epub, so CPT 2021/2022 and CPT Changes 2023 had no text at all, and 2018/2019/2024 indexed only from their PDF siblings. Added rex.comments.epub_text — stdlib only, shared with pfs.cpt_epub — that resolves an EPUB's own reading order via META-INF/container.xml -> the OPF's manifest + spine (falling back to sorted .xhtml/.html names when container.xml is missing), and strips tags/entities into plain text. extract_attachment's new .epub branch returns the same ExtractResult shape the PDF branch does; no change needed in llm.source._attachment_sections, which already tries every bib attachment regardless of extension. pfs.cpt_epub used to hard-code "OPS/" as the content-file prefix in three places; it now resolves the same way via epub_text.content_root, so a differently-templated EPUB would still locate its content instead of silently parsing to nothing.
107 lines
4.1 KiB
Python
107 lines
4.1 KiB
Python
"""rex.comments.epub_text — shared EPUB container/spine resolution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
from rex.comments.epub_text import content_root, extract_text, spine_paths
|
|
|
|
_CONTAINER_XML = (
|
|
'<?xml version="1.0"?>\n'
|
|
'<container version="1.0" '
|
|
'xmlns="urn:oasis:names:tc:opendocument:xmlns:container">\n'
|
|
"<rootfiles>\n"
|
|
'<rootfile full-path="OEBPS/content.opf" '
|
|
'media-type="application/oebps-package+xml"/>\n'
|
|
"</rootfiles>\n</container>"
|
|
)
|
|
|
|
|
|
def _opf(spine_idrefs: list[str]) -> str:
|
|
spine = "\n".join(f'<itemref idref="{i}"/>' for i in spine_idrefs)
|
|
return (
|
|
'<?xml version="1.0"?>\n'
|
|
'<package xmlns="http://www.idpf.org/2007/opf" version="3.0">\n'
|
|
"<manifest>\n"
|
|
'<item id="a" href="a.xhtml" media-type="application/xhtml+xml"/>\n'
|
|
'<item id="b" href="b.xhtml" media-type="application/xhtml+xml"/>\n'
|
|
"</manifest>\n"
|
|
f"<spine>\n{spine}\n</spine>\n"
|
|
"</package>"
|
|
)
|
|
|
|
|
|
def _build_epub_with_container(path: Path, *, spine_order: list[str]) -> None:
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
zf.writestr("mimetype", "application/epub+zip")
|
|
zf.writestr("META-INF/container.xml", _CONTAINER_XML)
|
|
zf.writestr("OEBPS/content.opf", _opf(spine_order))
|
|
zf.writestr("OEBPS/a.xhtml", "<html><body><p>Section A text.</p></body></html>")
|
|
zf.writestr("OEBPS/b.xhtml", "<html><body><p>Section B text.</p></body></html>")
|
|
|
|
|
|
def _build_epub_without_container(path: Path) -> None:
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
zf.writestr("mimetype", "application/epub+zip")
|
|
# Deliberately named so name-sort order differs from any
|
|
# plausible spine order — proves the fallback really is name-sort.
|
|
zf.writestr("OPS/z_first.xhtml", "<html><body><p>Z content.</p></body></html>")
|
|
zf.writestr("OPS/a_second.xhtml", "<html><body><p>A content.</p></body></html>")
|
|
|
|
|
|
class TestContentRoot:
|
|
def test_resolves_via_container_xml(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
_build_epub_with_container(path, spine_order=["a", "b"])
|
|
with zipfile.ZipFile(path) as zf:
|
|
assert content_root(zf) == "OEBPS/"
|
|
|
|
def test_falls_back_to_ops_without_container_xml(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
_build_epub_without_container(path)
|
|
with zipfile.ZipFile(path) as zf:
|
|
assert content_root(zf) == "OPS/"
|
|
|
|
|
|
class TestSpinePaths:
|
|
def test_resolves_spine_order_from_the_opf(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
_build_epub_with_container(path, spine_order=["b", "a"])
|
|
with zipfile.ZipFile(path) as zf:
|
|
assert spine_paths(zf) == ["OEBPS/b.xhtml", "OEBPS/a.xhtml"]
|
|
|
|
def test_none_without_container_xml(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
_build_epub_without_container(path)
|
|
with zipfile.ZipFile(path) as zf:
|
|
assert spine_paths(zf) is None
|
|
|
|
|
|
class TestExtractText:
|
|
def test_extracts_in_spine_order_not_alphabetical(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
# Spine deliberately reversed from alphabetical (a, b) name order.
|
|
_build_epub_with_container(path, spine_order=["b", "a"])
|
|
text = extract_text(path)
|
|
assert text.index("Section B text") < text.index("Section A text")
|
|
|
|
def test_falls_back_to_sorted_names_without_container_xml(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
_build_epub_without_container(path)
|
|
text = extract_text(path)
|
|
# a_second.xhtml sorts before z_first.xhtml by name.
|
|
assert text.index("A content") < text.index("Z content")
|
|
|
|
def test_strips_tags_and_unescapes_entities(self, tmp_path):
|
|
path = tmp_path / "book.epub"
|
|
with zipfile.ZipFile(path, "w") as zf:
|
|
zf.writestr("mimetype", "application/epub+zip")
|
|
zf.writestr(
|
|
"OPS/a.xhtml",
|
|
"<html><body><p>CPT® codes & values</p></body></html>",
|
|
)
|
|
text = extract_text(path)
|
|
assert "<p>" not in text
|
|
assert "CPT® codes & values" in text
|