fix(llm): ignore the section heading when locating a chunk's PDF page
Some checks failed
CI / lint (push) Successful in 35s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 6m42s
Infra CI / zotero (push) Successful in 11m36s
CI / test (push) Successful in 21m52s
Infra CI / docs (push) Successful in 1m22s
Infra CI / llm (push) Successful in 47s
Infra CI / api (push) Successful in 1m4s
Deploy / report (push) Successful in 13s
Infra CI / mc (push) Successful in 23s
Harden / build-scan-report (push) Successful in 21m52s
Renovate / renovate (push) Successful in 16s
Notebooks Integration / notebooks-integration (push) Successful in 9m7s
Zotero Sync / zotero-sync (push) Failing after 32s
Package Supply Chain / pkg-supply-chain (push) Successful in 58s
Some checks failed
CI / lint (push) Successful in 35s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 6m42s
Infra CI / zotero (push) Successful in 11m36s
CI / test (push) Successful in 21m52s
Infra CI / docs (push) Successful in 1m22s
Infra CI / llm (push) Successful in 47s
Infra CI / api (push) Successful in 1m4s
Deploy / report (push) Successful in 13s
Infra CI / mc (push) Successful in 23s
Harden / build-scan-report (push) Successful in 21m52s
Renovate / renovate (push) Successful in 16s
Notebooks Integration / notebooks-integration (push) Successful in 9m7s
Zotero Sync / zotero-sync (push) Failing after 32s
Package Supply Chain / pkg-supply-chain (push) Successful in 58s
This commit is contained in:
@@ -9,6 +9,7 @@ came from and finds the page containing the chunk's opening words.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import re
|
||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -18,12 +19,20 @@ log = logging.getLogger(__name__)
|
|||||||
|
|
||||||
_PROBE_CHARS = 60
|
_PROBE_CHARS = 60
|
||||||
_MIN_PROBE = 12
|
_MIN_PROBE = 12
|
||||||
|
_HEADING = re.compile(r"\A#{1,6}[^\n]*\n+")
|
||||||
|
|
||||||
|
|
||||||
def _norm(text: str) -> str:
|
def _norm(text: str) -> str:
|
||||||
return " ".join(text.split())
|
return " ".join(text.split())
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_text(text: str) -> str:
|
||||||
|
"""Chunk text minus a leading markdown heading — the first chunk of a
|
||||||
|
``## attachment.pdf`` section starts with that heading, which is not
|
||||||
|
in the PDF."""
|
||||||
|
return _HEADING.sub("", text, count=1)
|
||||||
|
|
||||||
|
|
||||||
def pdf_pages(path: Path) -> list[str]:
|
def pdf_pages(path: Path) -> list[str]:
|
||||||
"""Whitespace-normalized text per page; ``[]`` when unreadable."""
|
"""Whitespace-normalized text per page; ``[]`` when unreadable."""
|
||||||
try:
|
try:
|
||||||
@@ -68,7 +77,7 @@ def enrich_pdf_pages(doc: Doc, chunks: list[Chunk]) -> list[Chunk]:
|
|||||||
if path.lower().endswith(".pdf") and not md.get("page"):
|
if path.lower().endswith(".pdf") and not md.get("page"):
|
||||||
if path not in cache:
|
if path not in cache:
|
||||||
cache[path] = pdf_pages(Path(path))
|
cache[path] = pdf_pages(Path(path))
|
||||||
n = locate(cache[path], c.text)
|
n = locate(cache[path], _probe_text(c.text))
|
||||||
md["page"] = str(n) if n else ""
|
md["page"] = str(n) if n else ""
|
||||||
out.append(replace(c, metadata=md))
|
out.append(replace(c, metadata=md))
|
||||||
return out
|
return out
|
||||||
|
|||||||
@@ -66,6 +66,17 @@ class TestEnrich:
|
|||||||
assert "attachment" not in out[1].metadata
|
assert "attachment" not in out[1].metadata
|
||||||
assert out[0].id == "c" and out[0].text == chunks[0].text
|
assert out[0].id == "c" and out[0].text == chunks[0].text
|
||||||
|
|
||||||
|
def test_leading_section_heading_ignored_when_probing(self, pdf):
|
||||||
|
doc = Doc(
|
||||||
|
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
|
||||||
|
)
|
||||||
|
c = self._chunk(
|
||||||
|
"## attachment_1.pdf\n\nPage one talks about telehealth originating sites.",
|
||||||
|
"attachment_1.pdf",
|
||||||
|
)
|
||||||
|
(out,) = enrich_pdf_pages(doc, [c])
|
||||||
|
assert out.metadata["page"] == "1"
|
||||||
|
|
||||||
def test_unlocated_chunk_keeps_attachment_without_page(self, pdf):
|
def test_unlocated_chunk_keeps_attachment_without_page(self, pdf):
|
||||||
doc = Doc(
|
doc = Doc(
|
||||||
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
|
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
|
||||||
|
|||||||
Reference in New Issue
Block a user