fix(llm): ignore the section heading when locating a chunk's PDF page
Some checks failed
CI / lint (push) Successful in 35s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 6m42s
Infra CI / zotero (push) Successful in 11m36s
CI / test (push) Successful in 21m52s
Infra CI / docs (push) Successful in 1m22s
Infra CI / llm (push) Successful in 47s
Infra CI / api (push) Successful in 1m4s
Deploy / report (push) Successful in 13s
Infra CI / mc (push) Successful in 23s
Harden / build-scan-report (push) Successful in 21m52s
Renovate / renovate (push) Successful in 16s
Notebooks Integration / notebooks-integration (push) Successful in 9m7s
Zotero Sync / zotero-sync (push) Failing after 32s
Package Supply Chain / pkg-supply-chain (push) Successful in 58s
Some checks failed
CI / lint (push) Successful in 35s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 6m42s
Infra CI / zotero (push) Successful in 11m36s
CI / test (push) Successful in 21m52s
Infra CI / docs (push) Successful in 1m22s
Infra CI / llm (push) Successful in 47s
Infra CI / api (push) Successful in 1m4s
Deploy / report (push) Successful in 13s
Infra CI / mc (push) Successful in 23s
Harden / build-scan-report (push) Successful in 21m52s
Renovate / renovate (push) Successful in 16s
Notebooks Integration / notebooks-integration (push) Successful in 9m7s
Zotero Sync / zotero-sync (push) Failing after 32s
Package Supply Chain / pkg-supply-chain (push) Successful in 58s
This commit is contained in:
@@ -9,6 +9,7 @@ came from and finds the page containing the chunk's opening words.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import replace
|
||||
from pathlib import Path
|
||||
|
||||
@@ -18,12 +19,20 @@ log = logging.getLogger(__name__)
|
||||
|
||||
_PROBE_CHARS = 60
|
||||
_MIN_PROBE = 12
|
||||
_HEADING = re.compile(r"\A#{1,6}[^\n]*\n+")
|
||||
|
||||
|
||||
def _norm(text: str) -> str:
|
||||
return " ".join(text.split())
|
||||
|
||||
|
||||
def _probe_text(text: str) -> str:
|
||||
"""Chunk text minus a leading markdown heading — the first chunk of a
|
||||
``## attachment.pdf`` section starts with that heading, which is not
|
||||
in the PDF."""
|
||||
return _HEADING.sub("", text, count=1)
|
||||
|
||||
|
||||
def pdf_pages(path: Path) -> list[str]:
|
||||
"""Whitespace-normalized text per page; ``[]`` when unreadable."""
|
||||
try:
|
||||
@@ -68,7 +77,7 @@ def enrich_pdf_pages(doc: Doc, chunks: list[Chunk]) -> list[Chunk]:
|
||||
if path.lower().endswith(".pdf") and not md.get("page"):
|
||||
if path not in cache:
|
||||
cache[path] = pdf_pages(Path(path))
|
||||
n = locate(cache[path], c.text)
|
||||
n = locate(cache[path], _probe_text(c.text))
|
||||
md["page"] = str(n) if n else ""
|
||||
out.append(replace(c, metadata=md))
|
||||
return out
|
||||
|
||||
@@ -66,6 +66,17 @@ class TestEnrich:
|
||||
assert "attachment" not in out[1].metadata
|
||||
assert out[0].id == "c" and out[0].text == chunks[0].text
|
||||
|
||||
def test_leading_section_heading_ignored_when_probing(self, pdf):
|
||||
doc = Doc(
|
||||
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
|
||||
)
|
||||
c = self._chunk(
|
||||
"## attachment_1.pdf\n\nPage one talks about telehealth originating sites.",
|
||||
"attachment_1.pdf",
|
||||
)
|
||||
(out,) = enrich_pdf_pages(doc, [c])
|
||||
assert out.metadata["page"] == "1"
|
||||
|
||||
def test_unlocated_chunk_keeps_attachment_without_page(self, pdf):
|
||||
doc = Doc(
|
||||
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
|
||||
|
||||
Reference in New Issue
Block a user