fix(llm): ignore the section heading when locating a chunk's PDF page
Some checks failed
CI / lint (push) Successful in 35s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 6m42s
Infra CI / zotero (push) Successful in 11m36s
CI / test (push) Successful in 21m52s
Infra CI / docs (push) Successful in 1m22s
Infra CI / llm (push) Successful in 47s
Infra CI / api (push) Successful in 1m4s
Deploy / report (push) Successful in 13s
Infra CI / mc (push) Successful in 23s
Harden / build-scan-report (push) Successful in 21m52s
Renovate / renovate (push) Successful in 16s
Notebooks Integration / notebooks-integration (push) Successful in 9m7s
Zotero Sync / zotero-sync (push) Failing after 32s
Package Supply Chain / pkg-supply-chain (push) Successful in 58s

This commit is contained in:
kert
2026-09-03 10:55:34 -04:00
parent 3218eb7845
commit fafa3d4bc9
2 changed files with 21 additions and 1 deletions

View File

@@ -9,6 +9,7 @@ came from and finds the page containing the chunk's opening words.
from __future__ import annotations
import logging
import re
from dataclasses import replace
from pathlib import Path
@@ -18,12 +19,20 @@ log = logging.getLogger(__name__)
_PROBE_CHARS = 60
_MIN_PROBE = 12
_HEADING = re.compile(r"\A#{1,6}[^\n]*\n+")
def _norm(text: str) -> str:
return " ".join(text.split())
def _probe_text(text: str) -> str:
"""Chunk text minus a leading markdown heading — the first chunk of a
``## attachment.pdf`` section starts with that heading, which is not
in the PDF."""
return _HEADING.sub("", text, count=1)
def pdf_pages(path: Path) -> list[str]:
"""Whitespace-normalized text per page; ``[]`` when unreadable."""
try:
@@ -68,7 +77,7 @@ def enrich_pdf_pages(doc: Doc, chunks: list[Chunk]) -> list[Chunk]:
if path.lower().endswith(".pdf") and not md.get("page"):
if path not in cache:
cache[path] = pdf_pages(Path(path))
n = locate(cache[path], c.text)
n = locate(cache[path], _probe_text(c.text))
md["page"] = str(n) if n else ""
out.append(replace(c, metadata=md))
return out

View File

@@ -66,6 +66,17 @@ class TestEnrich:
assert "attachment" not in out[1].metadata
assert out[0].id == "c" and out[0].text == chunks[0].text
def test_leading_section_heading_ignored_when_probing(self, pdf):
doc = Doc(
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)
)
c = self._chunk(
"## attachment_1.pdf\n\nPage one talks about telehealth originating sites.",
"attachment_1.pdf",
)
(out,) = enrich_pdf_pages(doc, [c])
assert out.metadata["page"] == "1"
def test_unlocated_chunk_keeps_attachment_without_page(self, pdf):
doc = Doc(
key="K", text="", metadata={}, files=(("attachment_1.pdf", str(pdf)),)