2e22045ed2
All checks were successful
CI / lint (push) Successful in 29s
CI / notebooks-smoke (push) Successful in 1m29s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 55s
Infra CI / zotero (push) Successful in 17s
Infra CI / docs (push) Successful in 17s
Infra CI / api (push) Successful in 1m6s
Infra CI / llm (push) Successful in 47s
Infra CI / mc (push) Successful in 13s
Deploy / report (push) Successful in 13s
CI / test (push) Successful in 14m3s
Harden / build-scan-report (push) Successful in 26m9s
Renovate / renovate (push) Successful in 14s
Notebooks Integration / notebooks-integration (push) Successful in 7m21s
Zotero Sync / zotero-sync (push) Successful in 1m8s
Package Supply Chain / pkg-supply-chain (push) Successful in 1m2s
Extracted PDF comment text carries NUL (0x00) and form-feed bytes that Postgres text columns reject (DataError: cannot contain NUL bytes) — the 2017 pilot was clean by luck, the 2018+ dockets crash the indexer. Strip C0 control chars (except tab/newline/return) in chunk_doc.
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
"""llm.chunk — markdown-aware chunking with deterministic ids."""
|
|
|
|
import pytest
|
|
|
|
from llm.chunk import Doc, chunk_doc, content_hash
|
|
|
|
FRONTMATTER_DOC = """---
|
|
comment_id: CMS-2019-0111-0042
|
|
docket_id: CMS-2019-0111
|
|
---
|
|
|
|
# Re: CY 2020 PFS Proposed Rule
|
|
|
|
We object to the E/M consolidation.
|
|
|
|
## Telehealth
|
|
|
|
Originating-site rules should be relaxed.
|
|
"""
|
|
|
|
|
|
def _doc(text, key="K1"):
|
|
return Doc(key=key, text=text, metadata={"docket": "CMS-2019-0111"})
|
|
|
|
|
|
class TestChunkDoc:
|
|
def test_strips_yaml_frontmatter(self):
|
|
chunks = chunk_doc(_doc(FRONTMATTER_DOC))
|
|
assert "comment_id:" not in chunks[0].text
|
|
assert chunks[0].text.startswith("# Re:")
|
|
|
|
def test_deterministic_ids(self):
|
|
a = chunk_doc(_doc(FRONTMATTER_DOC))
|
|
b = chunk_doc(_doc(FRONTMATTER_DOC))
|
|
assert [c.id for c in a] == [c.id for c in b]
|
|
h = content_hash(FRONTMATTER_DOC)
|
|
assert a[0].id == f"K1:{h[:12]}:0000"
|
|
|
|
def test_metadata_carries_item_key_and_seq(self):
|
|
chunks = chunk_doc(_doc(FRONTMATTER_DOC))
|
|
assert chunks[0].metadata["item_key"] == "K1"
|
|
assert chunks[0].metadata["docket"] == "CMS-2019-0111"
|
|
assert chunks[0].metadata["seq"] == "0"
|
|
|
|
def test_splits_on_headings_before_size(self):
|
|
text = "# A\n\n" + "para. " * 100 + "\n\n# B\n\nshort."
|
|
chunks = chunk_doc(_doc(text), target_chars=300)
|
|
assert all(len(c.text) <= 300 + 200 for c in chunks)
|
|
assert any(c.text.lstrip().startswith("# B") for c in chunks)
|
|
|
|
def test_long_paragraph_hard_wrapped_with_overlap(self):
|
|
text = "x" * 5000
|
|
chunks = chunk_doc(_doc(text), target_chars=2000, overlap_chars=200)
|
|
assert len(chunks) == 3
|
|
assert chunks[1].text[:200] == chunks[0].text[-200:]
|
|
|
|
def test_empty_and_whitespace_yield_nothing(self):
|
|
assert chunk_doc(_doc("")) == []
|
|
assert chunk_doc(_doc(" \n\n ")) == []
|
|
|
|
def test_overlap_gte_target_raises(self):
|
|
with pytest.raises(ValueError, match="overlap_chars must be smaller"):
|
|
chunk_doc(_doc("some text"), target_chars=100, overlap_chars=100)
|
|
|
|
|
|
class TestControlCharStripping:
|
|
def test_strips_nul_and_control_chars(self):
|
|
doc = _doc("Clean text\x00 with\x0c a NUL\x07 and formfeed.")
|
|
chunks = chunk_doc(doc)
|
|
joined = "".join(c.text for c in chunks)
|
|
assert "\x00" not in joined
|
|
assert "\x0c" not in joined
|
|
assert "\x07" not in joined
|
|
assert "Clean text with a NUL and formfeed." in joined
|
|
|
|
def test_keeps_tab_newline_return(self):
|
|
doc = _doc("line1\n\nline2\twith tab")
|
|
joined = "".join(c.text for c in chunk_doc(doc))
|
|
assert "\t" in joined and "line2" in joined
|