6eb4f2cb8b
Some checks failed
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 1m52s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
CI / test (push) Failing after 3m8s
Infra CI / zotero (push) Successful in 31s
Infra CI / notebooks (push) Successful in 1m4s
Infra CI / api (push) Successful in 1m29s
Infra CI / docs (push) Successful in 1m55s
Infra CI / mc (push) Failing after 58s
Infra CI / llm (push) Successful in 1m20s
Deploy / report (push) Successful in 22s
Every Federal Register rule item was indexed twice: as anchored FR paragraphs in the rules collection (67k chunks, federalregister.gov links) and again from its PDF attachment into corpus (161k chunks, kind 'corpus', item-URL links). The PDF copies outnumbered the anchored ones 2:1 in retrieval, so proposed rules and corrections came back as 'corpus' citations. iter_corpus_refs now skips rule-typed and fr_anchors-bearing items; stack llm prune-rules deletes the corpus copies (and their index_state rows) and stamps rule_kind (proposed/final/correction, bib.frlink.rule_kind) on the rules chunks; new rule docs carry rule_kind from the start. as_source exposes it, the prompt's source line reads '(proposed rule, 2026-07-16)', and the chat and search badges show it instead of 'rule'. CMS Internet-Only Manual chapters: llm.manuals.sectionize turns each body section heading (the ones followed by their '(Rev.' line; the table of contents stays plain) into a markdown heading, so chunks split at section boundaries and carry section / iom_section / iom_title plus the chapter's publication number and chapter; enrich_pdf_pages lets chunks under a sub-heading inherit the enclosing file so they keep their page. Manual citations read 'IOM 100-04 ch.18 §10.1.2' and link to the chapter PDF page; /search takes doctype, manual, chapter and section filters.
118 lines
4.0 KiB
Python
118 lines
4.0 KiB
Python
"""llm.manuals — IOM chapters sectioned, stamped and labelled."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
|
|
from llm.chunk import Chunk, Doc, chunk_doc
|
|
from llm.manuals import enrich, label, metadata, parse_section, sectionize
|
|
|
|
CHAPTER = """Medicare Claims Processing Manual
|
|
Chapter 18 - Preventive and Screening Services
|
|
Table of Contents
|
|
10 - Pneumococcal Pneumonia, Influenza Virus, and Hepatitis B
|
|
10.1 - Coverage Requirements
|
|
10.1.2 - Influenza Virus Vaccine
|
|
|
|
10.1 - Coverage Requirements
|
|
(Rev. 11355; Issued:04-14-22; Effective: 01-01-22)
|
|
Medicare covers the vaccines listed below.
|
|
|
|
10.1.2 - Influenza Virus Vaccine
|
|
(Rev. 2253, Issued: 07-08-11, Effective: 10-01-11)
|
|
Influenza virus vaccine and its administration are covered.
|
|
|
|
10.2.1 - Healthcare Common Procedure Coding System (HCPCS) and
|
|
Diagnosis Codes
|
|
(Rev. 13677; Issued: 03-12-2026)
|
|
Use the HCPCS codes in the table.
|
|
"""
|
|
|
|
|
|
class TestSectionize:
|
|
def test_body_headings_become_markdown_and_toc_stays(self):
|
|
out = sectionize(CHAPTER)
|
|
assert "## 10.1 - Coverage Requirements\n(Rev. 11355" in out
|
|
assert "## 10.1.2 - Influenza Virus Vaccine\n(Rev. 2253" in out
|
|
# the wrapped title is joined; the wrapped line stays in the text
|
|
assert (
|
|
"## 10.2.1 - Healthcare Common Procedure Coding System (HCPCS) and Diagnosis Codes\nDiagnosis Codes\n(Rev. 13677"
|
|
in out
|
|
)
|
|
# table-of-contents entries (no (Rev. line) are untouched
|
|
assert (
|
|
"\n10.1 - Coverage Requirements\n10.1.2 - Influenza Virus Vaccine\n\n"
|
|
in out
|
|
)
|
|
assert out.count("## ") == 3
|
|
|
|
def test_chunks_carry_sections(self):
|
|
doc = Doc(
|
|
key="K1",
|
|
text=sectionize(CHAPTER),
|
|
metadata={"doctype": "manual", "kind": "corpus"},
|
|
)
|
|
chunks = enrich(doc, chunk_doc(doc))
|
|
sections = {
|
|
c.metadata["iom_section"]: c.metadata["iom_title"]
|
|
for c in chunks
|
|
if c.metadata["iom_section"]
|
|
}
|
|
assert sections == {
|
|
"10.1": "Coverage Requirements",
|
|
"10.1.2": "Influenza Virus Vaccine",
|
|
"10.2.1": "Healthcare Common Procedure Coding System (HCPCS) and Diagnosis Codes",
|
|
}
|
|
head = [c for c in chunks if not c.metadata["iom_section"]]
|
|
assert head and "Table of Contents" in head[0].text
|
|
assert isinstance(chunks[0], Chunk)
|
|
|
|
def test_enrich_is_identity_for_non_manuals(self):
|
|
doc = Doc(
|
|
key="K", text="## 1.1 - x\n(Rev. 1)\nbody", metadata={"doctype": "source"}
|
|
)
|
|
chunks = chunk_doc(doc)
|
|
assert enrich(doc, chunks) == list(chunks)
|
|
|
|
|
|
class TestParseAndLabel:
|
|
def test_parse(self):
|
|
assert parse_section("10.1.2 - Influenza Virus Vaccine") == (
|
|
"10.1.2",
|
|
"Influenza Virus Vaccine",
|
|
)
|
|
assert parse_section("clm104c18.pdf") == ("", "")
|
|
assert parse_section("") == ("", "")
|
|
|
|
def test_label(self):
|
|
assert (
|
|
label({"manual": "100-04", "chapter": "18", "iom_section": "10.1.2"})
|
|
== "IOM 100-04 ch.18 §10.1.2"
|
|
)
|
|
assert label({"manual": "100-04", "chapter": "18"}) == "IOM 100-04 ch.18"
|
|
assert label({}) == ""
|
|
|
|
def test_metadata_from_item(self):
|
|
item = SimpleNamespace(
|
|
extra_json='{"manual_name": "Claims Processing", "pub_number": "100-04", "chapter": "18"}'
|
|
)
|
|
assert metadata(item) == {
|
|
"manual": "100-04",
|
|
"manual_name": "Claims Processing",
|
|
"chapter": "18",
|
|
}
|
|
item2 = SimpleNamespace(
|
|
extra_json="",
|
|
pub_number="100-02",
|
|
manual_name="Benefit Policy",
|
|
chapter="15",
|
|
)
|
|
assert (
|
|
metadata(item2)["manual"] == "100-02" and metadata(item2)["chapter"] == "15"
|
|
)
|
|
assert metadata(SimpleNamespace(extra_json="not json")) == {
|
|
"manual": "",
|
|
"manual_name": "",
|
|
"chapter": "",
|
|
}
|