d726dca50e
All checks were successful
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 1m37s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / llm (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 1m5s
Infra CI / zotero (push) Successful in 13s
Infra CI / docs (push) Successful in 1m22s
Infra CI / api (push) Successful in 1m15s
Infra CI / llm (push) Successful in 44s
Infra CI / mc (push) Successful in 15s
Deploy / report (push) Successful in 13s
CI / test (push) Successful in 17m19s
Notebooks Integration / notebooks-integration (push) Successful in 7m27s
Zotero Sync / zotero-sync (push) Successful in 1m10s
Package Supply Chain / pkg-supply-chain (push) Successful in 59s
153 lines
5.3 KiB
Python
153 lines
5.3 KiB
Python
"""rex.frtext — shared FR rule-text cleaner (capture-time normalization).
|
|
|
|
Mirrors the scenarios that used to live against
|
|
``llm.source._strip_html_wrapper`` (see tests/llm/test_source.py) since
|
|
that's the logic this module now owns.
|
|
"""
|
|
|
|
from rex.frtext import clean_fr_text, looks_html_wrapped, normalize_txt_file
|
|
|
|
|
|
class TestHtmlWrapper:
|
|
def test_strips_html_pre_wrapper(self):
|
|
raw = (
|
|
"<html><head><title>x</title></head><body><pre>\n"
|
|
"The Secretary proposes to amend 42 CFR part 414.\n"
|
|
"</pre></body></html>"
|
|
)
|
|
cleaned = clean_fr_text(raw)
|
|
assert "The Secretary proposes to amend 42 CFR part 414." in cleaned
|
|
assert "<html>" not in cleaned
|
|
assert "<pre>" not in cleaned
|
|
|
|
def test_passthrough_when_no_wrapper(self):
|
|
raw = "Plain text, no wrapper at all."
|
|
assert clean_fr_text(raw) == raw
|
|
|
|
|
|
class TestInlineMarkupAndEntities:
|
|
def test_inline_html_and_entities_stripped(self):
|
|
raw = (
|
|
"<html><head><title>x</title></head><body><pre>\n"
|
|
"[[Page 12345]]\n"
|
|
'Contact us at <a href="/cdn-cgi/l/email-protection#x">'
|
|
"someone</a> regarding CMS & PFS.\n"
|
|
"A finding was significant (p<0.05 and >2 cm).\n"
|
|
"</pre></body></html>"
|
|
)
|
|
cleaned = clean_fr_text(raw)
|
|
assert "<a href=" not in cleaned
|
|
assert "</a>" not in cleaned
|
|
assert "someone" in cleaned
|
|
assert "CMS & PFS" in cleaned
|
|
assert "[[Page 12345]]" in cleaned
|
|
assert "p<0.05 and >2 cm" in cleaned
|
|
|
|
|
|
class TestControlChars:
|
|
def test_strips_nul_and_c0_control_chars(self):
|
|
raw = "line one\x00\x0bline two\x1f end"
|
|
cleaned = clean_fr_text(raw)
|
|
assert "\x00" not in cleaned
|
|
assert "\x0b" not in cleaned
|
|
assert "\x1f" not in cleaned
|
|
assert "line one" in cleaned
|
|
assert "line two" in cleaned
|
|
|
|
def test_preserves_tab_newline_cr(self):
|
|
raw = "a\tb\nc\rd"
|
|
cleaned = clean_fr_text(raw)
|
|
assert cleaned == "a\tb\nc\rd"
|
|
|
|
|
|
class TestIdempotency:
|
|
def test_wrapped_text_is_idempotent(self):
|
|
raw = (
|
|
"<html><head><title>x</title></head><body><pre>\n"
|
|
"[[Page 12345]]\n"
|
|
'Contact <a href="/cdn-cgi/l/email-protection#x">someone</a> '
|
|
"re CMS & PFS.\n"
|
|
"A finding (p<0.05 and >2 cm).\x00\x0b\n"
|
|
"</pre></body></html>"
|
|
)
|
|
once = clean_fr_text(raw)
|
|
twice = clean_fr_text(once)
|
|
assert once == twice
|
|
|
|
def test_plain_text_is_idempotent(self):
|
|
raw = "Plain text with p<0.05 and >2 cm, no markup."
|
|
assert clean_fr_text(raw) == clean_fr_text(clean_fr_text(raw))
|
|
|
|
def test_empty_string(self):
|
|
assert clean_fr_text("") == ""
|
|
|
|
|
|
class TestLooksHtmlWrapped:
|
|
def test_true_for_html_wrapped(self):
|
|
assert looks_html_wrapped("<html><head></head><body><pre>x</pre></body></html>")
|
|
|
|
def test_true_case_insensitive_and_leading_whitespace(self):
|
|
assert looks_html_wrapped(" \n<HTML><body><pre>x</pre></body></html>")
|
|
|
|
def test_false_for_plain_text(self):
|
|
assert not looks_html_wrapped("Plain text, no wrapper at all.")
|
|
|
|
def test_false_when_html_appears_deep_in_body(self):
|
|
raw = ("filler " * 100) + "<html>should not count, too deep"
|
|
assert not looks_html_wrapped(raw)
|
|
|
|
|
|
class TestNormalizeTxtFile:
|
|
def test_rewrites_wrapped_file_and_saves_raw_html(self, tmp_path):
|
|
txt = tmp_path / "2025-13271.txt"
|
|
raw = (
|
|
"<html><head><title>x</title></head><body><pre>\n"
|
|
"The Secretary proposes to amend 42 CFR part 414.\n"
|
|
"</pre></body></html>"
|
|
)
|
|
txt.write_text(raw)
|
|
|
|
changed = normalize_txt_file(txt)
|
|
|
|
assert changed is True
|
|
html_sibling = tmp_path / "2025-13271.html"
|
|
assert html_sibling.exists()
|
|
assert html_sibling.read_text() == raw
|
|
cleaned = txt.read_text()
|
|
assert "<html>" not in cleaned
|
|
assert "The Secretary proposes to amend 42 CFR part 414." in cleaned
|
|
|
|
def test_noop_for_plain_text(self, tmp_path):
|
|
txt = tmp_path / "plain.txt"
|
|
txt.write_text("Already plain text.")
|
|
|
|
changed = normalize_txt_file(txt)
|
|
|
|
assert changed is False
|
|
assert not (tmp_path / "plain.html").exists()
|
|
assert txt.read_text() == "Already plain text."
|
|
|
|
def test_does_not_clobber_existing_html_sibling(self, tmp_path):
|
|
txt = tmp_path / "2025-13271.txt"
|
|
txt.write_text("<html><body><pre>new wrapped text</pre></body></html>")
|
|
html_sibling = tmp_path / "2025-13271.html"
|
|
html_sibling.write_text("original raw capture, do not overwrite")
|
|
|
|
normalize_txt_file(txt)
|
|
|
|
assert html_sibling.read_text() == "original raw capture, do not overwrite"
|
|
|
|
def test_idempotent_on_repeat_calls(self, tmp_path):
|
|
txt = tmp_path / "2025-13271.txt"
|
|
txt.write_text(
|
|
"<html><body><pre>[[Page 1]] p<0.05 and >2 cm</pre></body></html>"
|
|
)
|
|
|
|
normalize_txt_file(txt)
|
|
first = txt.read_text()
|
|
# Second call sees already-cleaned plain text: no wrapper, no-op.
|
|
changed_again = normalize_txt_file(txt)
|
|
|
|
assert changed_again is False
|
|
assert txt.read_text() == first
|