stack/tests/bib/test_spider.py

"""Tests for bib.spider — citation discovery crawler."""

from __future__ import annotations

from pathlib import Path
from unittest.mock import patch

from bib.item import Rule, Source
from bib.spider import (
    _get_attachments,
    _get_sup_collection,
    _link_existing,
    _ordinal,
    _read_attachment,
    _translate_url,
    classify_url,
    crawl,
    crawl_all,
    extract_refs,
    extract_urls,
    extract_xml_refs,
)
from bib.store import Store

# ── classify_url ────────────────────────────────────────────────────


class TestClassifyUrl:
    def test_federal_register_html(self) -> None:
        url = "https://www.federalregister.gov/documents/2025/11/01/2025-19787/rule"
        assert classify_url(url) == "federal_register"

    def test_federal_register_api(self) -> None:
        url = "https://www.federalregister.gov/api/v1/documents/2025-19787.json"
        assert classify_url(url) == "federal_register"

    def test_cms_manual(self) -> None:
        url = "https://www.cms.gov/regulations-and-guidance/guidance/manuals/downloads/clm104c12.pdf"
        assert classify_url(url) == "cms_manual"

    def test_ecfr(self) -> None:
        url = "https://www.ecfr.gov/current/title-42/part-414"
        assert classify_url(url) == "ecfr"

    def test_cms_website(self) -> None:
        url = "https://www.cms.gov/medicare/payment/fee-schedules/physician"
        assert classify_url(url) == "cms_website"

    def test_unknown_url(self) -> None:
        assert classify_url("https://www.example.com/page") is None

    def test_cms_medicare(self) -> None:
        url = "https://www.cms.gov/providers/medicare/overview"
        assert classify_url(url) == "cms_website"


# ── _ordinal ────────────────────────────────────────────────────────


class TestOrdinal:
    def test_1st(self) -> None:
        assert _ordinal(1) == "1st"

    def test_2nd(self) -> None:
        assert _ordinal(2) == "2nd"

    def test_3rd(self) -> None:
        assert _ordinal(3) == "3rd"

    def test_4th(self) -> None:
        assert _ordinal(4) == "4th"

    def test_11th(self) -> None:
        assert _ordinal(11) == "11th"

    def test_12th(self) -> None:
        assert _ordinal(12) == "12th"

    def test_13th(self) -> None:
        assert _ordinal(13) == "13th"

    def test_21st(self) -> None:
        assert _ordinal(21) == "21st"

    def test_22nd(self) -> None:
        assert _ordinal(22) == "22nd"

    def test_117th(self) -> None:
        assert _ordinal(117) == "117th"

    def test_0th(self) -> None:
        assert _ordinal(0) == "0th"

    def test_100th(self) -> None:
        assert _ordinal(100) == "100th"

    def test_111th(self) -> None:
        assert _ordinal(111) == "111th"

    def test_112th(self) -> None:
        assert _ordinal(112) == "112th"

    def test_113th(self) -> None:
        assert _ordinal(113) == "113th"


# ── extract_xml_refs ────────────────────────────────────────────────


class TestExtractXmlRefs:
    def test_cfr_tag(self) -> None:
        xml = "<CFR>42 CFR Parts 410, 414</CFR>"
        urls = extract_xml_refs(xml)
        assert any("part-410" in u for u in urls)
        assert any("part-414" in u for u in urls)

    def test_cfr_title_detection(self) -> None:
        xml = "<CFR>45 CFR Parts 164</CFR>"
        urls = extract_xml_refs(xml)
        assert any("title-45" in u for u in urls)

    def test_sectno_tag(self) -> None:
        xml = "<CFR>42 CFR Parts 414</CFR><SECTNO>§ 414.22</SECTNO>"
        urls = extract_xml_refs(xml)
        assert any("section-414.22" in u for u in urls)

    def test_amdpar(self) -> None:
        xml = "<CFR>42 CFR Parts 414</CFR><AMDPAR>Section § 410.30 is amended</AMDPAR>"
        urls = extract_xml_refs(xml)
        assert any("section-410.30" in u for u in urls)

    def test_inline_section_refs(self) -> None:
        xml = "See § 414.22 and § 410.10 for details."
        urls = extract_xml_refs(xml)
        assert any("section-414.22" in u for u in urls)
        assert any("section-410.10" in u for u in urls)

    def test_fr_citation(self) -> None:
        xml = "See 74 FR 61738 for background."
        urls = extract_xml_refs(xml)
        assert any("74+FR+61738" in u for u in urls)

    def test_e_tag_url(self) -> None:
        xml = '<E T="03">https://www.cms.gov/test</E>'
        urls = extract_xml_refs(xml)
        assert "https://www.cms.gov/test" in urls

    def test_public_law(self) -> None:
        xml = "Enacted by Pub. L. 117-169 and Public Law 110-275."
        urls = extract_xml_refs(xml)
        assert any("117th-congress" in u for u in urls)
        assert any("110th-congress" in u for u in urls)

    def test_deduplication(self) -> None:
        xml = "§ 414.22 and again § 414.22"
        urls = extract_xml_refs(xml)
        section_urls = [u for u in urls if "section-414.22" in u]
        assert len(section_urls) == 1

    def test_empty_xml(self) -> None:
        assert extract_xml_refs("") == []

    def test_default_cfr_title(self) -> None:
        # No <CFR> tag, should use default 42
        xml = "<SECTNO>§ 414.22</SECTNO>"
        urls = extract_xml_refs(xml)
        assert any("title-42" in u for u in urls)


# ── extract_refs ────────────────────────────────────────────────────


class TestExtractRefs:
    def test_cfr_full_citation(self) -> None:
        text = "Under 42 C.F.R. § 414.22, providers must..."
        urls = extract_refs(text)
        assert any("section-414.22" in u for u in urls)
        assert any("title-42" in u for u in urls)

    def test_cfr_without_section_symbol(self) -> None:
        text = "Per 42 CFR 414.22"
        urls = extract_refs(text)
        assert any("section-414.22" in u for u in urls)

    def test_bare_section_ref(self) -> None:
        text = "See § 414.22 for details."
        urls = extract_refs(text, cfr_title="42")
        assert any("title-42" in u for u in urls)
        assert any("section-414.22" in u for u in urls)

    def test_fr_citation(self) -> None:
        text = "Published at 89 FR 98452."
        urls = extract_refs(text)
        assert any("89+FR+98452" in u for u in urls)

    def test_fed_reg_citation(self) -> None:
        text = "See 89 Fed. Reg. 98452."
        urls = extract_refs(text)
        assert any("89+FR+98452" in u for u in urls)

    def test_deduplication(self) -> None:
        text = "42 C.F.R. § 414.22 and again § 414.22"
        urls = extract_refs(text)
        section_414 = [u for u in urls if "section-414.22" in u]
        assert len(section_414) == 1

    def test_empty_text(self) -> None:
        assert extract_refs("") == []


# ── extract_urls ────────────────────────────────────────────────────


class TestExtractUrls:
    def test_extracts_hrefs(self) -> None:
        html = """
        <a href="https://example.com/page1">Link 1</a>
        <a href="https://example.com/page2">Link 2</a>
        """
        urls = extract_urls(html)
        assert "https://example.com/page1" in urls
        assert "https://example.com/page2" in urls

    def test_deduplication(self) -> None:
        html = """
        <a href="https://example.com">A</a>
        <a href="https://example.com">B</a>
        """
        urls = extract_urls(html)
        assert len(urls) == 1

    def test_empty_html(self) -> None:
        assert extract_urls("") == []

    def test_no_http_links(self) -> None:
        html = '<a href="/relative/path">X</a>'
        assert extract_urls(html) == []


# ── _get_attachments ────────────────────────────────────────────────


class TestGetAttachments:
    def test_returns_attachments(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        key = s.create(Rule(title="Test"))
        src = tmp_path / "doc.pdf"
        src.write_bytes(b"pdf")
        s.attach_file(key, src)

        atts = _get_attachments(s, key)
        assert len(atts) == 1
        assert atts[0]["filename"] == "doc.pdf"
        s.close()

    def test_no_item(self) -> None:
        s = Store(":memory:")
        atts = _get_attachments(s, "NOEXIST1")
        assert atts == []
        s.close()

    def test_no_attachments(self) -> None:
        s = Store(":memory:")
        key = s.create(Rule(title="Test"))
        atts = _get_attachments(s, key)
        assert atts == []
        s.close()


# ── _read_attachment ────────────────────────────────────────────────


class TestReadAttachment:
    def test_reads_file(self, tmp_path: Path) -> None:
        with patch("bib.spider._zotero_storage", return_value=tmp_path):
            att_dir = tmp_path / "ATT12345"
            att_dir.mkdir()
            (att_dir / "doc.txt").write_text("hello world")
            content = _read_attachment("ATT12345", "doc.txt")
            assert content == "hello world"

    def test_missing_file(self, tmp_path: Path) -> None:
        with patch("bib.spider._zotero_storage", return_value=tmp_path):
            content = _read_attachment("NOEXIST1", "doc.txt")
            assert content == ""

    def test_read_error(self, tmp_path: Path) -> None:
        with (
            patch("bib.spider._zotero_storage", return_value=tmp_path),
            patch("pathlib.Path.read_text", side_effect=OSError("read error")),
        ):
            att_dir = tmp_path / "ATT12345"
            att_dir.mkdir()
            (att_dir / "doc.txt").write_bytes(b"x")
            content = _read_attachment("ATT12345", "doc.txt")
            assert content == ""


# ── _get_sup_collection ─────────────────────────────────────────────


class TestGetSupCollection:
    def test_creates_supplemental_collection(self) -> None:
        s = Store(":memory:")
        item = Rule(title="Parent")
        key = s.create(item)
        parent = s.get(key)

        col_key = _get_sup_collection(s, parent)
        assert isinstance(col_key, str)
        assert len(col_key) == 8
        s.close()

    def test_finds_existing_supplemental(self) -> None:
        s = Store(":memory:")
        coll_map = s.ensure_collections({"Supplemental": {}})
        item = Rule(title="Test")
        key = s.create(item)
        parent = s.get(key)

        col_key = _get_sup_collection(s, parent)
        assert col_key == coll_map["Supplemental"]
        s.close()

    def test_with_parent_collection_existing_sup(self) -> None:
        s = Store(":memory:")
        coll_map = s.ensure_collections({"Rules": {"Supplemental": {}}})
        rules_key = coll_map["Rules"]
        sup_key = coll_map["Supplemental"]

        item = Rule(title="Test")
        key = s.create(item, collection=rules_key)
        parent = s.get(key)

        col_key = _get_sup_collection(s, parent)
        assert col_key == sup_key
        s.close()

    def test_with_parent_collection_creates_sup(self) -> None:
        """Line 270: parent has collection, but no Supplemental exists yet."""
        s = Store(":memory:")
        coll_map = s.ensure_collections({"Rules": {}})
        rules_key = coll_map["Rules"]

        item = Rule(title="Test")
        key = s.create(item, collection=rules_key)
        parent = s.get(key)

        col_key = _get_sup_collection(s, parent)
        assert isinstance(col_key, str)
        assert len(col_key) == 8
        # Verify the Supplemental collection was created under Rules
        colls = s.list_collections()
        sup = [c for c in colls if c["name"] == "Supplemental"]
        assert len(sup) == 1
        assert sup[0]["parent_key"] == rules_key
        s.close()


# ── _translate_url ──────────────────────────────────────────────────


class TestTranslateUrl:
    def test_federal_register_url(self) -> None:
        url = "https://www.federalregister.gov/documents/2025/11/01/2025-19787/rule"
        mock_rule = Rule(title="Test Rule", url=url)
        with patch("bib.translate.federal_register", return_value=mock_rule):
            result = _translate_url(url)
        assert result is not None
        assert result.title == "Test Rule"

    def test_ecfr_url(self) -> None:
        url = "https://www.ecfr.gov/current/title-42/part-414"
        from bib.item import Regulation

        mock_reg = Regulation(title="42 CFR Part 414", url=url)
        with patch("bib.translate.ecfr", return_value=mock_reg):
            result = _translate_url(url)
        assert result is not None

    def test_cms_manual_url(self) -> None:
        url = "https://www.cms.gov/regulations/manuals/downloads/clm104c12.pdf"
        from bib.item import Manual

        mock_manual = Manual(title="Chapter 12", url=url)
        with patch("bib.translate.cms_manual", return_value=mock_manual):
            result = _translate_url(url)
        assert result is not None

    def test_cms_website_url(self) -> None:
        url = "https://www.cms.gov/medicare/payment/fee-schedules/physician"
        from bib.item import Download

        mock_dl = Download(title="PFS", url=url)
        with patch("bib.translate.cms_website", return_value=mock_dl):
            result = _translate_url(url)
        assert result is not None

    def test_gov_url_generic(self) -> None:
        url = "https://www.congress.gov/bill/117th-congress/public-law-169"
        result = _translate_url(url)
        assert result is not None
        assert isinstance(result, Source)
        assert result.url == url

    def test_non_gov_url_returns_none(self) -> None:
        assert _translate_url("https://www.example.com/page") is None

    def test_translator_exception(self) -> None:
        url = "https://www.federalregister.gov/documents/2025/11/01/2025-19787/rule"
        with patch("bib.translate.federal_register", side_effect=Exception("fail")):
            result = _translate_url(url)
        assert result is None


# ── _link_existing ──────────────────────────────────────────────────


class TestLinkExisting:
    def test_adds_tag_to_existing(self) -> None:
        s = Store(":memory:")
        key = s.create(Rule(title="Existing", url="https://example.com/r"))
        _link_existing(s, "https://example.com/r", "PARENT12")
        item = s.get(key)
        assert "sup:PARENT12" in item.tags
        s.close()

    def test_no_match(self) -> None:
        s = Store(":memory:")
        _link_existing(s, "https://nonexistent.com", "PARENT12")
        s.close()


# ── crawl ───────────────────────────────────────────────────────────


class TestCrawl:
    def test_no_attachments(self) -> None:
        s = Store(":memory:")
        key = s.create(Rule(title="Test", url="https://example.com/r"))
        result = crawl(s, key)
        assert result == []
        s.close()

    def test_already_seen_url(self) -> None:
        s = Store(":memory:")
        key = s.create(Rule(title="Test", url="https://example.com/r"))
        seen = {"https://example.com/r"}
        result = crawl(s, key, _seen=seen)
        assert result == []
        s.close()

    def test_crawl_with_xml_attachment(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        key = s.create(
            Rule(
                title="PFS Rule",
                url="https://example.com/pfs-rule",
                tags=["module:pfs", "year:2026"],
            )
        )

        xml_content = "<CFR>42 CFR Parts 414</CFR>"

        mock_atts = [
            {"key": "ATT1", "filename": "rule.xml", "content_type": "text/xml"}
        ]

        mock_child = Source(
            title="42 CFR Part 414", url="https://ecfr.gov/title-42/part-414"
        )

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=xml_content),
            patch("bib.spider._translate_url", return_value=mock_child),
        ):
            result = crawl(s, key)

        assert len(result) >= 1
        s.close()

    def test_crawl_with_text_attachment(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        key = s.create(Rule(title="Test", url="https://example.com/test"))

        text_content = "Per 42 C.F.R. § 414.22, the payment..."

        mock_atts = [
            {"key": "ATT1", "filename": "doc.txt", "content_type": "text/plain"}
        ]

        mock_child = Source(
            title="42 CFR 414.22",
            url="https://www.ecfr.gov/current/title-42/part-414/section-414.22",
        )

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=text_content),
            patch("bib.spider._translate_url", return_value=mock_child),
        ):
            result = crawl(s, key)

        assert len(result) >= 1
        s.close()

    def test_crawl_skips_seen_urls(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        url_existing = "https://www.ecfr.gov/current/title-42/part-414/section-414.22"
        key = s.create(Rule(title="Test", url="https://example.com/test"))
        # Pre-create an item at the URL
        s.create(Rule(title="Already There", url=url_existing))

        text_content = "See § 414.22 for details."
        mock_atts = [
            {"key": "ATT1", "filename": "doc.txt", "content_type": "text/plain"}
        ]
        seen = {url_existing}

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=text_content),
        ):
            result = crawl(s, key, _seen=seen)

        # URL was already seen so should not produce new items
        assert result == []
        s.close()

    def test_crawl_empty_attachment_content(self) -> None:
        s = Store(":memory:")
        key = s.create(Rule(title="Test", url="https://example.com/test"))

        mock_atts = [
            {"key": "ATT1", "filename": "empty.xml", "content_type": "text/xml"}
        ]

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=""),
        ):
            result = crawl(s, key)

        assert result == []
        s.close()

    def test_crawl_translate_returns_none(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        key = s.create(Rule(title="Test", url="https://example.com/test"))

        text_content = "See § 414.22 for details."
        mock_atts = [
            {"key": "ATT1", "filename": "doc.txt", "content_type": "text/plain"}
        ]

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=text_content),
            patch("bib.spider._translate_url", return_value=None),
        ):
            result = crawl(s, key)

        assert result == []
        s.close()

    def test_crawl_depth_recursion(self, tmp_path: Path) -> None:
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        key = s.create(Rule(title="Parent", url="https://example.com/parent"))

        xml_content = "§ 414.22"
        mock_atts = [
            {"key": "ATT1", "filename": "rule.xml", "content_type": "text/xml"}
        ]
        child_url = "https://www.ecfr.gov/current/title-42/part-414/section-414.22"
        mock_child = Source(title="Child", url=child_url)

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=xml_content),
            patch("bib.spider._translate_url", return_value=mock_child),
        ):
            result = crawl(s, key, depth=2)

        assert len(result) >= 1
        s.close()


# ── crawl_all ───────────────────────────────────────────────────────


class TestCrawlAll:
    def test_crawl_all_basic(self) -> None:
        s = Store(":memory:")
        k1 = s.create(Rule(title="R1", url="https://example.com/r1"))
        k2 = s.create(Rule(title="R2", url="https://example.com/r2"))

        with patch("bib.spider.crawl", side_effect=[["C1"], []]):
            results = crawl_all(s, item_type="rule")

        assert k1 in results
        assert k2 not in results
        assert results[k1] == ["C1"]
        s.close()

    def test_crawl_all_with_tag_filter(self) -> None:
        s = Store(":memory:")
        s.create(
            Rule(title="R1", url="https://example.com/r1"),
            tags=["module:pfs"],
        )
        s.create(
            Rule(title="R2", url="https://example.com/r2"),
            tags=["module:aco"],
        )

        with patch("bib.spider.crawl", return_value=["C1"]):
            results = crawl_all(s, tag="module:pfs")

        assert len(results) == 1
        s.close()

    def test_crawl_all_empty(self) -> None:
        s = Store(":memory:")
        results = crawl_all(s)
        assert results == {}
        s.close()


# ── _zotero_storage default path (lines 44, 46) ─────────────────────


class TestZoteroStorage:
    def test_default_path(self) -> None:
        """Lines 44, 46: _zotero_storage reads from conf.path."""
        import sys

        from bib.spider import _zotero_storage

        fake_path = Path("/tmp/fake-zotero-storage")
        fake_conf = type(sys)("conf")
        fake_conf.path = lambda name: fake_path
        with patch.dict(sys.modules, {"conf": fake_conf}):
            result = _zotero_storage()
        assert result == fake_path


# ── crawl with rule_slug tag (line 397) ──────────────────────────────


class TestCrawlRuleSlugTag:
    def test_crawl_adds_sup_tag(self, tmp_path: Path) -> None:
        """Line 397: crawl adds Tag.sup(slug) when rule_slug is truthy."""
        db = tmp_path / "bib.sqlite"
        s = Store(db, storage_dir=tmp_path / "storage")
        # Title must match _RULE_RE: "CY 2026 PFS Final"
        key = s.create(
            Rule(
                title="Medicare Program; CY 2026 PFS Final Rule",
                url="https://example.com/pfs-rule-2026",
                tags=["module:pfs", "year:2026"],
            )
        )

        xml_content = "§ 414.22"
        mock_atts = [
            {"key": "ATT1", "filename": "rule.xml", "content_type": "text/xml"}
        ]
        child_url = "https://www.ecfr.gov/current/title-42/part-414/section-414.22"
        mock_child = Source(title="42 CFR 414.22", url=child_url)

        with (
            patch("bib.spider._get_attachments", return_value=mock_atts),
            patch("bib.spider._read_attachment", return_value=xml_content),
            patch("bib.spider._translate_url", return_value=mock_child),
        ):
            result = crawl(s, key)

        assert len(result) >= 1
        # The child item should have the sup:2026_PFS_FR tag
        child_item = s.get(result[0])
        assert any("sup:" in t for t in child_item.tags)
        s.close()