stack/tests/bib/test_frlink.py

"""Tests for bib.frlink — FR web anchor maps + jump-link resolution (P40)."""

from __future__ import annotations

import pytest

from bib import frlink
from bib.item import Rule
from bib.store import Store

# Mirrors the real body-HTML shape: paragraph elements carry id="p-N"
# AND an explicit data-page attribute (verified on doc 2026-14327);
# page divs carry id="page-NNNNN"; text has entities + inline markup.
FIXTURE_HTML = """
<html><body>
<div id="page-100"></div>
<p id="p-1" data-page="100">First para on 100 with &sect;&thinsp;414.1425 text.</p>
<p id="p-2" data-page="100">Second para <em>with markup</em> on 100.</p>
<div id="page-101"></div>
<p id="p-3" data-page="101">Only para on 101.</p>
<ul><li id="p-4" data-page="101">A list item anchor on 101.</li></ul>
</body></html>
"""

FIXTURE_HTML_SMALLER = """
<html><body>
<div id="page-100"></div>
<p id="p-1" data-page="100">First para on 100.</p>
<p id="p-2" data-page="100">Second para on 100.</p>
</body></html>
"""

DOC_META = {
    "html_url": "https://example.test/doc",
    "body_html_url": "https://example.test/full_text/doc.html",
    "start_page": 100,
    "end_page": 101,
    "volume": 91,
}


def _store_with_rule() -> tuple[Store, str]:
    s = Store(":memory:")
    key = s.create(
        Rule(
            title="Test Rule",
            url="https://www.federalregister.gov/documents/2026/07/16/2026-14327/test",
            document_number="2026-14327",
            fr_volume="91",
            fr_page="100",
        )
    )
    return s, key


def _grabbed_store(html: str = FIXTURE_HTML) -> tuple[Store, str]:
    s, key = _store_with_rule()
    frlink.grab(s, key, fetch=lambda _doc: (DOC_META, html))
    return s, key


# ── parse_anchors ───────────────────────────────────────────────────


class TestParseAnchors:
    def test_pages_ordinals_text(self) -> None:
        a = frlink.parse_anchors(FIXTURE_HTML)
        assert [(x.p_id, x.page, x.ordinal) for x in a] == [
            (1, 100, 1),
            (2, 100, 2),
            (3, 101, 1),
            (4, 101, 2),
        ]
        # entities decoded (&sect; &thinsp;), whitespace collapsed
        assert "414.1425" in a[0].text
        assert "§" in a[0].text
        assert "&sect;" not in a[0].text
        # inline markup stripped
        assert a[1].text == "Second para with markup on 100."
        # <li> anchors close with </li>, not </p> — text must not bleed
        assert a[3].text == "A list item anchor on 101."

    def test_empty_html_yields_nothing(self) -> None:
        assert frlink.parse_anchors("<html><body></body></html>") == []


# ── grab ────────────────────────────────────────────────────────────


class TestGrab:
    def test_grab_persists_anchors_and_doc_row(self) -> None:
        s, key = _grabbed_store()
        con = s._con()  # noqa: SLF001
        rows = con.execute(
            "SELECT p_id, page, ordinal FROM fr_anchors WHERE item_key = ? ORDER BY p_id",
            (key,),
        ).fetchall()
        assert [tuple(r) for r in rows] == [
            (1, 100, 1),
            (2, 100, 2),
            (3, 101, 1),
            (4, 101, 2),
        ]
        doc = con.execute(
            "SELECT document_number, html_url, start_page, end_page, fr_volume, "
            "n_paragraphs, n_pages FROM fr_anchor_docs WHERE item_key = ?",
            (key,),
        ).fetchone()
        assert tuple(doc) == (
            "2026-14327",
            "https://example.test/doc",
            100,
            101,
            91,
            4,
            2,
        )
        s.close()

    def test_regrab_replaces_partition(self) -> None:
        s, key = _grabbed_store()
        frlink.grab(
            s, key, force=True, fetch=lambda _doc: (DOC_META, FIXTURE_HTML_SMALLER)
        )
        con = s._con()  # noqa: SLF001
        n = con.execute(
            "SELECT count(*) FROM fr_anchors WHERE item_key = ?", (key,)
        ).fetchone()[0]
        assert n == 2
        s.close()

    def test_grab_skips_when_already_grabbed(self) -> None:
        s, key = _grabbed_store()
        out = frlink.grab(s, key, fetch=lambda _doc: (DOC_META, FIXTURE_HTML_SMALLER))
        assert out["skipped"] is True
        con = s._con()  # noqa: SLF001
        n = con.execute(
            "SELECT count(*) FROM fr_anchors WHERE item_key = ?", (key,)
        ).fetchone()[0]
        assert n == 4  # untouched
        s.close()


# ── resolve / transmutations / md_link (#635) ───────────────────────


class TestResolve:
    def test_page_ref(self) -> None:
        s, key = _grabbed_store()
        link = frlink.resolve("91 FR 100", store=s)
        assert link.url == "https://example.test/doc#page-100"
        assert (link.item_key, link.page, link.p_id) == (key, 100, None)
        s.close()

    def test_page_plus_ordinal(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.resolve("91 FR 100 ¶2", store=s)
        assert link.url == "https://example.test/doc#p-2"
        assert (link.page, link.p_id, link.ordinal) == (100, 2, 2)
        assert "markup" in link.snippet
        # alternate spellings
        assert frlink.resolve("91 FR 100, para 2", store=s).p_id == 2
        assert frlink.resolve("91 FR 101 p.2", store=s).p_id == 4
        s.close()

    def test_raw_anchor(self) -> None:
        s, key = _grabbed_store()
        link = frlink.resolve("p-3", store=s, item_key=key)
        assert link.url == "https://example.test/doc#p-3"
        assert link.page == 101
        with pytest.raises(ValueError, match="p-99"):
            frlink.resolve("p-99", store=s, item_key=key)
        s.close()

    def test_raw_anchor_without_key_single_doc_context(self) -> None:
        s, _key = _grabbed_store()
        # only one grabbed rule → unambiguous without item_key
        assert frlink.resolve("p-1", store=s).page == 100
        s.close()

    def test_quote(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.resolve("Only para on 101", store=s)
        assert link.p_id == 3
        assert link.page == 101
        s.close()

    def test_quote_multi_hit_prefers_shortest_start_match(self) -> None:
        dup_html = FIXTURE_HTML.replace(
            "</body>",
            '<p id="p-9" data-page="101">Only para on 101, repeated.</p></body>',
        )
        s, _key = _grabbed_store(dup_html)
        # both start with the quote → the tighter paragraph wins
        assert frlink.resolve("Only para on 101", store=s).p_id == 3
        s.close()

    def test_quote_no_hit_raises(self) -> None:
        s, _key = _grabbed_store()
        with pytest.raises(ValueError, match="[Nn]o paragraph"):
            frlink.resolve("nothing like this appears anywhere", store=s)
        s.close()

    def test_unknown_page_raises(self) -> None:
        s, _key = _grabbed_store()
        with pytest.raises(ValueError, match="99 FR 100"):
            frlink.resolve("99 FR 100", store=s)
        s.close()

    def test_ordinal_out_of_range_raises(self) -> None:
        s, _key = _grabbed_store()
        with pytest.raises(ValueError, match="2 paragraph"):
            frlink.resolve("91 FR 100 ¶7", store=s)
        s.close()


class TestTransmutations:
    def test_page_of(self) -> None:
        s, key = _grabbed_store()
        assert frlink.page_of(s, key, 3) == 101
        s.close()

    def test_paragraphs_of(self) -> None:
        s, _key = _grabbed_store()
        anchors = frlink.paragraphs_of("91 FR 100", store=s)
        assert [(a.p_id, a.ordinal) for a in anchors] == [(1, 1), (2, 2)]
        s.close()


class TestMdLink:
    def test_default_text_is_ref(self) -> None:
        s, _key = _grabbed_store()
        md = frlink.md_link("91 FR 100 ¶2", store=s, highlight=False)
        assert md == "[91 FR 100 ¶2](https://example.test/doc#p-2)"
        s.close()

    def test_custom_text(self) -> None:
        s, _key = _grabbed_store()
        md = frlink.md_link(
            "91 FR 100", store=s, text="the CF discussion", highlight=False
        )
        assert md == "[the CF discussion](https://example.test/doc#page-100)"
        s.close()


# ── place() (#637) ──────────────────────────────────────────────────


class TestPlace:
    def test_place_writes_fr_links_row(self) -> None:
        s, key = _grabbed_store()
        link = frlink.place(s, "91 FR 100 ¶2", label="the markup para")
        assert link.p_id == 2
        con = s._con()  # noqa: SLF001
        rows = con.execute(
            "SELECT item_key, p_id, page, label, url FROM fr_links"
        ).fetchall()
        assert [tuple(r) for r in rows] == [
            (key, 2, 100, "the markup para", "https://example.test/doc#p-2")
        ]
        s.close()

    def test_place_default_label_and_idempotency(self) -> None:
        s, _key = _grabbed_store()
        frlink.place(s, "91 FR 100")
        frlink.place(s, "91 FR 100")  # same URL → no duplicate
        con = s._con()  # noqa: SLF001
        rows = con.execute("SELECT label, url, p_id FROM fr_links").fetchall()
        assert len(rows) == 1
        assert rows[0]["label"] == "91 FR 100"
        assert rows[0]["p_id"] is None
        s.close()


# ── text fragments + highlight ──────────────────────────────────────


class TestTextFragment:
    def test_first_sentence_encoded(self) -> None:
        frag = frlink.text_fragment(
            "Under this proposal, the new G-codes apply. Second sentence."
        )
        assert frag == "Under%20this%20proposal%2C%20the%20new%20G%2Dcodes%20apply."

    def test_long_sentence_cut_at_word_boundary(self) -> None:
        text = "word " * 40
        frag = frlink.text_fragment(text.strip(), max_chars=22)
        assert frag == "word%20word%20word%20word"

    def test_empty(self) -> None:
        assert frlink.text_fragment("   ") == ""


class TestHighlight:
    def test_paragraph_ref_gets_fragment(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.resolve("91 FR 100 ¶2", store=s, highlight=True)
        assert link.url == (
            "https://example.test/doc#p-2:~:text=Second%20para%20with%20markup%20on%20100."
        )
        assert link.p_id == 2
        s.close()

    def test_page_cite_upgrades_to_first_paragraph_on_page(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.resolve("91 FR 101", store=s, highlight=True)
        assert link.p_id == 3
        assert link.url.startswith("https://example.test/doc#p-3:~:text=Only%20para")
        s.close()

    def test_page_cite_without_highlight_unchanged(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.resolve("91 FR 101", store=s)
        assert link.url == "https://example.test/doc#page-101"
        assert link.p_id is None
        s.close()

    def test_page_without_anchors_stays_page_link_even_when_highlighting(
        self,
    ) -> None:
        s, _key = _grabbed_store()
        s._con().execute("DELETE FROM fr_anchors WHERE page = 101")
        link = frlink.resolve("91 FR 101", store=s, highlight=True)
        assert link.url == "https://example.test/doc#page-101"
        s.close()

    def test_md_link_highlights_by_default(self) -> None:
        s, _key = _grabbed_store()
        md = frlink.md_link("91 FR 100 ¶1", store=s)
        assert ":~:text=" in md
        assert md.startswith("[91 FR 100 ¶1](")
        s.close()

    def test_place_never_records_fragment(self) -> None:
        s, _key = _grabbed_store()
        link = frlink.place(s, "91 FR 100 ¶1", label="x")
        assert ":~:text=" not in link.url
        (row,) = s._con().execute("SELECT url FROM fr_links").fetchall()
        assert ":~:text=" not in row["url"]
        s.close()


class TestQuoteTieBreak:
    def test_prefers_paragraph_starting_with_quote(self) -> None:
        html = """
        <p id="p-1" data-page="100">Shared quote text appears here first.</p>
        <p id="p-2" data-page="100">Before it, shared quote text appears here again.</p>
        """
        s, _key = _grabbed_store(html)
        link = frlink.resolve("shared quote text appears here", store=s)
        assert link.p_id == 1
        s.close()

    def test_still_raises_when_truly_ambiguous(self) -> None:
        html = """
        <p id="p-1" data-page="100">Shared quote text appears here first.</p>
        <p id="p-2" data-page="100">Shared quote text appears here again.</p>
        """
        s, _key = _grabbed_store(html)
        with pytest.raises(ValueError, match="matches 2 paragraphs"):
            frlink.resolve("shared quote text appears here", store=s)
        s.close()


# ── rule_kind ───────────────────────────────────────────────────────


def _store_with_text(title: str, paragraphs: list[str]) -> tuple[Store, str]:
    s, key = _store_with_rule()
    s._con().execute("UPDATE items SET title = ? WHERE key = ?", (title, key))  # noqa: SLF001
    s._con().executemany(  # noqa: SLF001
        "INSERT INTO fr_anchors (item_key, p_id, page, ordinal, text) VALUES (?,?,?,?,?)",
        [(key, i + 1, 100, i + 1, text) for i, text in enumerate(paragraphs)],
    )
    s._con().commit()  # noqa: SLF001
    return s, key


class TestRuleKind:
    def test_proposed_from_text_when_the_title_says_nothing(self) -> None:
        s, key = _store_with_text(
            "Medicare and Medicaid Programs; CY 2027 Payment Policies Under the PFS",
            [
                "In this proposed rule we propose to update the conversion factor.",
                "We are proposing to revise the telehealth list.",
                "Commenters on the CY 2026 final rule asked for more time.",
            ],
        )
        assert frlink.rule_kind(s, key) == "proposed"

    def test_final_from_text_when_the_title_says_nothing(self) -> None:
        s, key = _store_with_text(
            "Medicare and Medicaid Programs; CY 2025 Payment Policies Under the PFS",
            [
                "After considering comments, we are finalizing the policy as proposed.",
                "In this final rule we finalized the APCM codes.",
                "We proposed the change in the CY 2025 PFS proposed rule.",
            ],
        )
        assert frlink.rule_kind(s, key) == "final"

    def test_past_tense_we_proposed_is_not_a_proposed_signal(self) -> None:
        s, key = _store_with_text(
            "Medicare Program; CY 2021 Payment Policies Under the PFS",
            ["We proposed this in July. We are finalizing it now in this final rule."],
        )
        assert frlink.rule_kind(s, key) == "final"

    def test_silent_text_falls_back_to_the_title(self) -> None:
        s, key = _store_with_text(
            "Medicare Program; CY 2016 PFS Correction", ["Corrections to page 1."]
        )
        assert frlink.rule_kind(s, key) == "correction"
        s2, key2 = _store_with_text("Medicare Program; CY 2005 PFS Proposed Rule", [])
        assert frlink.rule_kind(s2, key2) == "proposed"

    def test_undecidable_is_rule_never_final(self) -> None:
        s, key = _store_with_text("Medicare Program; Revisions to Payment Policies", [])
        assert frlink.rule_kind(s, key) == "rule"

    def test_unknown_item_is_rule(self) -> None:
        s, _ = _store_with_rule()
        assert frlink.rule_kind(s, "NOPE0000") == "rule"

    def test_kind_from_title(self) -> None:
        assert frlink.kind_from_title("CY 2025 PFS Proposed Rule") == "proposed"
        assert frlink.kind_from_title("CY 2025 PFS Final Rule") == "final"
        assert (
            frlink.kind_from_title("CY 2025 PFS Final Rule; Correction") == "correction"
        )
        assert frlink.kind_from_title("CY 2025 Payment Policies") == "rule"

    def test_correction_title_wins_over_the_quoted_text(self) -> None:
        """A correction notice quotes the rule it corrects ("this final
        rule …"), so its text is not evidence of its own kind."""
        s, key = _store_with_text(
            "Medicare Program; CY 2025 PFS Final Rule; Correction",
            [
                "On page 97864 of this final rule, we are finalizing … is corrected to read"
            ],
        )
        assert frlink.rule_kind(s, key) == "correction"

    def test_title_kwarg_skips_the_item_lookup(self) -> None:
        s, key = _store_with_text("x", ["In this proposed rule we propose a change."])
        assert (
            frlink.rule_kind(s, "NOPE0000", title="CY 2005 PFS Final Rule") == "final"
        )
        assert frlink.rule_kind(s, key, title="") == "proposed"


# ── Force regrab keeps references (PI capture → official anchors) ───

_A = "Budget neutrality requires that increases or decreases in relative value units stay within limits."
_B = "The qualifying conversion factor receives a three quarters of a percent update each year."
_C = "Home or residence visits count toward assignment with or without the new complexity modifiers."
_D = "We received no comments on this section of the proposal and finalize it as proposed today."

PI_HTML = f"""<html><body>
<p id="p-1" data-page="7">{_A}</p>
<p id="p-2" data-page="7">{_B}</p>
<p id="p-3" data-page="8">{_C}</p>
<p id="p-4" data-page="9">{_D}</p>
</body></html>"""

# The official issue adds a front-matter paragraph, so every id shifts.
OFFICIAL_HTML = f"""<html><body>
<p id="p-1" data-page="70992">Billing Code 4120-01-P front matter for this document.</p>
<p id="p-2" data-page="70992">{_A}</p>
<p id="p-3" data-page="70993">{_B}</p>
<p id="p-4" data-page="70994">{_C} ( printed page 70995) [40]</p>
<p id="p-5" data-page="70995">{_D}</p>
</body></html>"""

PI_META = {**DOC_META, "html_url": "https://example.test/public-inspection/doc"}
OFFICIAL_META = {
    **DOC_META,
    "html_url": "https://example.test/documents/doc",
    "start_page": 70992,
    "end_page": 70995,
}


def _pinned_store() -> tuple[Store, str]:
    s, key = _store_with_rule()
    frlink.grab(s, key, fetch=lambda _d: (PI_META, PI_HTML))
    s.upsert_pincite(f"notes.{key}.x", key, "p-3", "fr_anchor", "home visits count")
    s.add_tag(key, f"pin:{key}/p-3")
    item_id = s._con().execute("SELECT id FROM items WHERE key=?", (key,)).fetchone()[0]  # noqa: SLF001
    s._con().execute(  # noqa: SLF001
        "INSERT INTO notes (item_id, title, content) VALUES (?, 'n', ?)",
        (
            item_id,
            '<p>See <a href="https://example.test/public-inspection/doc#p-3">p-3</a> (PI PDF p. 8).</p>',
        ),
    )
    s._con().commit()  # noqa: SLF001
    return s, key


class TestRegrabKeepsReferences:
    def test_pincite_tag_and_note_follow_the_paragraph_text(self) -> None:
        s, key = _pinned_store()
        out = frlink.grab(
            s, key, force=True, fetch=lambda _d: (OFFICIAL_META, OFFICIAL_HTML)
        )
        assert out["remapped"] == {3: 4}
        con = s._con()  # noqa: SLF001
        assert [
            r[0]
            for r in con.execute(
                "SELECT locator FROM pincites WHERE item_key=?", (key,)
            )
        ] == ["p-4"]
        assert (
            f"pin:{key}/p-4" in s.get(key).tags
            and f"pin:{key}/p-3" not in s.get(key).tags
        )
        note = con.execute("SELECT content FROM notes").fetchone()[0]
        assert (
            'href="https://example.test/documents/doc#p-4">p-4</a> (91 FR 70994)'
            in note
        )
        assert "public-inspection" not in note and "PI PDF" not in note

    def test_unplaceable_reference_refuses_and_writes_nothing(self) -> None:
        s, key = _pinned_store()
        moved = OFFICIAL_HTML.replace(
            _C, "An entirely different paragraph about something else here today."
        )
        with pytest.raises(frlink.UnresolvedAnchors, match="p-3"):
            frlink.grab(s, key, force=True, fetch=lambda _d: (OFFICIAL_META, moved))
        con = s._con()  # noqa: SLF001
        assert con.execute("SELECT locator FROM pincites").fetchone()[0] == "p-3"
        assert (
            con.execute("SELECT html_url FROM fr_anchor_docs").fetchone()[0]
            == PI_META["html_url"]
        )

    def test_override_places_what_text_cannot(self) -> None:
        s, key = _pinned_store()
        moved = OFFICIAL_HTML.replace(
            _C, "An entirely different paragraph about something else here today."
        )
        out = frlink.grab(
            s,
            key,
            force=True,
            fetch=lambda _d: (OFFICIAL_META, moved),
            overrides={3: 5},
        )
        assert out["remapped"] == {3: 5}
        assert s._con().execute("SELECT locator FROM pincites").fetchone()[0] == "p-5"  # noqa: SLF001


class TestFrGrabCli:
    def test_refusal_exits_nonzero_and_names_the_paragraph(self, monkeypatch) -> None:
        from typer.testing import CliRunner

        from cli.bib import app

        s, key = _pinned_store()
        moved = OFFICIAL_HTML.replace(
            _C, "An entirely different paragraph about something else here today."
        )
        real = frlink.grab
        monkeypatch.setattr("bib.connect", lambda: s)
        monkeypatch.setattr(
            frlink,
            "grab",
            lambda st, k, **kw: real(
                st, k, fetch=lambda _d: (OFFICIAL_META, moved), **kw
            ),
        )
        res = CliRunner().invoke(app, ["fr-grab", "--key", key, "--force"])
        assert res.exit_code == 1 and "p-3" in res.output
        res = CliRunner().invoke(
            app, ["fr-grab", "--key", key, "--force", "--map", "p-3=p-5"]
        )
        assert res.exit_code == 0, res.output
        assert "1 pinned paragraph ids moved" in res.output

    def test_bad_map_spec_is_rejected(self) -> None:
        from typer.testing import CliRunner

        from cli.bib import app

        res = CliRunner().invoke(app, ["fr-grab", "--key", "X", "--map", "3=5"])
        assert res.exit_code != 0