stack/tests/bib/test_regulations_gov_exercise.py

"""Exercise bib.regulations_gov — backfill_details, upsert_comment, helpers."""

from __future__ import annotations

from unittest.mock import MagicMock, patch

import httpx

from bib.regulations_gov import (
    Client,
    Comment,
    _byline,
    _docket_from_comment_id,
    _filename_from,
    _hash,
    _parse_comment,
    _reg_date,
    _slug,
    backfill_details,
    upsert_comment,
)


class TestHelpers:
    def test_docket_from_comment_id(self):
        assert _docket_from_comment_id("CMS-2025-0304-14107") == "CMS-2025-0304"
        assert _docket_from_comment_id("AB") == "AB"

    def test_slug(self):
        assert _slug("American Hospital Association") == "american-hospital-association"
        assert _slug("") == ""

    def test_filename_from(self):
        assert _filename_from("https://example.com/path/file.pdf") == "file.pdf"
        assert _filename_from("https://example.com/file.pdf?v=1") == "file.pdf"
        assert _filename_from("") == ""

    def test_hash(self):
        h = _hash("test")
        assert len(h) == 10

    def test_reg_date(self):
        assert _reg_date("2017-08-30T18:35:48Z") == "2017-08-30 18:35:48"
        assert _reg_date("2017-08-30 18:35:48") == "2017-08-30 18:35:48"
        assert _reg_date("") == ""


class TestParseComment:
    def test_full(self):
        row = {
            "id": "CMS-2023-0001-0001",
            "attributes": {
                "title": "My Comment",
                "postedDate": "2023-01-15T00:00:00Z",
                "receivedDate": "2023-01-10T00:00:00Z",
                "docketId": "CMS-2023-0001",
                "commentOnId": "obj1",
                "firstName": "John",
                "lastName": "Doe",
                "organization": "ACME Inc",
                "comment": "This is my comment",
                "attachmentCount": 2,
            },
        }
        c = _parse_comment(row)
        assert c.id == "CMS-2023-0001-0001"
        assert c.first_name == "John"
        assert c.attachment_count == 2

    def test_minimal(self):
        c = _parse_comment({"id": "C1", "attributes": {}})
        assert c.id == "C1"
        assert c.title == ""


class TestByline:
    def test_org(self):
        c = Comment(
            id="C1",
            title="T",
            posted_date="2023-01-01",
            received_date="2023-01-01",
            docket_id="D1",
            comment_on_id="OBJ1",
            organization="ACME",
        )
        assert _byline(c) == "ACME"

    def test_name(self):
        c = Comment(
            id="C1",
            title="T",
            posted_date="2023-01-01",
            received_date="2023-01-01",
            docket_id="D1",
            comment_on_id="OBJ1",
            first_name="John",
            last_name="Doe",
        )
        assert _byline(c) == "John Doe"

    def test_empty(self):
        c = Comment(
            id="C1",
            title="T",
            posted_date="2023-01-01",
            received_date="2023-01-01",
            docket_id="D1",
            comment_on_id="OBJ1",
        )
        assert _byline(c) == ""


class TestUpsertComment:
    def test_full(self):
        store = MagicMock()
        store.upsert.return_value = "KEY1"
        c = Comment(
            id="CMS-2023-0001-0001",
            title="My Comment",
            posted_date="2023-01-15",
            received_date="2023-01-10",
            docket_id="CMS-2023-0001",
            comment_on_id="obj1",
            organization="ACME Inc",
            comment_text="This is my comment",
        )
        key = upsert_comment(store, c, cms_id="CMS-1776-P", extra_tags=["custom:tag"])
        assert key == "KEY1"

    def test_no_title_uses_comment_on_id(self):
        store = MagicMock()
        store.upsert.return_value = "KEY2"
        c = Comment(
            id="C2",
            title="",
            posted_date="",
            received_date="2023-01-01",
            docket_id="D1",
            comment_on_id="OBJ1",
        )
        key = upsert_comment(store, c)
        assert key == "KEY2"


class TestBackfillDetails:
    def test_docket_scope_limits_candidates(self, tmp_path):
        """docket= restricts the walk to that docket's stubs, so a
        freshly farmed docket enriches without a full-store crawl."""
        from bib.item import Item
        from bib.store import Store

        store = Store(str(tmp_path / "bib.sqlite"), storage_dir=tmp_path / "storage")
        for docket, cid in [
            ("CMS-2026-2377", "CMS-2026-2377-0001"),
            ("CMS-2025-0304", "CMS-2025-0304-0001"),
        ]:
            store.create(
                Item(
                    item_type="source",
                    title=f"comment {cid}",
                    url=f"https://www.regulations.gov/comment/{cid}",
                    tags=["source:regulations-gov", f"reg-docket:{docket}"],
                )
            )

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "Body", "organization": "Org"}},
            "included": [],
        }
        stats = backfill_details(
            store,
            api,
            docket="CMS-2026-2377",
            log_path=tmp_path / "log.txt",
            scratch_root=tmp_path / "scratch",
        )
        store.close()

        assert stats["enriched"] == 1
        api.get_comment_detail.assert_called_once_with("CMS-2026-2377-0001")

    def test_basic_enrichment(self, tmp_path):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {
                "attributes": {
                    "comment": "Full body text here",
                    "organization": "Test Org",
                }
            },
            "included": [],
        }

        stats = backfill_details(store, api, limit=1, log_path=tmp_path / "log.txt")
        assert stats["enriched"] == 1
        assert stats["errors"] == 0

    def test_404_marks_gone(self):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        resp = MagicMock()
        resp.status_code = 404
        api.get_comment_detail.side_effect = httpx.HTTPStatusError(
            "404", request=MagicMock(), response=resp
        )

        stats = backfill_details(store, api, limit=1)
        assert stats["gone"] == 1

    def test_transport_error(self):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.side_effect = httpx.ConnectError("fail")

        stats = backfill_details(store, api, limit=1)
        assert stats["errors"] == 1

    def test_empty_url(self):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {"id": 1, "key": "K1", "url": ""},
        ]
        api = MagicMock()
        stats = backfill_details(store, api)
        assert stats["errors"] == 1

    def test_with_attachments(self, tmp_path):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "text", "organization": ""}},
            "included": [
                {
                    "type": "attachments",
                    "attributes": {
                        "fileFormats": [{"fileUrl": "https://example.com/file.pdf"}]
                    },
                }
            ],
        }
        api.download_attachments_batch.return_value = ([tmp_path / "file.pdf"], [])

        stats = backfill_details(
            store, api, limit=1, scratch_root=tmp_path, commit_every=1
        )
        assert stats["attached"] >= 1

    def test_attachment_gone_tags_item(self, tmp_path):
        """CDN serves 200 OK with empty body → item gets attachment:gone tag."""
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "text", "organization": ""}},
            "included": [
                {
                    "type": "attachments",
                    "attributes": {
                        "fileFormats": [{"fileUrl": "https://example.com/gone.pdf"}]
                    },
                }
            ],
        }
        api.download_attachments_batch.return_value = (
            [],
            ["https://example.com/gone.pdf"],
        )

        stats = backfill_details(
            store, api, limit=1, scratch_root=tmp_path, commit_every=1
        )
        assert stats["att_gone"] == 1
        assert stats["attached"] == 0
        store.add_tag.assert_any_call("K1", "attachment:gone")
        store.add_tag.assert_any_call("K1", "enriched:ok")

    def test_500_error_skips(self):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        resp = MagicMock()
        resp.status_code = 500
        api.get_comment_detail.side_effect = httpx.HTTPStatusError(
            "500", request=MagicMock(), response=resp
        )

        stats = backfill_details(store, api, limit=1)
        assert stats["errors"] == 1


class TestClientResolveDocket:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_finds_docket(self, mc_sleep):
        mock_http = MagicMock()
        resp = MagicMock()
        resp.status_code = 200
        resp.json.return_value = {
            "data": [{"attributes": {"docketId": "CMS-2023-0001"}}]
        }
        resp.raise_for_status = MagicMock()
        mock_http.get.return_value = resp
        c = Client(sleep=0, client=mock_http)
        assert c.resolve_docket("CMS-1676-P") == "CMS-2023-0001"

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_no_match(self, mc_sleep):
        mock_http = MagicMock()
        resp = MagicMock()
        resp.status_code = 200
        resp.json.return_value = {"data": []}
        resp.raise_for_status = MagicMock()
        mock_http.get.return_value = resp
        c = Client(sleep=0, client=mock_http)
        assert c.resolve_docket("NOPE") is None


class TestClientRateLimit:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_429_retries(self, mc_sleep):
        mock_http = MagicMock()
        rate_resp = MagicMock()
        rate_resp.status_code = 429
        ok_resp = MagicMock()
        ok_resp.status_code = 200
        ok_resp.json.return_value = {"data": []}
        ok_resp.raise_for_status = MagicMock()
        mock_http.get.side_effect = [rate_resp, ok_resp]
        c = Client(sleep=0, client=mock_http)
        result = c._get("/documents")
        assert result == {"data": []}


class TestIterCommentsPagination:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_date_cursor_advance(self, mc_sleep):
        mock_http = MagicMock()

        page1_rows = [
            {
                "id": f"C{i}",
                "attributes": {
                    "title": f"Comment {i}",
                    "postedDate": "2023-01-01T00:00:00Z",
                    "receivedDate": "2023-01-01T00:00:00Z",
                    "docketId": "D1",
                    "commentOnId": "OBJ1",
                    "lastModifiedDate": "2023-06-01T12:00:00Z",
                },
            }
            for i in range(250)
        ]
        page1_resp = MagicMock()
        page1_resp.status_code = 200
        page1_resp.json.return_value = {
            "data": page1_rows,
            "meta": {"totalPages": 1},
        }
        page1_resp.raise_for_status = MagicMock()

        page2_resp = MagicMock()
        page2_resp.status_code = 200
        page2_resp.json.return_value = {"data": [], "meta": {"totalPages": 1}}
        page2_resp.raise_for_status = MagicMock()

        mock_http.get.side_effect = [page1_resp, page2_resp]
        c = Client(sleep=0, client=mock_http)
        comments = list(c.iter_comments("OBJ1"))
        assert len(comments) == 250


class TestFindDocumentsPagination:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_multi_page(self, mc_sleep):
        mock_http = MagicMock()
        page1 = MagicMock()
        page1.status_code = 200
        page1.json.return_value = {
            "data": [{"id": "D1"}],
            "meta": {"totalPages": 2},
        }
        page1.raise_for_status = MagicMock()
        page2 = MagicMock()
        page2.status_code = 200
        page2.json.return_value = {
            "data": [{"id": "D2"}],
            "meta": {"totalPages": 2},
        }
        page2.raise_for_status = MagicMock()
        mock_http.get.side_effect = [page1, page2]
        c = Client(sleep=0, client=mock_http)
        docs = c.find_documents_in_docket("CMS-2023-0001")
        assert len(docs) == 2


class TestIterCommentsErrors:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_http_status_error(self, mc_sleep):
        mock_http = MagicMock()
        resp = MagicMock()
        resp.status_code = 400
        import httpx

        mock_http.get.side_effect = httpx.HTTPStatusError(
            "400", request=MagicMock(), response=resp
        )
        c = Client(sleep=0, client=mock_http)
        comments = list(c.iter_comments("OBJ1"))
        assert len(comments) == 0

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_same_cursor_breaks(self, mc_sleep):
        mock_http = MagicMock()
        rows = [
            {
                "id": f"C{i}",
                "attributes": {
                    "title": f"Comment {i}",
                    "postedDate": "2023-01-01T00:00:00Z",
                    "receivedDate": "2023-01-01",
                    "docketId": "D1",
                    "commentOnId": "OBJ1",
                    "lastModifiedDate": "2023-06-01T12:00:00Z",
                },
            }
            for i in range(250)
        ]
        # Page 1: full page, totalPages=1 → triggers cursor advance
        resp1 = MagicMock()
        resp1.status_code = 200
        resp1.json.return_value = {"data": rows, "meta": {"totalPages": 1}}
        resp1.raise_for_status = MagicMock()
        # Page 2: same cursor value → should break
        resp2 = MagicMock()
        resp2.status_code = 200
        resp2.json.return_value = {"data": rows, "meta": {"totalPages": 1}}
        resp2.raise_for_status = MagicMock()
        mock_http.get.side_effect = [resp1, resp2]
        c = Client(sleep=0, client=mock_http)
        comments = list(c.iter_comments("OBJ1"))
        # Should get 250 from page 1, then break on same cursor
        assert len(comments) == 500  # or 250 depending on cursor logic


class TestBackfillDetailsOrgTag:
    def test_org_tag_and_download_fail(self, tmp_path):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {
                "attributes": {
                    "comment": "My comment text",
                    "organization": "American Hospital Association",
                }
            },
            "included": [
                {
                    "type": "attachments",
                    "attributes": {
                        "fileFormats": [
                            {"fileUrl": "https://example.com/doc.pdf"},
                            {"fileUrl": ""},  # empty URL
                        ]
                    },
                }
            ],
        }
        api.download_attachment.return_value = None  # download fails
        api.download_attachments_batch.return_value = ([], [])  # all transient fails

        stats = backfill_details(
            store, api, limit=1, scratch_root=tmp_path, commit_every=1
        )
        assert stats["enriched"] == 1
        # org tag was attempted
        store.add_tag.assert_called()

    def test_add_tag_exception(self):
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]
        store.add_tag.side_effect = [
            Exception("org fail"),
            None,
        ]  # first org fails, second enriched:ok ok

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {
                "attributes": {
                    "comment": "text",
                    "organization": "Some Org",
                }
            },
            "included": [],
        }

        stats = backfill_details(store, api, limit=1)
        assert stats["enriched"] == 1


class TestClientDownloadEdge:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_existing_file(self, mc_sleep, tmp_path):
        f = tmp_path / "existing.pdf"
        f.write_bytes(b"existing")
        mock_http = MagicMock()
        c = Client(sleep=0, client=mock_http)
        result = c.download_attachment("https://x.com/existing.pdf", tmp_path)
        assert result == f
        mock_http.stream.assert_not_called()

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_http_error(self, mc_sleep, tmp_path):
        mock_http = MagicMock()
        mock_http.stream.side_effect = httpx.ConnectError("fail")
        c = Client(sleep=0, client=mock_http)
        result = c.download_attachment("https://x.com/new.pdf", tmp_path)
        assert result is None

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_non_200_status(self, mc_sleep, tmp_path):
        """Lines 295, 300: download_attachment returns None on non-200."""
        mock_http = MagicMock()
        stream_ctx = MagicMock()
        stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
        stream_ctx.__exit__ = MagicMock(return_value=False)
        stream_ctx.status_code = 403
        mock_http.stream.return_value = stream_ctx
        c = Client(sleep=0, client=mock_http)
        result = c.download_attachment("https://x.com/new.pdf", tmp_path)
        assert result is None

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_empty_body_raises_attachment_missing(self, mc_sleep, tmp_path):
        """200 OK with 0-byte body → raises AttachmentMissing, deletes file."""
        import pytest

        from bib.regulations_gov import AttachmentMissing

        mock_http = MagicMock()
        stream_ctx = MagicMock()
        stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
        stream_ctx.__exit__ = MagicMock(return_value=False)
        stream_ctx.status_code = 200
        stream_ctx.iter_bytes.return_value = iter([])  # empty body
        mock_http.stream.return_value = stream_ctx
        c = Client(sleep=0, client=mock_http)
        url = "https://x.com/empty.pdf"
        with pytest.raises(AttachmentMissing):
            c.download_attachment(url, tmp_path)
        assert not (tmp_path / "empty.pdf").exists()

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_existing_zero_byte_file_re_downloads(self, mc_sleep, tmp_path):
        """Cached 0-byte file is rejected — refetch happens instead of returning the stale empty path."""
        f = tmp_path / "stale.pdf"
        f.write_bytes(b"")  # 0 bytes
        mock_http = MagicMock()
        stream_ctx = MagicMock()
        stream_ctx.__enter__ = MagicMock(return_value=stream_ctx)
        stream_ctx.__exit__ = MagicMock(return_value=False)
        stream_ctx.status_code = 200
        stream_ctx.iter_bytes.return_value = iter([b"real bytes"])
        mock_http.stream.return_value = stream_ctx
        c = Client(sleep=0, client=mock_http)
        result = c.download_attachment("https://x.com/stale.pdf", tmp_path)
        assert result == f
        assert f.read_bytes() == b"real bytes"
        mock_http.stream.assert_called_once()


class TestIterCommentsPageAdvance:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_multi_page_within_total(self, mc_sleep):
        """Lines 222, 223: page increments when page < totalPages."""
        mock_http = MagicMock()
        # Page 1: 250 results, totalPages=2, so page advances to 2
        rows = [
            {
                "id": f"C{i}",
                "attributes": {
                    "title": f"C{i}",
                    "postedDate": "2023-01-01",
                    "receivedDate": "2023-01-01",
                    "docketId": "D1",
                    "commentOnId": "OBJ1",
                    "lastModifiedDate": "2023-06-01T12:00:00Z",
                },
            }
            for i in range(250)
        ]
        resp1 = MagicMock()
        resp1.status_code = 200
        resp1.json.return_value = {"data": rows, "meta": {"totalPages": 2}}
        resp1.raise_for_status = MagicMock()
        # Page 2: fewer than 250 → stops
        resp2 = MagicMock()
        resp2.status_code = 200
        resp2.json.return_value = {"data": [rows[0]], "meta": {"totalPages": 2}}
        resp2.raise_for_status = MagicMock()
        mock_http.get.side_effect = [resp1, resp2]
        c = Client(sleep=0, client=mock_http)
        comments = list(c.iter_comments("OBJ1"))
        assert len(comments) == 251

    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_no_last_modified_date_breaks(self, mc_sleep):
        """Line 228: breaks if no lastModifiedDate in last row."""
        mock_http = MagicMock()
        rows = [
            {
                "id": f"C{i}",
                "attributes": {
                    "title": f"C{i}",
                    "postedDate": "2023-01-01",
                    "receivedDate": "2023-01-01",
                    "docketId": "D1",
                    "commentOnId": "OBJ1",
                    # No lastModifiedDate!
                },
            }
            for i in range(250)
        ]
        resp = MagicMock()
        resp.status_code = 200
        resp.json.return_value = {"data": rows, "meta": {"totalPages": 1}}
        resp.raise_for_status = MagicMock()
        mock_http.get.return_value = resp
        c = Client(sleep=0, client=mock_http)
        comments = list(c.iter_comments("OBJ1"))
        assert len(comments) == 250


class TestAttachmentsForNonAttachment:
    @patch.dict("os.environ", {"REGULATIONS_GOV_API_KEY": "test"})
    @patch("bib.regulations_gov.time.sleep")
    def test_non_attachment_included_skipped(self, mc_sleep):
        """Line 246: included records with type != 'attachments' skipped."""
        mock_http = MagicMock()
        resp = MagicMock()
        resp.status_code = 200
        resp.json.return_value = {
            "data": {"attributes": {}},
            "included": [
                {"type": "other", "attributes": {}},
                {
                    "type": "attachments",
                    "attributes": {
                        "fileFormats": [
                            {"fileUrl": "https://x.com/a.pdf", "format": "pdf"}
                        ]
                    },
                },
            ],
        }
        resp.raise_for_status = MagicMock()
        mock_http.get.return_value = resp
        c = Client(sleep=0, client=mock_http)
        atts = c.attachments_for("C1")
        assert len(atts) == 1


class TestBackfillAddTagGone:
    def test_add_tag_gone_failure(self):
        """Lines 386-388: add_tag for 'enriched:gone' fails."""
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]
        store.add_tag.side_effect = Exception("add_tag fail")

        api = MagicMock()
        resp = MagicMock()
        resp.status_code = 404
        api.get_comment_detail.side_effect = httpx.HTTPStatusError(
            "404", request=MagicMock(), response=resp
        )
        stats = backfill_details(store, api, limit=1)
        assert stats["errors"] == 1

    def test_attach_file_exception(self, tmp_path):
        """Lines 426, 428: attach_file exception is caught."""
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]
        store.attach_file.side_effect = Exception("dup")

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "text", "organization": ""}},
            "included": [
                {
                    "type": "attachments",
                    "attributes": {
                        "fileFormats": [{"fileUrl": "https://x.com/doc.pdf"}]
                    },
                }
            ],
        }
        api.download_attachment.return_value = tmp_path / "doc.pdf"
        api.download_attachments_batch.return_value = ([tmp_path / "doc.pdf"], [])

        stats = backfill_details(
            store, api, limit=1, scratch_root=tmp_path, commit_every=1
        )
        assert stats["enriched"] == 1
        assert stats["attached"] == 0  # attach_file failed

    def test_enriched_ok_tag_fails(self):
        """Lines 432, 433: add_tag for 'enriched:ok' fails."""
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        # add_tag for org succeeds, add_tag for enriched:ok fails
        def add_tag_side(key, tag):
            if tag == "enriched:ok":
                raise Exception("fail")

        store.add_tag.side_effect = add_tag_side

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "text", "organization": "Org"}},
            "included": [],
        }

        stats = backfill_details(store, api, limit=1)
        assert stats["enriched"] == 1

    def test_empty_att_url_skipped(self, tmp_path):
        """Line 415: empty fileUrl in attachment is skipped."""
        store = MagicMock()
        con = MagicMock()
        store._con.return_value = con
        con.execute.return_value.fetchall.return_value = [
            {
                "id": 1,
                "key": "K1",
                "url": "https://www.regulations.gov/comment/CMS-2023-0001-0001",
            },
        ]

        api = MagicMock()
        api.get_comment_detail.return_value = {
            "data": {"attributes": {"comment": "text", "organization": ""}},
            "included": [
                {
                    "type": "attachments",
                    "attributes": {"fileFormats": [{"fileUrl": ""}]},
                }
            ],
        }

        stats = backfill_details(store, api, limit=1, scratch_root=tmp_path)
        assert stats["enriched"] == 1
        assert stats["attached"] == 0