diff --git a/src/prisma/harvest.py b/src/prisma/harvest.py new file mode 100644 index 0000000..1edf9d1 --- /dev/null +++ b/src/prisma/harvest.py @@ -0,0 +1,111 @@ +"""Grey-literature HTML parsers. The caller supplies the page. No HTTP. + +Each source has its own function. They share one stdlib parser because +the fixture contract is the same: one report link and one date. A source +whose real page differs gets its own fixture later. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from html.parser import HTMLParser + + +@dataclass(frozen=True) +class Harvested: + title: str + url: str + date: str + source: str + + +class _Page(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.links: list[tuple[str, str]] = [] + self.times: list[str] = [] + self._href: str | None = None + self._link_text: list[str] = [] + self._in_time = False + self._time_text: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + values = {name: value or "" for name, value in attrs} + if tag == "a" and values.get("href"): + self._href = values["href"] + self._link_text = [] + elif tag == "time": + self._in_time = True + self._time_text = [] + + def handle_endtag(self, tag: str) -> None: + if tag == "a" and self._href is not None: + self.links.append((self._href, "".join(self._link_text).strip())) + self._href = None + elif tag == "time" and self._in_time: + self.times.append("".join(self._time_text).strip()) + self._in_time = False + + def handle_data(self, data: str) -> None: + if self._href is not None: + self._link_text.append(data) + if self._in_time: + self._time_text.append(data) + + +def _parse(html: str, source: str) -> tuple[Harvested, ...]: + if not html: + return () + page = _Page() + page.feed(html) + page.close() + date = page.times[0] if page.times else "" + return tuple( + Harvested(title=title, url=url, date=date, source=source) + for url, title in page.links + ) + + +def parse_cms(html: str) -> tuple[Harvested, ...]: + return _parse(html, "cms") + + +def parse_medpac(html: str) -> tuple[Harvested, ...]: + return _parse(html, "medpac") + + +def parse_macpac(html: str) -> tuple[Harvested, ...]: + return _parse(html, "macpac") + + +def parse_aspe(html: str) -> tuple[Harvested, ...]: + return _parse(html, "aspe") + + +def parse_gao(html: str) -> tuple[Harvested, ...]: + return _parse(html, "gao") + + +def parse_cbo(html: str) -> tuple[Harvested, ...]: + return _parse(html, "cbo") + + +def parse_crs(html: str) -> tuple[Harvested, ...]: + return _parse(html, "crs") + + +def parse_kff(html: str) -> tuple[Harvested, ...]: + return _parse(html, "kff") + + +PARSERS: dict[str, Callable[[str], tuple[Harvested, ...]]] = { + "cms": parse_cms, + "medpac": parse_medpac, + "macpac": parse_macpac, + "aspe": parse_aspe, + "gao": parse_gao, + "cbo": parse_cbo, + "crs": parse_crs, + "kff": parse_kff, +} diff --git a/tests/prisma/test_harvest.py b/tests/prisma/test_harvest.py new file mode 100644 index 0000000..944b9f6 --- /dev/null +++ b/tests/prisma/test_harvest.py @@ -0,0 +1,27 @@ +"""Grey-literature HTML parsers. No HTTP.""" + +from __future__ import annotations + +import pytest + +from prisma.harvest import PARSERS, Harvested + +_SOURCES = ("cms", "medpac", "macpac", "aspe", "gao", "cbo", "crs", "kff") +_HTML = 'Report title' + + +@pytest.mark.parametrize("source", _SOURCES) +def test_parser_reads_the_report_link_and_date(source: str): + assert PARSERS[source](_HTML) == ( + Harvested( + title="Report title", + url="https://example.test/r", + date="2026-01-02", + source=source, + ), + ) + + +@pytest.mark.parametrize("source", _SOURCES) +def test_empty_html_returns_nothing(source: str): + assert PARSERS[source]("") == ()