feat(drugs): drug-tier foundation — hoard and manifest, code levels, NDC-11, stack drugs group
This commit is contained in:
@@ -19,6 +19,7 @@ from cli.bls import app as bls_app
|
||||
from cli.comments import app as comments_app
|
||||
from cli.db import app as db_app
|
||||
from cli.docs import app as docs_app
|
||||
from cli.drugs import app as drugs_app
|
||||
from cli.generate import app as generate_app
|
||||
from cli.health import health
|
||||
from cli.lake import app as lake_app
|
||||
@@ -89,6 +90,11 @@ app.add_typer(
|
||||
name="market",
|
||||
help="Market and enrichment tier: exposed companies, prices, lobbying, Congress.",
|
||||
)
|
||||
app.add_typer(
|
||||
drugs_app,
|
||||
name="drugs",
|
||||
help="Products, drugs and diseases: ASP, NDC registry, code lineage, MCD links.",
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
91
src/cli/drugs.py
Normal file
91
src/cli/drugs.py
Normal file
@@ -0,0 +1,91 @@
|
||||
"""``stack drugs`` — products, drugs and diseases tier."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import typer
|
||||
|
||||
from drugs.tables import table_counts
|
||||
from market.tables import read_gaps, write_gap
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
app = typer.Typer(no_args_is_help=True)
|
||||
|
||||
#: gap sources owned by this tier (``<provider>:<command>``)
|
||||
SOURCES = frozenset(
|
||||
{
|
||||
"cms-asp:asp",
|
||||
"cms-asp:xwalk",
|
||||
"fda-ndc:fda",
|
||||
"fda-orange:fda",
|
||||
"fda-purple:fda",
|
||||
"sec-edgar:registry",
|
||||
"pfs:links",
|
||||
"cms-hcpcs:hcpcs",
|
||||
"cdc-icd:icd",
|
||||
"cms-icd:icd",
|
||||
"cms-mcd:mcd",
|
||||
"zotero:books",
|
||||
"fr:drug-events",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
def _batch() -> Any:
|
||||
from conf.connect import duckdb_batch
|
||||
|
||||
return duckdb_batch("aco")
|
||||
|
||||
|
||||
def _read() -> Any:
|
||||
from conf.connect import duckdb
|
||||
|
||||
return duckdb("aco", read_only=True)
|
||||
|
||||
|
||||
def _store() -> Any:
|
||||
from conf.connect import bib
|
||||
|
||||
return bib()
|
||||
|
||||
|
||||
def _publish() -> None:
|
||||
from conf.connect import publish_replica
|
||||
|
||||
typer.echo(f"replica → {publish_replica('aco')}")
|
||||
|
||||
|
||||
def _gap(source: str, note: str, *, write: bool) -> None:
|
||||
"""Print ``gap: <source>: <note>``; with *write* record it (and nothing
|
||||
else)."""
|
||||
typer.echo(f"gap: {source}: {note}")
|
||||
if write:
|
||||
with _batch() as con:
|
||||
write_gap(con, source, note)
|
||||
_publish()
|
||||
|
||||
|
||||
@app.command()
|
||||
def status() -> None:
|
||||
"""Row counts of the drugs tables and this tier's open gaps."""
|
||||
con = _read()
|
||||
try:
|
||||
counts = table_counts(con)
|
||||
try:
|
||||
gaps = [g for g in read_gaps(con) if g.source in SOURCES]
|
||||
except Exception as exc:
|
||||
if not is_missing_table_error(exc):
|
||||
raise
|
||||
gaps = []
|
||||
finally:
|
||||
con.close()
|
||||
if not counts:
|
||||
typer.echo("no drugs tables yet")
|
||||
for name, n in counts.items():
|
||||
typer.echo(f"{name}: {n}")
|
||||
if not gaps:
|
||||
typer.echo("no gaps recorded")
|
||||
for g in gaps:
|
||||
typer.echo(f"gap: {g.source}: {g.note} ({g.recorded_on})")
|
||||
2
src/drugs/__init__.py
Normal file
2
src/drugs/__init__.py
Normal file
@@ -0,0 +1,2 @@
|
||||
"""Products, drugs and diseases tier: ASP series, NDC registry, code lineage,
|
||||
MCD links. Nothing here opens a database or imports numpy/pandas at import."""
|
||||
100
src/drugs/codes.py
Normal file
100
src/drugs/codes.py
Normal file
@@ -0,0 +1,100 @@
|
||||
"""Code-level helpers: HCPCS levels, NDC-11, application ids, quarters."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
_L1 = re.compile(r"^\d{4}[0-9A-Z]$")
|
||||
_L2 = re.compile(r"^[ABCEGHJKLMPQRSTUV]\d{4}$")
|
||||
_CDT = re.compile(r"^D\d{4}$")
|
||||
|
||||
|
||||
def level(code: str) -> str:
|
||||
"""``"I"`` (CPT), ``"II"`` (HCPCS Level II), ``"CDT"`` or ``""``."""
|
||||
if _L1.match(code):
|
||||
return "I"
|
||||
if _L2.match(code):
|
||||
return "II"
|
||||
if _CDT.match(code):
|
||||
return "CDT"
|
||||
return ""
|
||||
|
||||
|
||||
def licensed(code: str) -> bool:
|
||||
"""Level I and CDT descriptors are licensed text and never stored."""
|
||||
return level(code) in ("I", "CDT")
|
||||
|
||||
|
||||
def ndc11(raw: str) -> str | None:
|
||||
"""The 11-digit NDC of a 4-4-2, 5-3-2, 5-4-1 or 5-4-2 hyphenated code, or
|
||||
of 11 bare digits; anything else is ``None``."""
|
||||
s = raw.strip()
|
||||
if re.fullmatch(r"\d{11}", s):
|
||||
return s
|
||||
m = re.fullmatch(r"(\d+)-(\d+)-(\d+)", s)
|
||||
if not m:
|
||||
return None
|
||||
a, b, c = m.groups()
|
||||
shape = (len(a), len(b), len(c))
|
||||
if shape == (4, 4, 2):
|
||||
return "0" + a + b + c
|
||||
if shape == (5, 3, 2):
|
||||
return a + "0" + b + c
|
||||
if shape == (5, 4, 1):
|
||||
return a + b + "0" + c
|
||||
if shape == (5, 4, 2):
|
||||
return a + b + c
|
||||
return None
|
||||
|
||||
|
||||
_APPL = {"N": "NDA", "A": "ANDA", "BLA": "BLA"}
|
||||
|
||||
|
||||
def appl_id(kind: str, number: str) -> str:
|
||||
"""``("N","21436")`` -> ``NDA021436``; an id already spelled out passes."""
|
||||
if re.fullmatch(r"[A-Z]+\d{6}", kind):
|
||||
return kind
|
||||
return f"{_APPL.get(kind, kind)}{int(number):06d}"
|
||||
|
||||
|
||||
def expand(lo: str, hi: str) -> list[str]:
|
||||
"""Every code from *lo* to *hi* when they share a letter prefix and a
|
||||
numeric tail; otherwise just *lo*."""
|
||||
a = re.fullmatch(r"([A-Z]+)(\d+)", lo)
|
||||
b = re.fullmatch(r"([A-Z]+)(\d+)", hi)
|
||||
if not a or not b or a.group(1) != b.group(1):
|
||||
return [lo]
|
||||
width = len(a.group(2))
|
||||
start, end = int(a.group(2)), int(b.group(2))
|
||||
if end < start or len(b.group(2)) != width:
|
||||
return [lo]
|
||||
return [f"{a.group(1)}{n:0{width}d}" for n in range(start, end + 1)]
|
||||
|
||||
|
||||
_Q = {1: 1, 4: 2, 7: 3, 10: 4}
|
||||
|
||||
|
||||
def quarter(year: int, month: int) -> str | None:
|
||||
"""``"2026-Q4"`` for a quarter's first month, else ``None``."""
|
||||
q = _Q.get(month)
|
||||
return None if q is None else f"{year}-Q{q}"
|
||||
|
||||
|
||||
def quarter_start(q: str) -> date:
|
||||
y, n = q.split("-Q")
|
||||
return date(int(y), 3 * (int(n) - 1) + 1, 1)
|
||||
|
||||
|
||||
def quarter_end(q: str) -> date:
|
||||
s = quarter_start(q)
|
||||
nxt = date(s.year + (s.month == 10), (s.month + 3 - 1) % 12 + 1, 1)
|
||||
return nxt - timedelta(days=1)
|
||||
|
||||
|
||||
def fy_effective(vintage: str) -> date:
|
||||
"""``"2026"`` (a fiscal year) -> 2025-10-01; ``"2026-04"`` -> 2026-04-01."""
|
||||
if "-" in vintage:
|
||||
y, m = vintage.split("-")
|
||||
return date(int(y), int(m), 1)
|
||||
return date(int(vintage) - 1, 10, 1)
|
||||
107
src/drugs/http.py
Normal file
107
src/drugs/http.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""The drug tier's hoard: polite fetches through ``market.http`` (honest
|
||||
User-Agent, pacer, one retry loop), files never overwritten, one manifest
|
||||
line per distinct file, one bib ``Source`` per file (never an attachment)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from market.http import Pacer, client, get, newest, save # noqa: F401
|
||||
|
||||
PACE = {"cms": 1.0, "fda": 1.0, "cdc": 1.0, "mcd": 1.0}
|
||||
MANIFEST = "MANIFEST.jsonl"
|
||||
|
||||
|
||||
def data_root() -> Path:
|
||||
import conf
|
||||
|
||||
return conf.path("db.aco").parent
|
||||
|
||||
|
||||
def path(*parts: str) -> Path:
|
||||
return data_root().joinpath(*parts)
|
||||
|
||||
|
||||
def pacer(provider: str) -> Pacer:
|
||||
return Pacer(PACE[provider])
|
||||
|
||||
|
||||
_MANIFESTS: dict[Path, set[tuple[str, str]]] = {}
|
||||
|
||||
|
||||
def manifest(p: Path, url: str, note: str = "") -> None:
|
||||
"""Append one line to ``data/drugs/MANIFEST.jsonl`` for the hoarded file
|
||||
*p* (url, path relative to ``data/``, sha256, bytes, fetched_at), once per
|
||||
distinct ``(path, sha256)``."""
|
||||
root = data_root()
|
||||
mf = root / "drugs" / MANIFEST
|
||||
body = p.read_bytes()
|
||||
sha = hashlib.sha256(body).hexdigest()
|
||||
rel = p.relative_to(root).as_posix()
|
||||
seen = _MANIFESTS.get(mf)
|
||||
if seen is None:
|
||||
seen = set()
|
||||
if mf.exists():
|
||||
for line in mf.read_text().splitlines():
|
||||
rec = json.loads(line)
|
||||
seen.add((rec["path"], rec["sha256"]))
|
||||
_MANIFESTS[mf] = seen
|
||||
if (rel, sha) in seen:
|
||||
return
|
||||
fetched = _dt.datetime.fromtimestamp(p.stat().st_mtime, _dt.timezone.utc)
|
||||
rec: dict[str, Any] = {
|
||||
"url": url,
|
||||
"path": rel,
|
||||
"sha256": sha,
|
||||
"bytes": len(body),
|
||||
"fetched_at": fetched.isoformat(timespec="seconds"),
|
||||
}
|
||||
if note:
|
||||
rec["note"] = note
|
||||
mf.parent.mkdir(parents=True, exist_ok=True)
|
||||
with mf.open("a") as fh:
|
||||
fh.write(json.dumps(rec) + "\n")
|
||||
seen.add((rel, sha))
|
||||
|
||||
|
||||
def fetch(
|
||||
cl: Any,
|
||||
url: str,
|
||||
dest: Path,
|
||||
*,
|
||||
provider: str,
|
||||
source: str,
|
||||
refresh: bool = False,
|
||||
note: str = "",
|
||||
) -> Path:
|
||||
"""The newest hoarded copy of *dest*, downloading it first when absent or
|
||||
when *refresh* (different bytes land beside the old copy). Raises
|
||||
``market.http.Refused`` on a refusal; nothing is written then."""
|
||||
if not refresh and dest.exists():
|
||||
return newest(dest)
|
||||
r = get(cl, url, pacer=pacer(provider), source=source)
|
||||
out = save(dest, r.content)
|
||||
manifest(out, url, note)
|
||||
return out
|
||||
|
||||
|
||||
def cite_file(store: Any, *, url: str, title: str, provider: str, table: str) -> str:
|
||||
"""A bib ``Source`` key for a hoarded file, deduplicated by URL; a
|
||||
citation only, never an attachment (bib syncs to Zotero)."""
|
||||
from bib.item import Source
|
||||
from bib.tag import Tag
|
||||
|
||||
item = Source(
|
||||
title=title,
|
||||
url=url,
|
||||
doc_type="dataset",
|
||||
access_date=_dt.date.today().isoformat(),
|
||||
)
|
||||
return store.upsert(
|
||||
item,
|
||||
tags=[Tag.module("drugs"), Tag.source(provider), Tag.table(table)],
|
||||
)
|
||||
63
src/drugs/tables.py
Normal file
63
src/drugs/tables.py
Normal file
@@ -0,0 +1,63 @@
|
||||
"""DDL, row dataclasses, writers and readers for the ``drugs`` schema.
|
||||
|
||||
Every ``drugs.*`` table has its DDL in :data:`_DDL`. Later tasks append
|
||||
tables. Nothing opens a database at import.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import astuple, fields
|
||||
from typing import Any
|
||||
|
||||
_DDL = """
|
||||
CREATE SCHEMA IF NOT EXISTS drugs
|
||||
"""
|
||||
|
||||
|
||||
def ensure_tables(con: Any) -> None:
|
||||
for stmt in _DDL.strip().split(";"):
|
||||
if stmt.strip():
|
||||
con.execute(stmt)
|
||||
|
||||
|
||||
def table_counts(con: Any) -> dict[str, int]:
|
||||
"""Row count of every ``drugs.*`` table (empty without the schema)."""
|
||||
names = [
|
||||
n
|
||||
for (n,) in con.execute(
|
||||
"SELECT table_name FROM information_schema.tables "
|
||||
"WHERE table_schema = 'drugs' ORDER BY table_name"
|
||||
).fetchall()
|
||||
]
|
||||
return {
|
||||
f"drugs.{n}": con.execute(f"SELECT count(*) FROM drugs.{n}").fetchone()[0]
|
||||
for n in names
|
||||
}
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements."""
|
||||
if not rows:
|
||||
return 0
|
||||
cls = type(rows[0])
|
||||
names = [f.name for f in fields(cls)]
|
||||
one = f"({', '.join('?' * len(names))})"
|
||||
for part in _chunks(rows, chunk):
|
||||
con.execute(
|
||||
f"INSERT INTO {table} ({', '.join(names)}) VALUES "
|
||||
+ ", ".join([one] * len(part)),
|
||||
[v for r in part for v in astuple(r)],
|
||||
)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _select(con: Any, cls: type, table: str, where: str = "", params: Sequence = ()):
|
||||
cols = ", ".join(f.name for f in fields(cls))
|
||||
sql = f"SELECT {cols} FROM {table}" + (f" WHERE {where}" if where else "")
|
||||
return con.execute(sql, list(params)).fetchall()
|
||||
|
||||
|
||||
def _chunks(items: Sequence[Any], n: int = 500) -> Iterable[Sequence[Any]]:
|
||||
for i in range(0, len(items), n):
|
||||
yield items[i : i + n]
|
||||
142
tests/cli/test_drugs_cli.py
Normal file
142
tests/cli/test_drugs_cli.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""stack drugs — status."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import duckdb
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
import cli.drugs as dg
|
||||
from cli import app
|
||||
from drugs import tables as T
|
||||
from market import tables as MT
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
class _Batch:
|
||||
def __init__(self, c):
|
||||
self.c = c
|
||||
|
||||
def __enter__(self):
|
||||
return self.c
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def env(monkeypatch):
|
||||
c = duckdb.connect(":memory:")
|
||||
monkeypatch.setattr(dg, "_batch", lambda: _Batch(c))
|
||||
monkeypatch.setattr(dg, "_read", lambda: c.cursor())
|
||||
published: list[bool] = []
|
||||
monkeypatch.setattr(dg, "_publish", lambda: published.append(True))
|
||||
yield c, published
|
||||
c.close()
|
||||
|
||||
|
||||
def test_status_prints_counts_and_gaps(env):
|
||||
c, _ = env
|
||||
T.ensure_tables(c)
|
||||
c.execute("CREATE TABLE drugs.probe (a INTEGER)")
|
||||
c.execute("INSERT INTO drugs.probe VALUES (1), (2)")
|
||||
MT.write_gap(c, "cms-asp:asp", "quarter 2026-Q4 missing")
|
||||
MT.write_gap(c, "tiingo:prices", "no key")
|
||||
res = runner.invoke(app, ["drugs", "status"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "drugs.probe: 2" in res.output
|
||||
assert "gap: cms-asp:asp: quarter 2026-Q4 missing" in res.output
|
||||
assert "tiingo" not in res.output
|
||||
|
||||
|
||||
def test_status_without_tables_or_gaps(env, monkeypatch):
|
||||
c, _ = env
|
||||
res = runner.invoke(app, ["drugs", "status"])
|
||||
assert res.exit_code == 0 and "no drugs tables yet" in res.output
|
||||
T.ensure_tables(c)
|
||||
res = runner.invoke(app, ["drugs", "status"])
|
||||
assert "no gaps recorded" in res.output
|
||||
|
||||
|
||||
def test_status_missing_market_table_and_errors(env, monkeypatch):
|
||||
c, _ = env
|
||||
T.ensure_tables(c)
|
||||
res = runner.invoke(app, ["drugs", "status"])
|
||||
assert res.exit_code == 0
|
||||
|
||||
class _Broken:
|
||||
def execute(self, *a, **k):
|
||||
raise RuntimeError("x")
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dg, "_read", lambda: _Broken())
|
||||
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
|
||||
|
||||
|
||||
def test_gap_helper_writes_only_gap(env):
|
||||
c, published = env
|
||||
dg._gap("cms-asp:asp", "x", write=False)
|
||||
assert MT.read_gaps(c) == [] if _has_gap_table(c) else True
|
||||
assert published == []
|
||||
dg._gap("cms-asp:asp", "x", write=True)
|
||||
assert [g.source for g in MT.read_gaps(c)] == ["cms-asp:asp"]
|
||||
assert published == [True]
|
||||
assert (
|
||||
c.execute("SELECT count(*) FROM cms.ingest_log").fetchone() == (0,)
|
||||
if _has_log(c)
|
||||
else True
|
||||
)
|
||||
|
||||
|
||||
def _has_gap_table(c):
|
||||
return bool(
|
||||
c.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_name='source_gap'"
|
||||
).fetchall()
|
||||
)
|
||||
|
||||
|
||||
def _has_log(c):
|
||||
return bool(
|
||||
c.execute(
|
||||
"SELECT 1 FROM information_schema.tables WHERE table_name='ingest_log'"
|
||||
).fetchall()
|
||||
)
|
||||
|
||||
|
||||
def test_default_indirections_resolve_lazily(monkeypatch, capsys):
|
||||
import conf.connect as cc
|
||||
|
||||
calls: list[str] = []
|
||||
monkeypatch.setattr(cc, "duckdb_batch", lambda n: calls.append(f"batch {n}") or n)
|
||||
monkeypatch.setattr(
|
||||
cc,
|
||||
"duckdb",
|
||||
lambda n, read_only=False: calls.append(f"read {n} {read_only}") or n,
|
||||
)
|
||||
monkeypatch.setattr(cc, "bib", lambda: calls.append("bib") or "store")
|
||||
monkeypatch.setattr(cc, "publish_replica", lambda n: f"/r/{n}")
|
||||
assert dg._batch() == "aco" and dg._read() == "aco" and dg._store() == "store"
|
||||
dg._publish()
|
||||
assert calls == ["batch aco", "read aco True", "bib"]
|
||||
assert "replica → /r/aco" in capsys.readouterr().out
|
||||
|
||||
|
||||
def test_status_gap_read_error_propagates(env, monkeypatch):
|
||||
c, _ = env
|
||||
T.ensure_tables(c)
|
||||
|
||||
class _Con:
|
||||
def execute(self, sql, *a):
|
||||
if "source_gap" in sql:
|
||||
raise RuntimeError("boom")
|
||||
return c.execute(sql, *a)
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(dg, "_read", lambda: _Con())
|
||||
assert isinstance(runner.invoke(app, ["drugs", "status"]).exception, RuntimeError)
|
||||
0
tests/drugs/__init__.py
Normal file
0
tests/drugs/__init__.py
Normal file
56
tests/drugs/test_codes.py
Normal file
56
tests/drugs/test_codes.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from datetime import date
|
||||
|
||||
from drugs import codes
|
||||
|
||||
|
||||
def test_level():
|
||||
assert codes.level("90371") == "I"
|
||||
assert codes.level("0202U") == "I"
|
||||
assert codes.level("J9035") == "II"
|
||||
assert codes.level("Q4101") == "II"
|
||||
assert codes.level("D0120") == "CDT"
|
||||
assert codes.level("A1") == ""
|
||||
assert codes.licensed("D0120") is True
|
||||
assert codes.licensed("90371") is True
|
||||
assert codes.licensed("J9035") is False
|
||||
assert codes.licensed("") is False
|
||||
|
||||
|
||||
def test_ndc11_three_layouts():
|
||||
n = codes.ndc11
|
||||
assert n("0002-1433-80") == "00002143380"
|
||||
assert n("50242-040-62") == "50242004062"
|
||||
assert n("60574-4114-1") == "60574411401"
|
||||
assert n("13533-0636-01") == "13533063601"
|
||||
assert n("13533063601") == "13533063601"
|
||||
assert n("1353306360") is None
|
||||
assert n("ALT-ID 12") is None
|
||||
assert n("123-45-6") is None
|
||||
assert n(" 0002-1433-80 ") == "00002143380"
|
||||
|
||||
|
||||
def test_appl_id():
|
||||
assert codes.appl_id("N", "21436") == "NDA021436"
|
||||
assert codes.appl_id("A", "90123") == "ANDA090123"
|
||||
assert codes.appl_id("BLA", "125057") == "BLA125057"
|
||||
assert codes.appl_id("NDA021436", "") == "NDA021436"
|
||||
assert codes.appl_id("X", "5") == "X000005"
|
||||
|
||||
|
||||
def test_expand():
|
||||
assert codes.expand("J0287", "J0289") == ["J0287", "J0288", "J0289"]
|
||||
assert codes.expand("J0287", "K0289") == ["J0287"]
|
||||
assert codes.expand("J0287", "J0287") == ["J0287"]
|
||||
assert codes.expand("J0289", "J0287") == ["J0289"]
|
||||
assert codes.expand("JABCD", "JABCF") == ["JABCD"]
|
||||
|
||||
|
||||
def test_quarters_and_fy():
|
||||
assert codes.quarter(2026, 10) == "2026-Q4"
|
||||
assert codes.quarter(2026, 1) == "2026-Q1"
|
||||
assert codes.quarter(2026, 3) is None
|
||||
assert codes.quarter_start("2026-Q4") == date(2026, 10, 1)
|
||||
assert codes.quarter_end("2026-Q4") == date(2026, 12, 31)
|
||||
assert codes.quarter_end("2026-Q1") == date(2026, 3, 31)
|
||||
assert codes.fy_effective("2026") == date(2025, 10, 1)
|
||||
assert codes.fy_effective("2026-04") == date(2026, 4, 1)
|
||||
144
tests/drugs/test_http.py
Normal file
144
tests/drugs/test_http.py
Normal file
@@ -0,0 +1,144 @@
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from drugs import http as dh
|
||||
from market import http as mh
|
||||
|
||||
|
||||
class _NoWait:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def root(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
||||
dh._MANIFESTS.clear()
|
||||
return tmp_path
|
||||
|
||||
|
||||
def _client(body, status=200, seen=None):
|
||||
def handler(req):
|
||||
if seen is not None:
|
||||
seen.append(req)
|
||||
return httpx.Response(
|
||||
status, content=body[0] if isinstance(body, list) else body
|
||||
)
|
||||
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def _fetch(cl, dest, **kw):
|
||||
return dh.fetch(
|
||||
cl, "https://x.test/a.csv", dest, provider="cms", source="cms-asp", **kw
|
||||
)
|
||||
|
||||
|
||||
def test_pace_and_pacer():
|
||||
assert dh.PACE == {"cms": 1.0, "fda": 1.0, "cdc": 1.0, "mcd": 1.0}
|
||||
p = dh.pacer("fda")
|
||||
assert isinstance(p, mh.Pacer) and p.min_interval == 1.0
|
||||
|
||||
|
||||
def test_data_root_and_path(monkeypatch, tmp_path):
|
||||
import conf
|
||||
|
||||
monkeypatch.setattr(conf, "path", lambda k: tmp_path / "aco.duckdb")
|
||||
assert dh.data_root() == tmp_path
|
||||
monkeypatch.setattr(dh, "data_root", lambda: tmp_path)
|
||||
assert dh.path("drugs", "a") == tmp_path / "drugs" / "a"
|
||||
|
||||
|
||||
def test_fetch_hoards_and_reuses(root, monkeypatch):
|
||||
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
||||
seen: list = []
|
||||
dest = root / "drugs" / "asp" / "a.csv"
|
||||
cl = _client(b"one", seen=seen)
|
||||
out = _fetch(cl, dest, note="n")
|
||||
assert out == dest and dest.read_bytes() == b"one"
|
||||
mf = root / "drugs" / "MANIFEST.jsonl"
|
||||
lines = mf.read_text().splitlines()
|
||||
assert len(lines) == 1
|
||||
rec = json.loads(lines[0])
|
||||
assert rec["url"] == "https://x.test/a.csv"
|
||||
assert rec["path"] == "drugs/asp/a.csv" and rec["bytes"] == 3
|
||||
assert rec["note"] == "n" and len(rec["sha256"]) == 64
|
||||
# reuse: no request
|
||||
assert _fetch(cl, dest) == dest
|
||||
assert len(seen) == 1
|
||||
# refresh with different bytes lands beside
|
||||
cl2 = _client(b"two", seen=seen)
|
||||
out2 = _fetch(cl2, dest, refresh=True)
|
||||
assert out2 != dest and out2.suffix == ".csv" and out2.stem.startswith("a.")
|
||||
assert out2.read_bytes() == b"two" and dest.read_bytes() == b"one"
|
||||
assert len(mf.read_text().splitlines()) == 2
|
||||
# newest copy is returned on reuse
|
||||
assert _fetch(cl2, dest) == out2
|
||||
# same bytes refreshed again: still two lines
|
||||
_fetch(cl2, dest, refresh=True)
|
||||
assert len(mf.read_text().splitlines()) == 2
|
||||
|
||||
|
||||
def test_manifest_dedupes_across_processes_and_omits_empty_note(root):
|
||||
p = root / "drugs" / "f.txt"
|
||||
p.parent.mkdir(parents=True)
|
||||
p.write_bytes(b"z")
|
||||
dh.manifest(p, "u")
|
||||
dh._MANIFESTS.clear()
|
||||
dh.manifest(p, "u")
|
||||
lines = (root / "drugs" / "MANIFEST.jsonl").read_text().splitlines()
|
||||
assert len(lines) == 1 and "note" not in json.loads(lines[0])
|
||||
|
||||
|
||||
def test_fetch_403_raises_refused(root, monkeypatch):
|
||||
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
||||
dest = root / "drugs" / "a.csv"
|
||||
with pytest.raises(mh.Refused) as ei:
|
||||
_fetch(_client(b"no", status=403), dest)
|
||||
assert ei.value.status == 403
|
||||
assert not dest.exists() and not (root / "drugs" / "MANIFEST.jsonl").exists()
|
||||
|
||||
|
||||
def test_user_agent(root, monkeypatch):
|
||||
monkeypatch.setattr(dh, "pacer", lambda provider: _NoWait())
|
||||
seen: list = []
|
||||
cl = mh.client("cms")
|
||||
cl._transport = httpx.MockTransport(
|
||||
lambda req: seen.append(req) or httpx.Response(200, content=b"x")
|
||||
)
|
||||
_fetch(cl, root / "drugs" / "u.csv")
|
||||
assert (
|
||||
seen[0].headers["user-agent"] == "stack-pfs-dossier/1.0 (ci@mail.fhirworx.io)"
|
||||
)
|
||||
|
||||
|
||||
class _Store:
|
||||
def __init__(self):
|
||||
self.items: dict[str, tuple] = {}
|
||||
self.attached = 0
|
||||
|
||||
def upsert(self, item, *, tags=None, collection=""):
|
||||
key = self.items.setdefault(item.url, (f"K{len(self.items)}", item, tags))[0]
|
||||
return key
|
||||
|
||||
def attach_file(self, *a, **k):
|
||||
self.attached += 1
|
||||
|
||||
|
||||
def test_cite_file_dedupes_by_url_no_attachment():
|
||||
st = _Store()
|
||||
kw = dict(
|
||||
url="https://x.test/a.csv", title="ASP", provider="cms", table="drugs.asp"
|
||||
)
|
||||
k1 = dh.cite_file(st, **kw)
|
||||
k2 = dh.cite_file(st, **kw)
|
||||
assert k1 == k2 and len(st.items) == 1 and st.attached == 0
|
||||
_, item, tags = st.items["https://x.test/a.csv"]
|
||||
assert item.item_type == "source"
|
||||
assert {(t.namespace, t.value) for t in tags} == {
|
||||
("module", "drugs"),
|
||||
("source", "cms"),
|
||||
("table", "drugs.asp"),
|
||||
}
|
||||
32
tests/drugs/test_imports.py
Normal file
32
tests/drugs/test_imports.py
Normal file
@@ -0,0 +1,32 @@
|
||||
"""The drug tier must not drag numpy/pandas into ``stack`` start-up."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
MODULES = ["drugs.http", "drugs.codes", "drugs.tables"]
|
||||
|
||||
_PROBE = """
|
||||
import builtins, importlib, sys
|
||||
_real = builtins.__import__
|
||||
def _fake(name, *a, **k):
|
||||
if name.split('.')[0] in {'numpy', 'pandas'}:
|
||||
raise ModuleNotFoundError(name)
|
||||
return _real(name, *a, **k)
|
||||
builtins.__import__ = _fake
|
||||
for m in %r:
|
||||
importlib.import_module(m)
|
||||
print('ok')
|
||||
"""
|
||||
|
||||
|
||||
def test_drugs_modules_import_without_numpy_pandas():
|
||||
r = subprocess.run(
|
||||
[sys.executable, "-c", _PROBE % MODULES],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert r.returncode == 0, r.stderr[-1200:]
|
||||
assert "ok" in r.stdout
|
||||
39
tests/drugs/test_tables.py
Normal file
39
tests/drugs/test_tables.py
Normal file
@@ -0,0 +1,39 @@
|
||||
import duckdb
|
||||
|
||||
from drugs import tables as T
|
||||
|
||||
|
||||
def test_ensure_tables_idempotent():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.ensure_tables(con)
|
||||
assert con.execute(
|
||||
"SELECT count(*) FROM information_schema.schemata WHERE schema_name='drugs'"
|
||||
).fetchone() == (1,)
|
||||
|
||||
|
||||
def test_table_names_empty():
|
||||
con = duckdb.connect(":memory:")
|
||||
assert T.table_counts(con) == {}
|
||||
T.ensure_tables(con)
|
||||
assert T.table_counts(con) == {}
|
||||
|
||||
|
||||
def test_helpers():
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class R:
|
||||
a: int
|
||||
b: str
|
||||
|
||||
con = duckdb.connect(":memory:")
|
||||
con.execute("CREATE TABLE t (a INTEGER, b VARCHAR)")
|
||||
assert T._insert(con, "t", []) == 0
|
||||
assert T._insert(con, "t", [R(1, "x"), R(2, "y"), R(3, "z")], chunk=2) == 3
|
||||
assert T._select(con, R, "t", "a > ?", [1]) == [(2, "y"), (3, "z")]
|
||||
assert T._select(con, R, "t") and [len(c) for c in T._chunks([1, 2, 3], 2)] == [
|
||||
2,
|
||||
1,
|
||||
]
|
||||
con.execute("CREATE TABLE drugs.x (a INTEGER)") if False else None
|
||||
Reference in New Issue
Block a user