From cd4806dff6e2c39ca703fb2f3fb4f679a6dd1719 Mon Sep 17 00:00:00 2001 From: kert Date: Tue, 17 Nov 2026 12:23:14 -0500 Subject: [PATCH] =?UTF-8?q?feat(market):=20joint=20readers=20=E2=80=94=20m?= =?UTF-8?q?arket=20series=20beside=20BLS,=20FRED,=20SOI,=20PSPS=20and=20ut?= =?UTF-8?q?ilization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli/market.py | 59 ++++- src/market/enrich.py | 465 +++++++++++++++++++++++++++++++++ src/market/footprint.py | 44 +++- tests/cli/test_market_cli.py | 78 ++++++ tests/market/test_enrich.py | 431 ++++++++++++++++++++++++++++++ tests/market/test_footprint.py | 69 +++++ tests/market/test_imports.py | 1 + 7 files changed, 1140 insertions(+), 7 deletions(-) create mode 100644 src/market/enrich.py create mode 100644 tests/market/test_enrich.py diff --git a/src/cli/market.py b/src/cli/market.py index f3f9d00..4d6e91c 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -11,6 +11,7 @@ uv run stack market events --write # releases, meetings, tenures uv run stack market fred --write # FRED fredgraph series uv run stack market soi --write # IRS SOI ZIP income uv run stack market revenue --write # EDGAR annual revenue +uv run stack market joint --family HOME-VISITS # market beside BLS, FRED, PSPS uv run stack market gaps # sources that could not be read Every command fetches first, then opens one short DuckDB batch that only @@ -32,6 +33,7 @@ import typer from market import congress as congress_api from market import districts as district_api from market import edgar, http, tiingo +from market import enrich as enrich_api from market import events as events_api from market import fec as fec_api from market import footprint as footprint_api @@ -1972,8 +1974,8 @@ def district_view( congress: int = typer.Option(footprint_api.CONGRESS, "--congress"), top: int = typer.Option(20, "--top", help="How many districts to print."), ) -> None: - """Stored footprint joined to that congress's House member and health - committees. ``agi_k`` is empty until the SOI join.""" + """Stored footprint joined to that congress's House member, health + committees, and SOI income rolled by ZIP weight.""" _exactly_one(bool(entity), bool(family), "pass exactly one of --entity or --family") level = "family" if family else "entity" key = family or entity @@ -1990,10 +1992,11 @@ def district_view( return for view in shown: seats = ",".join(view.health_committees) or "-" + agi = f"{view.agi_k:g}" if view.agi_k is not None else "-" typer.echo( f"{view.district_id}\t{view.member}\t{view.party}\t{seats}\t" f"providers={view.n_providers:g}\tsites={view.n_sites:g}\t" - f"services={view.services:g}\tpayment={view.payment:g}" + f"services={view.services:g}\tpayment={view.payment:g}\tagi={agi}" ) @@ -2541,6 +2544,56 @@ def revenue( typer.echo(f"stored: {n} revenue rows") +def _echo_joint(label: str, values: dict[str, float | None]) -> None: + parts = [label] + for key, value in values.items(): + if value is None: + continue + parts.append(f"{key}={value:g}") + typer.echo("\t".join(parts)) + + +@app.command() +def joint( + family: str = typer.Option("", "--family", help="A code-family key."), + specialty: str = typer.Option("", "--specialty", help="A CMS specialty key."), + from_year: int = typer.Option(2015, "--from"), + to_year: int = typer.Option(2024, "--to"), + monthly: bool = typer.Option(False, "--monthly", help="One row per month."), +) -> None: + """Market series beside utilization, PSPS, OEWS, CES and FRED. + + Pass either flag or both. PSPS allowed charges need both, because they + sum the family's codes for one specialty. + """ + if not family and not specialty: + typer.echo("pass --family or --specialty") + raise typer.Exit(2) + years = tuple(range(from_year, to_year + 1)) + con = _read() + try: + if monthly: + rows = enrich_api.joint_monthly( + con, + family=family or None, + specialty=specialty or None, + years=years, + ) + for row in rows: + _echo_joint(f"{row.year}-{row.month:02d}", row.values) + return + annual = enrich_api.joint_annual( + con, + family=family or None, + specialty=specialty or None, + years=years, + ) + finally: + con.close() + for row in annual: + _echo_joint(str(row.year), row.values) + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/enrich.py b/src/market/enrich.py new file mode 100644 index 0000000..a09f34d --- /dev/null +++ b/src/market/enrich.py @@ -0,0 +1,465 @@ +"""Readers that set market series beside BLS, FRED, PSPS and utilization. + +No numpy or pandas at import time. A missing input is ``None``, never 0. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date +from typing import Any + +from pfs.codetables import is_missing_table_error + +SPECIALTY_SOC: dict[str, tuple[str, ...]] = { + "FAMILY PRACTICE": ("29-1215",), + "NEUROLOGY": ("29-1217",), + "INTERNAL MEDICINE": ("29-1216",), + "OBSTETRICS GYNECOLOGY": ("29-1218",), + "PEDIATRIC MEDICINE": ("29-1221",), + "PATHOLOGY": ("29-1222",), + "PSYCHIATRY": ("29-1223",), + "DIAGNOSTIC RADIOLOGY": ("29-1224",), + "ANESTHESIOLOGY": ("29-1211",), + "OPHTHALMOLOGY": ("29-1241",), + "CARDIOLOGY": ("29-1212",), + "ORTHOPEDIC SURGERY": ("29-1242",), + "DERMATOLOGY": ("29-1213",), + "GENERAL SURGERY": ("29-1249",), + "EMERGENCY MEDICINE": ("29-1214",), + "PHYSICAL THERAPIST": ("29-1123",), + "NURSE PRACTITIONER": ("29-1171",), + "PHYSICIAN ASSISTANT": ("29-1071",), +} + +FAMILY_INDUSTRY: dict[str, tuple[str, ...]] = { + "HOME-VISITS": ("621100", "621600"), +} +_DEFAULT_INDUSTRY = ("621100",) +_TRI = ("XLV", "IHF", "SPY") + + +@dataclass(frozen=True) +class JointYear: + """One calendar year of the joint display.""" + + year: int + values: dict[str, float | None] + sources: dict[str, str] + + +@dataclass(frozen=True) +class JointMonth: + """One calendar month of the joint display.""" + + year: int + month: int + values: dict[str, float | None] + sources: dict[str, str] + + +def _call(fn, default): + """``fn()``, or *default* when the table it reads is not there.""" + try: + return fn() + except Exception as exc: + if is_missing_table_error(exc): + return default + raise + + +def annual_tri(con: Any, symbol: str) -> dict[int, float]: + """Mean of the symbol's monthly total-return index, by calendar year. + + A non-``tri`` series for the same symbol is ignored. + """ + from market.tables import read_series + + rows = _call(lambda: read_series(con, [symbol]), None) + if rows is None: + return {} + buckets: dict[int, list[float]] = defaultdict(list) + for row in rows: + if row.series_id.endswith(":tri"): + buckets[row.year].append(row.value) + return {year: sum(vals) / len(vals) for year, vals in sorted(buckets.items())} + + +def joint_annual( + con: Any, + *, + family: str | None = None, + specialty: str | None = None, + years: Sequence[int], +) -> list[JointYear]: + """One :class:`JointYear` per requested year. + + ``psps::allowed`` is the family's codes only, matched on + :func:`pfs.commenters.specialty_key`. Specialty without a family leaves + that column out and sets ``sources["psps"]`` to ``"needs a family"``. + """ + from market.tables import FRED_SERIES + + wanted = tuple(int(year) for year in years) + industries = _industries(family) if family else () + socs = SPECIALTY_SOC.get(specialty, ()) if specialty else () + util, util_keys = _utilization(con, family) if family else ({}, {}) + psps = _psps(con, family, specialty, wanted) if family and specialty else {} + oews = _oews(con, socs, wanted) + ces = _ces_annual(con, industries, wanted) + fred = _fred_annual(con) + out: list[JointYear] = [] + for year in wanted: + values: dict[str, float | None] = {} + sources: dict[str, str] = {} + if family: + _put_market(con, values, sources, "family", family, year) + item = ",".join(sorted(util_keys.get(year, ()))) + _put( + values, + sources, + f"util:{family}:n_services", + util.get(year), + f"pfs.utilization:{item}", + ) + if specialty: + _put_market(con, values, sources, "specialty", specialty, year) + if family and specialty: + total, item = psps.get(year, (None, "")) + _put( + values, + sources, + f"psps:{specialty}:allowed", + total, + f"pfs.utilization_specialty:{item}", + ) + elif specialty: + sources["psps"] = "needs a family" + for soc in socs: + median, item = oews.get((soc, year), (None, "")) + _put( + values, + sources, + f"oews:{soc}:a_median", + median, + f"bls.oews:{item}", + ) + for industry in industries: + employment, item = ces.get((industry, year), (None, "")) + _put( + values, + sources, + f"ces:{industry}:employment", + employment, + f"bls.ces:{item}", + ) + for series_id in FRED_SERIES: + value, item = fred.get((series_id, year), (None, "")) + _put(values, sources, f"fred:{series_id}", value, f"econ.fred:{item}") + out.append(JointYear(year, values, sources)) + return out + + +def joint_monthly( + con: Any, + *, + family: str | None = None, + specialty: str | None = None, + years: Sequence[int], +) -> list[JointMonth]: + """Months in *years* that have a CES, FRED or benchmark total-return row. + + CES is the family's industries, or offices of physicians (``621100``) + when no family is named. *specialty* is accepted and unused: monthly + columns are employment, prices and the FRED mean, not a wage or a BHAR. + """ + del specialty + from market.tables import FRED_SERIES, read_fred, read_series + + wanted = {int(year) for year in years} + industries = _industries(family) if family else _DEFAULT_INDUSTRY + ces_at = _ces_monthly(con, industries, wanted) + fred_at: dict[tuple[str, int, int], list] = defaultdict(list) + fred_rows = _call(lambda: read_fred(con), None) or [] + for row in fred_rows: + parsed = _year_month(row.period) + if parsed is None or parsed[0] not in wanted: + continue + fred_at[(row.series_id, parsed[0], parsed[1])].append(row) + tri_at: dict[tuple[str, int, int], tuple[float, str]] = {} + series_rows = _call(lambda: read_series(con, list(_TRI)), None) or [] + for row in series_rows: + if not row.series_id.endswith(":tri") or row.year not in wanted: + continue + tri_at[(row.symbol, row.year, row.month)] = (row.value, row.item_key) + stamps = {(year, month) for _industry, year, month in ces_at} + stamps |= {(year, month) for _series, year, month in fred_at} + stamps |= {(year, month) for _symbol, year, month in tri_at} + out: list[JointMonth] = [] + for year, month in sorted(stamps): + values: dict[str, float | None] = {} + sources: dict[str, str] = {} + for industry in industries: + value, item = ces_at.get((industry, year, month), (None, "")) + _put( + values, + sources, + f"ces:{industry}:employment", + value, + f"bls.ces:{item}", + ) + for series_id in FRED_SERIES: + group = [ + row + for row in fred_at.get((series_id, year, month), ()) + if row.value is not None + ] + if group: + mean = sum(row.value for row in group) / len(group) + item = max(group, key=lambda row: row.period).item_key + else: + mean, item = None, "" + _put(values, sources, f"fred:{series_id}", mean, f"econ.fred:{item}") + for symbol in _TRI: + value, item = tri_at.get((symbol, year, month), (None, "")) + _put( + values, sources, f"market:{symbol}:tri", value, f"market.series:{item}" + ) + out.append(JointMonth(year, month, values, sources)) + return out + + +def _put( + values: dict[str, float | None], + sources: dict[str, str], + key: str, + value: float | None, + source: str, +) -> None: + values[key] = value + sources[key] = source + + +def _industries(family: str) -> tuple[str, ...]: + return FAMILY_INDUSTRY.get(family, _DEFAULT_INDUSTRY) + + +def _put_market( + con: Any, + values: dict[str, float | None], + sources: dict[str, str], + scope: str, + target: str, + year: int, +) -> None: + value, item = _market_bhar(con, scope, target, year) + _put( + values, sources, f"market:{scope}:{target}:bhar", value, f"market.series:{item}" + ) + + +def _market_bhar( + con: Any, scope: str, target: str, year: int +) -> tuple[float | None, str]: + """Equal-weight portfolio BHAR against SPY. The cite is SPY's tri series.""" + from market.portfolio import members, portfolio_returns, year_bhar + from market.tables import read_series + from market.tiingo import read_returns + + item = "" + spy = _call(lambda: read_series(con, ["SPY"]), None) or [] + for row in spy: + if row.series_id.endswith(":tri") and row.item_key: + item = row.item_key + break + found = _call( + lambda: members(con, scope=scope, target=target, on=date(year, 12, 31)), + None, + ) + if found is None: + return None, item + chosen, _gaps = found + if not chosen: + return None, item + returns = _call( + lambda: read_returns( + con, + [member.ticker for member in chosen] + ["SPY"], + date(year, 1, 1), + date(year, 12, 31), + ), + None, + ) + if returns is None: + return None, item + port = portfolio_returns(returns, chosen) + return year_bhar(port, returns.get("SPY", {}), year), item + + +def _family_codes(con: Any, family: str) -> list[str] | None: + """Member codes, or ``None`` when ``pfs.code_family`` is not on *con*.""" + found = _call( + lambda: con.execute( + "SELECT DISTINCT code FROM pfs.code_family WHERE key = ?", + [family], + ).fetchall(), + None, + ) + if found is None: + return None + return [row[0] for row in found] + + +def _utilization(con: Any, family: str) -> tuple[dict[int, float], dict[int, set[str]]]: + from pfs.event_study import family_series + + codes = _family_codes(con, family) + if not codes: + return {}, {} + totals = _call(lambda: family_series(con, family), None) + if totals is None: + return {}, {} + keys: dict[int, set[str]] = defaultdict(set) + marks = ",".join("?" * len(codes)) + found = con.execute( + "SELECT year, item_key FROM pfs.utilization " + f"WHERE hcpcs IN ({marks}) AND item_key <> ''", + [code.upper() for code in codes], + ).fetchall() + for year, item in found: + keys[int(year)].add(item) + return totals, keys + + +def _psps( + con: Any, family: str, specialty: str, years: Sequence[int] +) -> dict[int, tuple[float | None, str]]: + from pfs.commenters import specialty_key + from pfs.psps import read_by_specialty + + codes = _family_codes(con, family) + if not codes: + return {} + rows = _call(lambda: read_by_specialty(con, codes, years=years), None) + if rows is None: + return {} + wanted = specialty_key(specialty) + totals: dict[int, float] = {} + keys: dict[int, set[str]] = defaultdict(set) + seen: set[int] = set() + for row in rows: + if specialty_key(row.specialty) != wanted or row.allowed is None: + continue + seen.add(row.year) + totals[row.year] = totals.get(row.year, 0.0) + float(row.allowed) + if row.item_key: + keys[row.year].add(row.item_key) + return { + year: (totals.get(year), ",".join(sorted(keys.get(year, ())))) for year in seen + } + + +def _oews( + con: Any, socs: tuple[str, ...], years: Sequence[int] +) -> dict[tuple[str, int], tuple[float | None, str]]: + from bls.oews import read_occupations + + if not socs: + return {} + rows = _call(lambda: read_occupations(con, socs, years=years), None) + if rows is None: + return {} + out: dict[tuple[str, int], tuple[float | None, str]] = {} + for row in rows: + out[(row.occ_code, row.year)] = (row.a_median, row.item_key) + return out + + +def _employment_ids(industries: Sequence[str]) -> list[str]: + from bls.ces import SERIES + + ids: list[str] = [] + for industry in industries: + for series in SERIES.values(): + if series.industry_code == industry and series.data_type == "employment": + ids.append(series.series_id) + return ids + + +def _ces_annual( + con: Any, industries: Sequence[str], years: Sequence[int] +) -> dict[tuple[str, int], tuple[float | None, str]]: + from bls.ces import SERIES, read_series + + ids = _employment_ids(industries) + if not ids: + return {} + rows = _call(lambda: read_series(con, ids, annual=True), None) + if rows is None: + return {} + wanted = set(years) + out: dict[tuple[str, int], tuple[float | None, str]] = {} + for row in rows: + if row.year not in wanted: + continue + meta = SERIES[row.series_id] + out[(meta.industry_code, row.year)] = (row.value, row.item_key) + return out + + +def _ces_monthly( + con: Any, industries: Sequence[str], years: set[int] +) -> dict[tuple[str, int, int], tuple[float | None, str]]: + from bls.ces import SERIES, read_series + + ids = _employment_ids(industries) + rows = _call(lambda: read_series(con, ids, annual=False), None) if ids else [] + if rows is None: + return {} + out: dict[tuple[str, int, int], tuple[float | None, str]] = {} + for row in rows: + if row.year not in years or not 1 <= row.month <= 12: + continue + meta = SERIES[row.series_id] + out[(meta.industry_code, row.year, row.month)] = (row.value, row.item_key) + return out + + +def _fred_annual(con: Any) -> dict[tuple[str, int], tuple[float | None, str]]: + from market.tables import read_fred + + rows = _call(lambda: read_fred(con), None) + if rows is None: + return {} + grouped: dict[tuple[str, int], list] = defaultdict(list) + for row in rows: + year = row.year + if year is None and len(row.period) >= 4 and row.period[:4].isdigit(): + year = int(row.period[:4]) + if year is None: + continue + grouped[(row.series_id, int(year))].append(row) + out: dict[tuple[str, int], tuple[float | None, str]] = {} + for key, group in grouped.items(): + usable = [row for row in group if row.value is not None] + if not usable: + out[key] = (None, "") + continue + latest = max(usable, key=lambda row: row.period) + out[key] = (sum(row.value for row in usable) / len(usable), latest.item_key) + return out + + +def _year_month(period: str) -> tuple[int, int] | None: + parts = period.split("-") + if len(parts) < 2 or not (len(parts[0]) == 4 and parts[0].isdigit()): + return None + month_text = parts[1][:2] + if not month_text.isdigit(): + return None + month = int(month_text) + if not 1 <= month <= 12: + return None + return int(parts[0]), month diff --git a/src/market/footprint.py b/src/market/footprint.py index 844b61f..fee7d79 100644 --- a/src/market/footprint.py +++ b/src/market/footprint.py @@ -4,7 +4,8 @@ Sites come from the CMS All Owners snapshot of 2026-09-29 joined to enrollment ZIPs, and from NPPES organization rows. Family billers come from the 2024 Medicare physician-and-other-practitioners file. Amounts split across districts by :func:`market.districts.zip_weights`. An unmatched ZIP -is kept as ``??``. ``agi_k`` stays empty until the SOI join. +is kept as ``??``. ``agi_k`` is IRS SOI income rolled by ``zip_weights``, +using the latest SOI year at or before the footprint year. """ from __future__ import annotations @@ -24,6 +25,7 @@ from market.tables import ( read_assignments, read_footprints, read_members, + read_soi_zip, ) from pfs.codetables import is_missing_table_error @@ -73,8 +75,8 @@ class AllocItem: @dataclass(frozen=True) class DistrictView: - """One district's footprint beside its House member. ``agi_k`` is filled - by the SOI join.""" + """One district's footprint beside its House member. ``agi_k`` is ZIP + income rolled onto the district (thousands of dollars).""" district_id: str bioguide_id: str @@ -334,6 +336,7 @@ def district_view( if row.congress != congress or row.committee_id not in HEALTH_COMMITTEES: continue committees.setdefault(row.bioguide_id, set()).add(row.committee_id) + agi = _agi_by_district(con, congress, year) views: list[DistrictView] = [] for district, (providers, sites, services, payment) in totals.items(): member = members.get(district) @@ -349,13 +352,46 @@ def district_view( sites, services, payment, - None, + agi.get(district), ) ) views.sort(key=lambda v: (-v.payment, v.district_id)) return views +def _agi_by_district(con: Any, congress: int, year: int) -> dict[str, float]: + """SOI AGI (thousands) times ZIP weight. The latest SOI year at or + before *year* is used: the footprint year is 2024 and SOI stops at 2023. + + A non-positive weight adds nothing, so a zero-weight crosswalk stays + ``None`` rather than 0. + """ + try: + chosen = con.execute( + "SELECT max(year) FROM econ.soi_zip WHERE year <= ? AND agi_k IS NOT NULL", + [year], + ).fetchone()[0] + except Exception as exc: + if is_missing_table_error(exc): + return {} + raise + if chosen is None: + return {} + agi = { + row.zip: row.agi_k + for row in read_soi_zip(con, year=int(chosen)) + if row.agi_k is not None + } + weights = zip_weights(con, congress) + out: dict[str, float] = {} + for zipcode, amount in agi.items(): + for district, weight in weights.get(zipcode, ()): + if weight <= 0: + continue + out[district] = out.get(district, 0.0) + float(amount) * float(weight) + return out + + def _missing(exc: Exception) -> bool: return is_missing_table_error(exc) diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 0f7b223..2ee99dd 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -3563,3 +3563,81 @@ def test_revenue_cli_refusal_prints_gap(env, monkeypatch): assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output assert T.read_revenue(c, "0000001234") == [] assert published == [True] + + +def test_joint_cli_prints(env): + c, _, published, _ = env + T.write_fred(c, [T.FredRow("CPIMEDSL", 2020, "2020-06-01", 9.0, "fk")]) + res = runner.invoke( + app, + [ + "market", + "joint", + "--family", + "HOME-VISITS", + "--from", + "2020", + "--to", + "2020", + ], + ) + assert res.exit_code == 0, res.output + assert "2020" in res.output + assert "fred:CPIMEDSL=9" in res.output + assert published == [] + + +def test_joint_cli_monthly(env): + from bls.ces import CesRow + from bls.ces import write as write_ces + + c, _, published, _ = env + write_ces( + c, + [ + CesRow( + "CES6562110001", + "621100", + "Offices of physicians", + "employment", + "S", + 2024, + 3, + 7.0, + "ces-m", + ) + ], + ) + T.write_fred(c, [T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m")]) + T.write_series( + c, + "tiingo:XLV:tri", + [T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k")], + ) + res = runner.invoke( + app, + [ + "market", + "joint", + "--family", + "HOME-VISITS", + "--from", + "2024", + "--to", + "2024", + "--monthly", + ], + ) + assert res.exit_code == 0, res.output + assert "2024-03" in res.output + assert "ces:621100:employment=7" in res.output + assert "fred:CPIMEDSL=1.5" in res.output + assert "market:XLV:tri=120" in res.output + assert published == [] + + +def test_joint_cli_requires_family_or_specialty(env): + res = runner.invoke(app, ["market", "joint"]) + assert res.exit_code == 2 + assert "family" in res.output + assert "specialty" in res.output diff --git a/tests/market/test_enrich.py b/tests/market/test_enrich.py new file mode 100644 index 0000000..e22ce6a --- /dev/null +++ b/tests/market/test_enrich.py @@ -0,0 +1,431 @@ +"""Joint annual and monthly readers beside BLS, FRED, PSPS and utilization.""" + +from __future__ import annotations + +import re +from datetime import date, timedelta + +import duckdb + +from bls.ces import CesRow +from bls.ces import write as write_ces +from bls.oews import OewsRow +from bls.oews import write as write_oews +from market import enrich as E +from market import tables as T +from pfs.codetables import PspsRow, ensure_tables, write_psps +from pfs.utilization import UtilizationRow +from pfs.utilization import write as write_util + + +def _days(n: int, start: date = date(2020, 1, 2)) -> list[date]: + out: list[date] = [] + day = start + while len(out) < n: + if day.weekday() < 5: + out.append(day) + day += timedelta(days=1) + return out + + +def _oews(year: int, code: str, median: float, item_key: str) -> OewsRow: + return OewsRow( + year, + code, + "Family Medicine Physicians", + "detailed", + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + median, + None, + None, + "OEUN", + item_key, + ) + + +def _seed_joint(con) -> None: + """2020 inputs whose column values are fixed numbers.""" + T.ensure_tables(con) + ensure_tables(con) + con.execute( + "CREATE TABLE IF NOT EXISTS pfs.code_family (" + "key VARCHAR, name VARCHAR, code VARCHAR, role VARCHAR, since INTEGER, " + "until INTEGER, item_key VARCHAR, p_id INTEGER, note VARCHAR)" + ) + con.execute( + "INSERT INTO pfs.code_family VALUES " + "('HOME-VISITS', 'Home', '99349', 'member', 2015, NULL, 'fam', 0, ''), " + "('HOME-VISITS', 'Home', '99350', 'member', 2015, NULL, 'fam', 0, '')" + ) + write_util( + con, + 2020, + [ + UtilizationRow( + 2020, + "99349", + "", + "N", + "O", + 1, + 1, + 30.0, + None, + None, + None, + None, + None, + "d", + "util-k", + ), + UtilizationRow( + 2020, + "99350", + "", + "N", + "O", + 1, + 1, + 50.0, + None, + None, + None, + None, + None, + "d", + "util-k", + ), + UtilizationRow( + 2020, + "11111", + "", + "N", + "O", + 1, + 1, + 5.0, + None, + None, + None, + None, + None, + "d", + "util-other", + ), + ], + ) + write_psps( + con, + 2020, + "99349", + [PspsRow(2020, "99349", "08", "Family Practice", 1, 25.0, 1, "ds", "psps-k")], + ) + write_psps( + con, + 2020, + "99350", + [ + PspsRow( + 2020, "99350", "08", "Family Practice", 1, 100.0, 1, "ds", "psps-k" + ), + PspsRow( + 2020, "99350", "08", "Family Practice", 1, None, 1, "ds", "psps-null" + ), + PspsRow(2020, "99350", "09", "Family Practice", 1, 0.0, 1, "ds", ""), + PspsRow( + 2020, + "99350", + "16", + "Obstetrics/Gynecology", + 1, + 999.0, + 1, + "ds", + "psps-ob", + ), + ], + ) + write_psps( + con, + 2020, + "11111", + [PspsRow(2020, "11111", "08", "Family Practice", 1, 50.0, 1, "ds", "psps-k")], + ) + write_oews(con, 2020, [_oews(2020, "29-1215", 211000.0, "oews-k")]) + write_ces( + con, + [ + CesRow( + "CES6562110001", + "621100", + "Offices of physicians", + "employment", + "S", + 2020, + 1, + 10.0, + "ces-k", + ), + CesRow( + "CES6562110001", + "621100", + "Offices of physicians", + "employment", + "S", + 2020, + 2, + 30.0, + "ces-k", + ), + ], + ) + T.write_fred( + con, + [ + T.FredRow("CPIMEDSL", 2020, "2020-01-01", 2.0, "fred-k"), + T.FredRow("CPIMEDSL", 2020, "2020-02-01", 4.0, "fred-k"), + T.FredRow("CPIMEDSL", 2020, "2020-03-01", None, "fred-k"), + T.FredRow("CPIAUCSL", 2020, "2020-01-01", None, "none-k"), + T.FredRow("DTB3", None, "2020-05-01", 1.0, "parsed-k"), + T.FredRow("VIXCLS", None, "nope", 1.0, "bad-k"), + ], + ) + T.write_entities(con, [T.EntityRow("E1", "Home Co", "company")]) + T.write_securities( + con, "E1", [T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")] + ) + T.write_links( + con, + [ + T.LinkRow( + "E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x" + ), + T.LinkRow( + "E1", + "specialty", + "FAMILY PRACTICE", + "curated", + "curated", + "item_key", + "x", + ), + ], + ) + days = _days(200) + con.executemany( + "INSERT INTO market.price_daily (ticker, day, ret) VALUES (?, ?, 0)", + [(ticker, day) for ticker in ("HV", "SPY") for day in days], + ) + T.write_series( + con, + "tiingo:SPY:tri", + [T.MonthlyRow("tiingo:SPY:tri", "SPY", "tiingo", 2020, 12, 100.0, "spy-k")], + ) + + +def test_joint_annual_columns_and_sources(): + con = duckdb.connect(":memory:") + _seed_joint(con) + rows = E.joint_annual( + con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,) + ) + con.close() + assert [row.year for row in rows] == [2020] + values, sources = rows[0].values, rows[0].sources + assert values["market:family:HOME-VISITS:bhar"] == 0.0 + assert values["market:specialty:FAMILY PRACTICE:bhar"] == 0.0 + assert values["util:HOME-VISITS:n_services"] == 80.0 + assert values["psps:FAMILY PRACTICE:allowed"] == 125.0 + assert values["oews:29-1215:a_median"] == 211000.0 + assert values["ces:621100:employment"] == 20.0 + assert values["ces:621600:employment"] is None + assert values["fred:CPIMEDSL"] == 3.0 + assert {key for key in values if key.startswith("fred:")} == { + f"fred:{series}" for series in T.FRED_SERIES + } + assert sources["market:family:HOME-VISITS:bhar"] == "market.series:spy-k" + assert sources["util:HOME-VISITS:n_services"] == "pfs.utilization:util-k" + assert sources["psps:FAMILY PRACTICE:allowed"] == "pfs.utilization_specialty:psps-k" + assert sources["oews:29-1215:a_median"] == "bls.oews:oews-k" + assert sources["ces:621100:employment"] == "bls.ces:ces-k" + assert sources["fred:CPIMEDSL"] == "econ.fred:fred-k" + + +def test_joint_psps_uses_codes_and_specialty_key(): + con = duckdb.connect(":memory:") + _seed_joint(con) + rows = E.joint_annual( + con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,) + ) + con.close() + assert rows[0].values["psps:FAMILY PRACTICE:allowed"] == 125.0 + + +def test_joint_specialty_only_has_no_psps(): + con = duckdb.connect(":memory:") + _seed_joint(con) + rows = E.joint_annual(con, specialty="FAMILY PRACTICE", years=(2020,)) + con.close() + assert not any(key.startswith("psps:") for key in rows[0].values) + assert rows[0].sources["psps"] == "needs a family" + + +def test_joint_missing_inputs_are_none_not_zero(): + con = duckdb.connect(":memory:") + _seed_joint(con) + rows = E.joint_annual(con, family="CCM", years=(2021,)) + assert rows[0].values["fred:CPIMEDSL"] is None + assert rows[0].values["ces:621100:employment"] is None + assert "ces:621600:employment" not in rows[0].values + assert rows[0].values["util:CCM:n_services"] is None + both = E.joint_annual(con, family="CCM", specialty="FAMILY PRACTICE", years=(2021,)) + assert both[0].values["psps:FAMILY PRACTICE:allowed"] is None + con.close() + + bare = duckdb.connect(":memory:") + assert E.annual_tri(bare, "XLV") == {} + assert E.joint_monthly(bare, years=(2020,)) == [] + bare_rows = E.joint_annual( + bare, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,) + ) + assert bare_rows[0].values["fred:CPIMEDSL"] is None + assert bare_rows[0].values["util:HOME-VISITS:n_services"] is None + assert bare_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None + bare.close() + + partial = duckdb.connect(":memory:") + partial.execute("CREATE SCHEMA pfs") + partial.execute("CREATE TABLE pfs.code_family (key VARCHAR, code VARCHAR)") + partial.execute("INSERT INTO pfs.code_family VALUES ('HOME-VISITS', '99350')") + partial_rows = E.joint_annual( + partial, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,) + ) + assert partial_rows[0].values["util:HOME-VISITS:n_services"] is None + assert partial_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None + partial.close() + + priced = duckdb.connect(":memory:") + T.ensure_tables(priced) + T.write_entities(priced, [T.EntityRow("E1", "Home Co", "company")]) + T.write_securities( + priced, + "E1", + [T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")], + ) + T.write_links( + priced, + [ + T.LinkRow( + "E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x" + ) + ], + ) + priced.execute("DROP TABLE market.price_daily") + priced_rows = E.joint_annual(priced, family="HOME-VISITS", years=(2020,)) + assert priced_rows[0].values["market:family:HOME-VISITS:bhar"] is None + priced.close() + + +def test_joint_monthly_alignment(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + write_ces( + con, + [ + CesRow( + "CES6562110001", + "621100", + "Offices of physicians", + "employment", + "S", + 2024, + 3, + 7.0, + "ces-m", + ), + CesRow( + "CES6562110001", + "621100", + "Offices of physicians", + "employment", + "S", + 2024, + 13, + 99.0, + "ces-m", + ), + ], + ) + T.write_fred( + con, + [ + T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m"), + T.FredRow("CPIMEDSL", 2024, "nope", 9.0, "bad"), + T.FredRow("CPIMEDSL", 2019, "2019-03-01", 9.0, "old"), + T.FredRow("VIXCLS", 2024, "2024-13-01", 9.0, "badm"), + T.FredRow("USREC", 2024, "2024-xx", 9.0, "badm"), + ], + ) + T.write_series( + con, + "tiingo:XLV:tri", + [ + T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k"), + T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2019, 3, 1.0, "old"), + ], + ) + T.write_series( + con, + "tiingo:XLV:close", + [T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 3, 1.0, "c")], + ) + months = E.joint_monthly(con, years=(2024,)) + con.close() + march = next(row for row in months if row.year == 2024 and row.month == 3) + assert march.values["ces:621100:employment"] == 7.0 + assert march.values["fred:CPIMEDSL"] == 1.5 + assert march.values["market:XLV:tri"] == 120.0 + + +def test_annual_tri(): + con = duckdb.connect(":memory:") + T.ensure_tables(con) + T.write_series( + con, + "tiingo:XLV:tri", + [ + T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 1, 100.0, "k"), + T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 2, 110.0, "k"), + ], + ) + T.write_series( + con, + "tiingo:XLV:close", + [T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 1, 999.0, "k")], + ) + got = E.annual_tri(con, "XLV") + con.close() + assert got == {2024: 105.0} + + +def test_specialty_soc_keys_are_specialty_keys(): + from pfs.commenters import specialty_key + + assert E.FAMILY_INDUSTRY == {"HOME-VISITS": ("621100", "621600")} + assert E.SPECIALTY_SOC + for key, codes in E.SPECIALTY_SOC.items(): + assert key == specialty_key(key) + assert codes + for code in codes: + assert re.fullmatch(r"\d{2}-\d{4}", code) diff --git a/tests/market/test_footprint.py b/tests/market/test_footprint.py index 8df9e74..d5c2850 100644 --- a/tests/market/test_footprint.py +++ b/tests/market/test_footprint.py @@ -212,6 +212,75 @@ def test_district_view_joins_member_and_committees(): assert views[0].payment == 75 +def test_district_view_agi_from_soi(): + """ZIP 48201's 2023 AGI rolls onto the 2024 footprint by district weight. + + SOI has no 2024 file, so the latest year at or before the footprint year + is the one that fills ``agi_k``. + """ + con = duckdb.connect(":memory:") + T.ensure_tables(con) + T.write_footprints( + con, + [ + T.FootprintRow( + "family", + "HOME-VISITS", + 119, + "MI-12", + 2024, + 1, + 0, + 1, + 1, + "zip-weight", + "prov-svc", + ), + T.FootprintRow( + "family", + "HOME-VISITS", + 119, + "MI-14", + 2024, + 1, + 0, + 1, + 1, + "zip-weight", + "prov-svc", + ), + ], + ) + T.write_zip_districts( + con, + [ + T.ZipDistrictRow(119, "48201", "MI-12", 0.75, "hud-bus", "2026Q2", ""), + T.ZipDistrictRow(119, "48201", "MI-13", 0.25, "hud-bus", "2026Q2", ""), + T.ZipDistrictRow(119, "00000", "MI-14", 0.0, "hud-bus", "2026Q2", ""), + ], + ) + T.write_soi_zip( + con, + [ + T.SoiZipRow(2023, "48201", "26", 10, 1000, "soi-k"), + T.SoiZipRow(2023, "00000", "26", 1, 500, "soi-k"), + T.SoiZipRow(2025, "48201", "26", 1, 9999, "later"), + ], + ) + views = F.district_view( + con, congress=119, level="family", key="HOME-VISITS", year=2024 + ) + by_id = {view.district_id: view for view in views} + assert by_id["MI-12"].agi_k == 750.0 + assert by_id["MI-14"].agi_k is None + con.execute("DROP TABLE econ.soi_zip") + missing = F.district_view( + con, congress=119, level="family", key="HOME-VISITS", year=2024 + ) + con.close() + assert all(view.agi_k is None for view in missing) + + def test_enrichment_root_blank_npi_and_null_payment(tmp_path): assert F.enrichment_root().name == "lead_enrichment" assert ( diff --git a/tests/market/test_imports.py b/tests/market/test_imports.py index 3edf6a2..e163e75 100644 --- a/tests/market/test_imports.py +++ b/tests/market/test_imports.py @@ -20,6 +20,7 @@ MODULES = [ "market.events", "market.abnormal", "market.portfolio", + "market.enrich", ] _PROBE = """