feat(market): joint readers — market series beside BLS, FRED, SOI, PSPS and utilization
All checks were successful
CI / lint (push) Successful in 1m9s
CI / notebooks-smoke (push) Successful in 1m49s
Deploy / changes (push) Successful in 17s
Infra CI / notebooks (push) Successful in 59s
Infra CI / zotero (push) Successful in 17s
CI / test (push) Successful in 4m56s
Infra CI / docs (push) Successful in 27s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m53s
Infra CI / mc (push) Successful in 22s
Infra CI / llm (push) Successful in 1m3s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m46s
Deploy / bcda (push) Successful in 2m1s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m31s
Deploy / report (push) Successful in 22s

This commit is contained in:
kert
2026-11-17 12:23:14 -05:00
parent d008735c77
commit cd4806dff6
7 changed files with 1140 additions and 7 deletions

View File

@@ -11,6 +11,7 @@ uv run stack market events --write # releases, meetings, tenures
uv run stack market fred --write # FRED fredgraph series
uv run stack market soi --write # IRS SOI ZIP income
uv run stack market revenue --write # EDGAR annual revenue
uv run stack market joint --family HOME-VISITS # market beside BLS, FRED, PSPS
uv run stack market gaps # sources that could not be read
Every command fetches first, then opens one short DuckDB batch that only
@@ -32,6 +33,7 @@ import typer
from market import congress as congress_api
from market import districts as district_api
from market import edgar, http, tiingo
from market import enrich as enrich_api
from market import events as events_api
from market import fec as fec_api
from market import footprint as footprint_api
@@ -1972,8 +1974,8 @@ def district_view(
congress: int = typer.Option(footprint_api.CONGRESS, "--congress"),
top: int = typer.Option(20, "--top", help="How many districts to print."),
) -> None:
"""Stored footprint joined to that congress's House member and health
committees. ``agi_k`` is empty until the SOI join."""
"""Stored footprint joined to that congress's House member, health
committees, and SOI income rolled by ZIP weight."""
_exactly_one(bool(entity), bool(family), "pass exactly one of --entity or --family")
level = "family" if family else "entity"
key = family or entity
@@ -1990,10 +1992,11 @@ def district_view(
return
for view in shown:
seats = ",".join(view.health_committees) or "-"
agi = f"{view.agi_k:g}" if view.agi_k is not None else "-"
typer.echo(
f"{view.district_id}\t{view.member}\t{view.party}\t{seats}\t"
f"providers={view.n_providers:g}\tsites={view.n_sites:g}\t"
f"services={view.services:g}\tpayment={view.payment:g}"
f"services={view.services:g}\tpayment={view.payment:g}\tagi={agi}"
)
@@ -2541,6 +2544,56 @@ def revenue(
typer.echo(f"stored: {n} revenue rows")
def _echo_joint(label: str, values: dict[str, float | None]) -> None:
parts = [label]
for key, value in values.items():
if value is None:
continue
parts.append(f"{key}={value:g}")
typer.echo("\t".join(parts))
@app.command()
def joint(
family: str = typer.Option("", "--family", help="A code-family key."),
specialty: str = typer.Option("", "--specialty", help="A CMS specialty key."),
from_year: int = typer.Option(2015, "--from"),
to_year: int = typer.Option(2024, "--to"),
monthly: bool = typer.Option(False, "--monthly", help="One row per month."),
) -> None:
"""Market series beside utilization, PSPS, OEWS, CES and FRED.
Pass either flag or both. PSPS allowed charges need both, because they
sum the family's codes for one specialty.
"""
if not family and not specialty:
typer.echo("pass --family or --specialty")
raise typer.Exit(2)
years = tuple(range(from_year, to_year + 1))
con = _read()
try:
if monthly:
rows = enrich_api.joint_monthly(
con,
family=family or None,
specialty=specialty or None,
years=years,
)
for row in rows:
_echo_joint(f"{row.year}-{row.month:02d}", row.values)
return
annual = enrich_api.joint_annual(
con,
family=family or None,
specialty=specialty or None,
years=years,
)
finally:
con.close()
for row in annual:
_echo_joint(str(row.year), row.values)
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

465
src/market/enrich.py Normal file
View File

@@ -0,0 +1,465 @@
"""Readers that set market series beside BLS, FRED, PSPS and utilization.
No numpy or pandas at import time. A missing input is ``None``, never 0.
"""
from __future__ import annotations
from collections import defaultdict
from collections.abc import Sequence
from dataclasses import dataclass
from datetime import date
from typing import Any
from pfs.codetables import is_missing_table_error
SPECIALTY_SOC: dict[str, tuple[str, ...]] = {
"FAMILY PRACTICE": ("29-1215",),
"NEUROLOGY": ("29-1217",),
"INTERNAL MEDICINE": ("29-1216",),
"OBSTETRICS GYNECOLOGY": ("29-1218",),
"PEDIATRIC MEDICINE": ("29-1221",),
"PATHOLOGY": ("29-1222",),
"PSYCHIATRY": ("29-1223",),
"DIAGNOSTIC RADIOLOGY": ("29-1224",),
"ANESTHESIOLOGY": ("29-1211",),
"OPHTHALMOLOGY": ("29-1241",),
"CARDIOLOGY": ("29-1212",),
"ORTHOPEDIC SURGERY": ("29-1242",),
"DERMATOLOGY": ("29-1213",),
"GENERAL SURGERY": ("29-1249",),
"EMERGENCY MEDICINE": ("29-1214",),
"PHYSICAL THERAPIST": ("29-1123",),
"NURSE PRACTITIONER": ("29-1171",),
"PHYSICIAN ASSISTANT": ("29-1071",),
}
FAMILY_INDUSTRY: dict[str, tuple[str, ...]] = {
"HOME-VISITS": ("621100", "621600"),
}
_DEFAULT_INDUSTRY = ("621100",)
_TRI = ("XLV", "IHF", "SPY")
@dataclass(frozen=True)
class JointYear:
"""One calendar year of the joint display."""
year: int
values: dict[str, float | None]
sources: dict[str, str]
@dataclass(frozen=True)
class JointMonth:
"""One calendar month of the joint display."""
year: int
month: int
values: dict[str, float | None]
sources: dict[str, str]
def _call(fn, default):
"""``fn()``, or *default* when the table it reads is not there."""
try:
return fn()
except Exception as exc:
if is_missing_table_error(exc):
return default
raise
def annual_tri(con: Any, symbol: str) -> dict[int, float]:
"""Mean of the symbol's monthly total-return index, by calendar year.
A non-``tri`` series for the same symbol is ignored.
"""
from market.tables import read_series
rows = _call(lambda: read_series(con, [symbol]), None)
if rows is None:
return {}
buckets: dict[int, list[float]] = defaultdict(list)
for row in rows:
if row.series_id.endswith(":tri"):
buckets[row.year].append(row.value)
return {year: sum(vals) / len(vals) for year, vals in sorted(buckets.items())}
def joint_annual(
con: Any,
*,
family: str | None = None,
specialty: str | None = None,
years: Sequence[int],
) -> list[JointYear]:
"""One :class:`JointYear` per requested year.
``psps:<specialty>:allowed`` is the family's codes only, matched on
:func:`pfs.commenters.specialty_key`. Specialty without a family leaves
that column out and sets ``sources["psps"]`` to ``"needs a family"``.
"""
from market.tables import FRED_SERIES
wanted = tuple(int(year) for year in years)
industries = _industries(family) if family else ()
socs = SPECIALTY_SOC.get(specialty, ()) if specialty else ()
util, util_keys = _utilization(con, family) if family else ({}, {})
psps = _psps(con, family, specialty, wanted) if family and specialty else {}
oews = _oews(con, socs, wanted)
ces = _ces_annual(con, industries, wanted)
fred = _fred_annual(con)
out: list[JointYear] = []
for year in wanted:
values: dict[str, float | None] = {}
sources: dict[str, str] = {}
if family:
_put_market(con, values, sources, "family", family, year)
item = ",".join(sorted(util_keys.get(year, ())))
_put(
values,
sources,
f"util:{family}:n_services",
util.get(year),
f"pfs.utilization:{item}",
)
if specialty:
_put_market(con, values, sources, "specialty", specialty, year)
if family and specialty:
total, item = psps.get(year, (None, ""))
_put(
values,
sources,
f"psps:{specialty}:allowed",
total,
f"pfs.utilization_specialty:{item}",
)
elif specialty:
sources["psps"] = "needs a family"
for soc in socs:
median, item = oews.get((soc, year), (None, ""))
_put(
values,
sources,
f"oews:{soc}:a_median",
median,
f"bls.oews:{item}",
)
for industry in industries:
employment, item = ces.get((industry, year), (None, ""))
_put(
values,
sources,
f"ces:{industry}:employment",
employment,
f"bls.ces:{item}",
)
for series_id in FRED_SERIES:
value, item = fred.get((series_id, year), (None, ""))
_put(values, sources, f"fred:{series_id}", value, f"econ.fred:{item}")
out.append(JointYear(year, values, sources))
return out
def joint_monthly(
con: Any,
*,
family: str | None = None,
specialty: str | None = None,
years: Sequence[int],
) -> list[JointMonth]:
"""Months in *years* that have a CES, FRED or benchmark total-return row.
CES is the family's industries, or offices of physicians (``621100``)
when no family is named. *specialty* is accepted and unused: monthly
columns are employment, prices and the FRED mean, not a wage or a BHAR.
"""
del specialty
from market.tables import FRED_SERIES, read_fred, read_series
wanted = {int(year) for year in years}
industries = _industries(family) if family else _DEFAULT_INDUSTRY
ces_at = _ces_monthly(con, industries, wanted)
fred_at: dict[tuple[str, int, int], list] = defaultdict(list)
fred_rows = _call(lambda: read_fred(con), None) or []
for row in fred_rows:
parsed = _year_month(row.period)
if parsed is None or parsed[0] not in wanted:
continue
fred_at[(row.series_id, parsed[0], parsed[1])].append(row)
tri_at: dict[tuple[str, int, int], tuple[float, str]] = {}
series_rows = _call(lambda: read_series(con, list(_TRI)), None) or []
for row in series_rows:
if not row.series_id.endswith(":tri") or row.year not in wanted:
continue
tri_at[(row.symbol, row.year, row.month)] = (row.value, row.item_key)
stamps = {(year, month) for _industry, year, month in ces_at}
stamps |= {(year, month) for _series, year, month in fred_at}
stamps |= {(year, month) for _symbol, year, month in tri_at}
out: list[JointMonth] = []
for year, month in sorted(stamps):
values: dict[str, float | None] = {}
sources: dict[str, str] = {}
for industry in industries:
value, item = ces_at.get((industry, year, month), (None, ""))
_put(
values,
sources,
f"ces:{industry}:employment",
value,
f"bls.ces:{item}",
)
for series_id in FRED_SERIES:
group = [
row
for row in fred_at.get((series_id, year, month), ())
if row.value is not None
]
if group:
mean = sum(row.value for row in group) / len(group)
item = max(group, key=lambda row: row.period).item_key
else:
mean, item = None, ""
_put(values, sources, f"fred:{series_id}", mean, f"econ.fred:{item}")
for symbol in _TRI:
value, item = tri_at.get((symbol, year, month), (None, ""))
_put(
values, sources, f"market:{symbol}:tri", value, f"market.series:{item}"
)
out.append(JointMonth(year, month, values, sources))
return out
def _put(
values: dict[str, float | None],
sources: dict[str, str],
key: str,
value: float | None,
source: str,
) -> None:
values[key] = value
sources[key] = source
def _industries(family: str) -> tuple[str, ...]:
return FAMILY_INDUSTRY.get(family, _DEFAULT_INDUSTRY)
def _put_market(
con: Any,
values: dict[str, float | None],
sources: dict[str, str],
scope: str,
target: str,
year: int,
) -> None:
value, item = _market_bhar(con, scope, target, year)
_put(
values, sources, f"market:{scope}:{target}:bhar", value, f"market.series:{item}"
)
def _market_bhar(
con: Any, scope: str, target: str, year: int
) -> tuple[float | None, str]:
"""Equal-weight portfolio BHAR against SPY. The cite is SPY's tri series."""
from market.portfolio import members, portfolio_returns, year_bhar
from market.tables import read_series
from market.tiingo import read_returns
item = ""
spy = _call(lambda: read_series(con, ["SPY"]), None) or []
for row in spy:
if row.series_id.endswith(":tri") and row.item_key:
item = row.item_key
break
found = _call(
lambda: members(con, scope=scope, target=target, on=date(year, 12, 31)),
None,
)
if found is None:
return None, item
chosen, _gaps = found
if not chosen:
return None, item
returns = _call(
lambda: read_returns(
con,
[member.ticker for member in chosen] + ["SPY"],
date(year, 1, 1),
date(year, 12, 31),
),
None,
)
if returns is None:
return None, item
port = portfolio_returns(returns, chosen)
return year_bhar(port, returns.get("SPY", {}), year), item
def _family_codes(con: Any, family: str) -> list[str] | None:
"""Member codes, or ``None`` when ``pfs.code_family`` is not on *con*."""
found = _call(
lambda: con.execute(
"SELECT DISTINCT code FROM pfs.code_family WHERE key = ?",
[family],
).fetchall(),
None,
)
if found is None:
return None
return [row[0] for row in found]
def _utilization(con: Any, family: str) -> tuple[dict[int, float], dict[int, set[str]]]:
from pfs.event_study import family_series
codes = _family_codes(con, family)
if not codes:
return {}, {}
totals = _call(lambda: family_series(con, family), None)
if totals is None:
return {}, {}
keys: dict[int, set[str]] = defaultdict(set)
marks = ",".join("?" * len(codes))
found = con.execute(
"SELECT year, item_key FROM pfs.utilization "
f"WHERE hcpcs IN ({marks}) AND item_key <> ''",
[code.upper() for code in codes],
).fetchall()
for year, item in found:
keys[int(year)].add(item)
return totals, keys
def _psps(
con: Any, family: str, specialty: str, years: Sequence[int]
) -> dict[int, tuple[float | None, str]]:
from pfs.commenters import specialty_key
from pfs.psps import read_by_specialty
codes = _family_codes(con, family)
if not codes:
return {}
rows = _call(lambda: read_by_specialty(con, codes, years=years), None)
if rows is None:
return {}
wanted = specialty_key(specialty)
totals: dict[int, float] = {}
keys: dict[int, set[str]] = defaultdict(set)
seen: set[int] = set()
for row in rows:
if specialty_key(row.specialty) != wanted or row.allowed is None:
continue
seen.add(row.year)
totals[row.year] = totals.get(row.year, 0.0) + float(row.allowed)
if row.item_key:
keys[row.year].add(row.item_key)
return {
year: (totals.get(year), ",".join(sorted(keys.get(year, ())))) for year in seen
}
def _oews(
con: Any, socs: tuple[str, ...], years: Sequence[int]
) -> dict[tuple[str, int], tuple[float | None, str]]:
from bls.oews import read_occupations
if not socs:
return {}
rows = _call(lambda: read_occupations(con, socs, years=years), None)
if rows is None:
return {}
out: dict[tuple[str, int], tuple[float | None, str]] = {}
for row in rows:
out[(row.occ_code, row.year)] = (row.a_median, row.item_key)
return out
def _employment_ids(industries: Sequence[str]) -> list[str]:
from bls.ces import SERIES
ids: list[str] = []
for industry in industries:
for series in SERIES.values():
if series.industry_code == industry and series.data_type == "employment":
ids.append(series.series_id)
return ids
def _ces_annual(
con: Any, industries: Sequence[str], years: Sequence[int]
) -> dict[tuple[str, int], tuple[float | None, str]]:
from bls.ces import SERIES, read_series
ids = _employment_ids(industries)
if not ids:
return {}
rows = _call(lambda: read_series(con, ids, annual=True), None)
if rows is None:
return {}
wanted = set(years)
out: dict[tuple[str, int], tuple[float | None, str]] = {}
for row in rows:
if row.year not in wanted:
continue
meta = SERIES[row.series_id]
out[(meta.industry_code, row.year)] = (row.value, row.item_key)
return out
def _ces_monthly(
con: Any, industries: Sequence[str], years: set[int]
) -> dict[tuple[str, int, int], tuple[float | None, str]]:
from bls.ces import SERIES, read_series
ids = _employment_ids(industries)
rows = _call(lambda: read_series(con, ids, annual=False), None) if ids else []
if rows is None:
return {}
out: dict[tuple[str, int, int], tuple[float | None, str]] = {}
for row in rows:
if row.year not in years or not 1 <= row.month <= 12:
continue
meta = SERIES[row.series_id]
out[(meta.industry_code, row.year, row.month)] = (row.value, row.item_key)
return out
def _fred_annual(con: Any) -> dict[tuple[str, int], tuple[float | None, str]]:
from market.tables import read_fred
rows = _call(lambda: read_fred(con), None)
if rows is None:
return {}
grouped: dict[tuple[str, int], list] = defaultdict(list)
for row in rows:
year = row.year
if year is None and len(row.period) >= 4 and row.period[:4].isdigit():
year = int(row.period[:4])
if year is None:
continue
grouped[(row.series_id, int(year))].append(row)
out: dict[tuple[str, int], tuple[float | None, str]] = {}
for key, group in grouped.items():
usable = [row for row in group if row.value is not None]
if not usable:
out[key] = (None, "")
continue
latest = max(usable, key=lambda row: row.period)
out[key] = (sum(row.value for row in usable) / len(usable), latest.item_key)
return out
def _year_month(period: str) -> tuple[int, int] | None:
parts = period.split("-")
if len(parts) < 2 or not (len(parts[0]) == 4 and parts[0].isdigit()):
return None
month_text = parts[1][:2]
if not month_text.isdigit():
return None
month = int(month_text)
if not 1 <= month <= 12:
return None
return int(parts[0]), month

View File

@@ -4,7 +4,8 @@ Sites come from the CMS All Owners snapshot of 2026-09-29 joined to
enrollment ZIPs, and from NPPES organization rows. Family billers come from
the 2024 Medicare physician-and-other-practitioners file. Amounts split
across districts by :func:`market.districts.zip_weights`. An unmatched ZIP
is kept as ``??``. ``agi_k`` stays empty until the SOI join.
is kept as ``??``. ``agi_k`` is IRS SOI income rolled by ``zip_weights``,
using the latest SOI year at or before the footprint year.
"""
from __future__ import annotations
@@ -24,6 +25,7 @@ from market.tables import (
read_assignments,
read_footprints,
read_members,
read_soi_zip,
)
from pfs.codetables import is_missing_table_error
@@ -73,8 +75,8 @@ class AllocItem:
@dataclass(frozen=True)
class DistrictView:
"""One district's footprint beside its House member. ``agi_k`` is filled
by the SOI join."""
"""One district's footprint beside its House member. ``agi_k`` is ZIP
income rolled onto the district (thousands of dollars)."""
district_id: str
bioguide_id: str
@@ -334,6 +336,7 @@ def district_view(
if row.congress != congress or row.committee_id not in HEALTH_COMMITTEES:
continue
committees.setdefault(row.bioguide_id, set()).add(row.committee_id)
agi = _agi_by_district(con, congress, year)
views: list[DistrictView] = []
for district, (providers, sites, services, payment) in totals.items():
member = members.get(district)
@@ -349,13 +352,46 @@ def district_view(
sites,
services,
payment,
None,
agi.get(district),
)
)
views.sort(key=lambda v: (-v.payment, v.district_id))
return views
def _agi_by_district(con: Any, congress: int, year: int) -> dict[str, float]:
"""SOI AGI (thousands) times ZIP weight. The latest SOI year at or
before *year* is used: the footprint year is 2024 and SOI stops at 2023.
A non-positive weight adds nothing, so a zero-weight crosswalk stays
``None`` rather than 0.
"""
try:
chosen = con.execute(
"SELECT max(year) FROM econ.soi_zip WHERE year <= ? AND agi_k IS NOT NULL",
[year],
).fetchone()[0]
except Exception as exc:
if is_missing_table_error(exc):
return {}
raise
if chosen is None:
return {}
agi = {
row.zip: row.agi_k
for row in read_soi_zip(con, year=int(chosen))
if row.agi_k is not None
}
weights = zip_weights(con, congress)
out: dict[str, float] = {}
for zipcode, amount in agi.items():
for district, weight in weights.get(zipcode, ()):
if weight <= 0:
continue
out[district] = out.get(district, 0.0) + float(amount) * float(weight)
return out
def _missing(exc: Exception) -> bool:
return is_missing_table_error(exc)

View File

@@ -3563,3 +3563,81 @@ def test_revenue_cli_refusal_prints_gap(env, monkeypatch):
assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output
assert T.read_revenue(c, "0000001234") == []
assert published == [True]
def test_joint_cli_prints(env):
c, _, published, _ = env
T.write_fred(c, [T.FredRow("CPIMEDSL", 2020, "2020-06-01", 9.0, "fk")])
res = runner.invoke(
app,
[
"market",
"joint",
"--family",
"HOME-VISITS",
"--from",
"2020",
"--to",
"2020",
],
)
assert res.exit_code == 0, res.output
assert "2020" in res.output
assert "fred:CPIMEDSL=9" in res.output
assert published == []
def test_joint_cli_monthly(env):
from bls.ces import CesRow
from bls.ces import write as write_ces
c, _, published, _ = env
write_ces(
c,
[
CesRow(
"CES6562110001",
"621100",
"Offices of physicians",
"employment",
"S",
2024,
3,
7.0,
"ces-m",
)
],
)
T.write_fred(c, [T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m")])
T.write_series(
c,
"tiingo:XLV:tri",
[T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k")],
)
res = runner.invoke(
app,
[
"market",
"joint",
"--family",
"HOME-VISITS",
"--from",
"2024",
"--to",
"2024",
"--monthly",
],
)
assert res.exit_code == 0, res.output
assert "2024-03" in res.output
assert "ces:621100:employment=7" in res.output
assert "fred:CPIMEDSL=1.5" in res.output
assert "market:XLV:tri=120" in res.output
assert published == []
def test_joint_cli_requires_family_or_specialty(env):
res = runner.invoke(app, ["market", "joint"])
assert res.exit_code == 2
assert "family" in res.output
assert "specialty" in res.output

431
tests/market/test_enrich.py Normal file
View File

@@ -0,0 +1,431 @@
"""Joint annual and monthly readers beside BLS, FRED, PSPS and utilization."""
from __future__ import annotations
import re
from datetime import date, timedelta
import duckdb
from bls.ces import CesRow
from bls.ces import write as write_ces
from bls.oews import OewsRow
from bls.oews import write as write_oews
from market import enrich as E
from market import tables as T
from pfs.codetables import PspsRow, ensure_tables, write_psps
from pfs.utilization import UtilizationRow
from pfs.utilization import write as write_util
def _days(n: int, start: date = date(2020, 1, 2)) -> list[date]:
out: list[date] = []
day = start
while len(out) < n:
if day.weekday() < 5:
out.append(day)
day += timedelta(days=1)
return out
def _oews(year: int, code: str, median: float, item_key: str) -> OewsRow:
return OewsRow(
year,
code,
"Family Medicine Physicians",
"detailed",
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
None,
median,
None,
None,
"OEUN",
item_key,
)
def _seed_joint(con) -> None:
"""2020 inputs whose column values are fixed numbers."""
T.ensure_tables(con)
ensure_tables(con)
con.execute(
"CREATE TABLE IF NOT EXISTS pfs.code_family ("
"key VARCHAR, name VARCHAR, code VARCHAR, role VARCHAR, since INTEGER, "
"until INTEGER, item_key VARCHAR, p_id INTEGER, note VARCHAR)"
)
con.execute(
"INSERT INTO pfs.code_family VALUES "
"('HOME-VISITS', 'Home', '99349', 'member', 2015, NULL, 'fam', 0, ''), "
"('HOME-VISITS', 'Home', '99350', 'member', 2015, NULL, 'fam', 0, '')"
)
write_util(
con,
2020,
[
UtilizationRow(
2020,
"99349",
"",
"N",
"O",
1,
1,
30.0,
None,
None,
None,
None,
None,
"d",
"util-k",
),
UtilizationRow(
2020,
"99350",
"",
"N",
"O",
1,
1,
50.0,
None,
None,
None,
None,
None,
"d",
"util-k",
),
UtilizationRow(
2020,
"11111",
"",
"N",
"O",
1,
1,
5.0,
None,
None,
None,
None,
None,
"d",
"util-other",
),
],
)
write_psps(
con,
2020,
"99349",
[PspsRow(2020, "99349", "08", "Family Practice", 1, 25.0, 1, "ds", "psps-k")],
)
write_psps(
con,
2020,
"99350",
[
PspsRow(
2020, "99350", "08", "Family Practice", 1, 100.0, 1, "ds", "psps-k"
),
PspsRow(
2020, "99350", "08", "Family Practice", 1, None, 1, "ds", "psps-null"
),
PspsRow(2020, "99350", "09", "Family Practice", 1, 0.0, 1, "ds", ""),
PspsRow(
2020,
"99350",
"16",
"Obstetrics/Gynecology",
1,
999.0,
1,
"ds",
"psps-ob",
),
],
)
write_psps(
con,
2020,
"11111",
[PspsRow(2020, "11111", "08", "Family Practice", 1, 50.0, 1, "ds", "psps-k")],
)
write_oews(con, 2020, [_oews(2020, "29-1215", 211000.0, "oews-k")])
write_ces(
con,
[
CesRow(
"CES6562110001",
"621100",
"Offices of physicians",
"employment",
"S",
2020,
1,
10.0,
"ces-k",
),
CesRow(
"CES6562110001",
"621100",
"Offices of physicians",
"employment",
"S",
2020,
2,
30.0,
"ces-k",
),
],
)
T.write_fred(
con,
[
T.FredRow("CPIMEDSL", 2020, "2020-01-01", 2.0, "fred-k"),
T.FredRow("CPIMEDSL", 2020, "2020-02-01", 4.0, "fred-k"),
T.FredRow("CPIMEDSL", 2020, "2020-03-01", None, "fred-k"),
T.FredRow("CPIAUCSL", 2020, "2020-01-01", None, "none-k"),
T.FredRow("DTB3", None, "2020-05-01", 1.0, "parsed-k"),
T.FredRow("VIXCLS", None, "nope", 1.0, "bad-k"),
],
)
T.write_entities(con, [T.EntityRow("E1", "Home Co", "company")])
T.write_securities(
con, "E1", [T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")]
)
T.write_links(
con,
[
T.LinkRow(
"E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x"
),
T.LinkRow(
"E1",
"specialty",
"FAMILY PRACTICE",
"curated",
"curated",
"item_key",
"x",
),
],
)
days = _days(200)
con.executemany(
"INSERT INTO market.price_daily (ticker, day, ret) VALUES (?, ?, 0)",
[(ticker, day) for ticker in ("HV", "SPY") for day in days],
)
T.write_series(
con,
"tiingo:SPY:tri",
[T.MonthlyRow("tiingo:SPY:tri", "SPY", "tiingo", 2020, 12, 100.0, "spy-k")],
)
def test_joint_annual_columns_and_sources():
con = duckdb.connect(":memory:")
_seed_joint(con)
rows = E.joint_annual(
con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
)
con.close()
assert [row.year for row in rows] == [2020]
values, sources = rows[0].values, rows[0].sources
assert values["market:family:HOME-VISITS:bhar"] == 0.0
assert values["market:specialty:FAMILY PRACTICE:bhar"] == 0.0
assert values["util:HOME-VISITS:n_services"] == 80.0
assert values["psps:FAMILY PRACTICE:allowed"] == 125.0
assert values["oews:29-1215:a_median"] == 211000.0
assert values["ces:621100:employment"] == 20.0
assert values["ces:621600:employment"] is None
assert values["fred:CPIMEDSL"] == 3.0
assert {key for key in values if key.startswith("fred:")} == {
f"fred:{series}" for series in T.FRED_SERIES
}
assert sources["market:family:HOME-VISITS:bhar"] == "market.series:spy-k"
assert sources["util:HOME-VISITS:n_services"] == "pfs.utilization:util-k"
assert sources["psps:FAMILY PRACTICE:allowed"] == "pfs.utilization_specialty:psps-k"
assert sources["oews:29-1215:a_median"] == "bls.oews:oews-k"
assert sources["ces:621100:employment"] == "bls.ces:ces-k"
assert sources["fred:CPIMEDSL"] == "econ.fred:fred-k"
def test_joint_psps_uses_codes_and_specialty_key():
con = duckdb.connect(":memory:")
_seed_joint(con)
rows = E.joint_annual(
con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
)
con.close()
assert rows[0].values["psps:FAMILY PRACTICE:allowed"] == 125.0
def test_joint_specialty_only_has_no_psps():
con = duckdb.connect(":memory:")
_seed_joint(con)
rows = E.joint_annual(con, specialty="FAMILY PRACTICE", years=(2020,))
con.close()
assert not any(key.startswith("psps:") for key in rows[0].values)
assert rows[0].sources["psps"] == "needs a family"
def test_joint_missing_inputs_are_none_not_zero():
con = duckdb.connect(":memory:")
_seed_joint(con)
rows = E.joint_annual(con, family="CCM", years=(2021,))
assert rows[0].values["fred:CPIMEDSL"] is None
assert rows[0].values["ces:621100:employment"] is None
assert "ces:621600:employment" not in rows[0].values
assert rows[0].values["util:CCM:n_services"] is None
both = E.joint_annual(con, family="CCM", specialty="FAMILY PRACTICE", years=(2021,))
assert both[0].values["psps:FAMILY PRACTICE:allowed"] is None
con.close()
bare = duckdb.connect(":memory:")
assert E.annual_tri(bare, "XLV") == {}
assert E.joint_monthly(bare, years=(2020,)) == []
bare_rows = E.joint_annual(
bare, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
)
assert bare_rows[0].values["fred:CPIMEDSL"] is None
assert bare_rows[0].values["util:HOME-VISITS:n_services"] is None
assert bare_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None
bare.close()
partial = duckdb.connect(":memory:")
partial.execute("CREATE SCHEMA pfs")
partial.execute("CREATE TABLE pfs.code_family (key VARCHAR, code VARCHAR)")
partial.execute("INSERT INTO pfs.code_family VALUES ('HOME-VISITS', '99350')")
partial_rows = E.joint_annual(
partial, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
)
assert partial_rows[0].values["util:HOME-VISITS:n_services"] is None
assert partial_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None
partial.close()
priced = duckdb.connect(":memory:")
T.ensure_tables(priced)
T.write_entities(priced, [T.EntityRow("E1", "Home Co", "company")])
T.write_securities(
priced,
"E1",
[T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")],
)
T.write_links(
priced,
[
T.LinkRow(
"E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x"
)
],
)
priced.execute("DROP TABLE market.price_daily")
priced_rows = E.joint_annual(priced, family="HOME-VISITS", years=(2020,))
assert priced_rows[0].values["market:family:HOME-VISITS:bhar"] is None
priced.close()
def test_joint_monthly_alignment():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
write_ces(
con,
[
CesRow(
"CES6562110001",
"621100",
"Offices of physicians",
"employment",
"S",
2024,
3,
7.0,
"ces-m",
),
CesRow(
"CES6562110001",
"621100",
"Offices of physicians",
"employment",
"S",
2024,
13,
99.0,
"ces-m",
),
],
)
T.write_fred(
con,
[
T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m"),
T.FredRow("CPIMEDSL", 2024, "nope", 9.0, "bad"),
T.FredRow("CPIMEDSL", 2019, "2019-03-01", 9.0, "old"),
T.FredRow("VIXCLS", 2024, "2024-13-01", 9.0, "badm"),
T.FredRow("USREC", 2024, "2024-xx", 9.0, "badm"),
],
)
T.write_series(
con,
"tiingo:XLV:tri",
[
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k"),
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2019, 3, 1.0, "old"),
],
)
T.write_series(
con,
"tiingo:XLV:close",
[T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 3, 1.0, "c")],
)
months = E.joint_monthly(con, years=(2024,))
con.close()
march = next(row for row in months if row.year == 2024 and row.month == 3)
assert march.values["ces:621100:employment"] == 7.0
assert march.values["fred:CPIMEDSL"] == 1.5
assert march.values["market:XLV:tri"] == 120.0
def test_annual_tri():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
T.write_series(
con,
"tiingo:XLV:tri",
[
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 1, 100.0, "k"),
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 2, 110.0, "k"),
],
)
T.write_series(
con,
"tiingo:XLV:close",
[T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 1, 999.0, "k")],
)
got = E.annual_tri(con, "XLV")
con.close()
assert got == {2024: 105.0}
def test_specialty_soc_keys_are_specialty_keys():
from pfs.commenters import specialty_key
assert E.FAMILY_INDUSTRY == {"HOME-VISITS": ("621100", "621600")}
assert E.SPECIALTY_SOC
for key, codes in E.SPECIALTY_SOC.items():
assert key == specialty_key(key)
assert codes
for code in codes:
assert re.fullmatch(r"\d{2}-\d{4}", code)

View File

@@ -212,6 +212,75 @@ def test_district_view_joins_member_and_committees():
assert views[0].payment == 75
def test_district_view_agi_from_soi():
"""ZIP 48201's 2023 AGI rolls onto the 2024 footprint by district weight.
SOI has no 2024 file, so the latest year at or before the footprint year
is the one that fills ``agi_k``.
"""
con = duckdb.connect(":memory:")
T.ensure_tables(con)
T.write_footprints(
con,
[
T.FootprintRow(
"family",
"HOME-VISITS",
119,
"MI-12",
2024,
1,
0,
1,
1,
"zip-weight",
"prov-svc",
),
T.FootprintRow(
"family",
"HOME-VISITS",
119,
"MI-14",
2024,
1,
0,
1,
1,
"zip-weight",
"prov-svc",
),
],
)
T.write_zip_districts(
con,
[
T.ZipDistrictRow(119, "48201", "MI-12", 0.75, "hud-bus", "2026Q2", ""),
T.ZipDistrictRow(119, "48201", "MI-13", 0.25, "hud-bus", "2026Q2", ""),
T.ZipDistrictRow(119, "00000", "MI-14", 0.0, "hud-bus", "2026Q2", ""),
],
)
T.write_soi_zip(
con,
[
T.SoiZipRow(2023, "48201", "26", 10, 1000, "soi-k"),
T.SoiZipRow(2023, "00000", "26", 1, 500, "soi-k"),
T.SoiZipRow(2025, "48201", "26", 1, 9999, "later"),
],
)
views = F.district_view(
con, congress=119, level="family", key="HOME-VISITS", year=2024
)
by_id = {view.district_id: view for view in views}
assert by_id["MI-12"].agi_k == 750.0
assert by_id["MI-14"].agi_k is None
con.execute("DROP TABLE econ.soi_zip")
missing = F.district_view(
con, congress=119, level="family", key="HOME-VISITS", year=2024
)
con.close()
assert all(view.agi_k is None for view in missing)
def test_enrichment_root_blank_npi_and_null_payment(tmp_path):
assert F.enrichment_root().name == "lead_enrichment"
assert (

View File

@@ -20,6 +20,7 @@ MODULES = [
"market.events",
"market.abnormal",
"market.portfolio",
"market.enrich",
]
_PROBE = """