feat(market): joint readers — market series beside BLS, FRED, SOI, PSPS and utilization
All checks were successful
CI / lint (push) Successful in 1m9s
CI / notebooks-smoke (push) Successful in 1m49s
Deploy / changes (push) Successful in 17s
Infra CI / notebooks (push) Successful in 59s
Infra CI / zotero (push) Successful in 17s
CI / test (push) Successful in 4m56s
Infra CI / docs (push) Successful in 27s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m53s
Infra CI / mc (push) Successful in 22s
Infra CI / llm (push) Successful in 1m3s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m46s
Deploy / bcda (push) Successful in 2m1s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m31s
Deploy / report (push) Successful in 22s
All checks were successful
CI / lint (push) Successful in 1m9s
CI / notebooks-smoke (push) Successful in 1m49s
Deploy / changes (push) Successful in 17s
Infra CI / notebooks (push) Successful in 59s
Infra CI / zotero (push) Successful in 17s
CI / test (push) Successful in 4m56s
Infra CI / docs (push) Successful in 27s
Infra CI / api (push) Successful in 1m21s
Infra CI / bcda (push) Successful in 1m53s
Infra CI / mc (push) Successful in 22s
Infra CI / llm (push) Successful in 1m3s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Successful in 1m46s
Deploy / bcda (push) Successful in 2m1s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m31s
Deploy / report (push) Successful in 22s
This commit is contained in:
@@ -11,6 +11,7 @@ uv run stack market events --write # releases, meetings, tenures
|
||||
uv run stack market fred --write # FRED fredgraph series
|
||||
uv run stack market soi --write # IRS SOI ZIP income
|
||||
uv run stack market revenue --write # EDGAR annual revenue
|
||||
uv run stack market joint --family HOME-VISITS # market beside BLS, FRED, PSPS
|
||||
uv run stack market gaps # sources that could not be read
|
||||
|
||||
Every command fetches first, then opens one short DuckDB batch that only
|
||||
@@ -32,6 +33,7 @@ import typer
|
||||
from market import congress as congress_api
|
||||
from market import districts as district_api
|
||||
from market import edgar, http, tiingo
|
||||
from market import enrich as enrich_api
|
||||
from market import events as events_api
|
||||
from market import fec as fec_api
|
||||
from market import footprint as footprint_api
|
||||
@@ -1972,8 +1974,8 @@ def district_view(
|
||||
congress: int = typer.Option(footprint_api.CONGRESS, "--congress"),
|
||||
top: int = typer.Option(20, "--top", help="How many districts to print."),
|
||||
) -> None:
|
||||
"""Stored footprint joined to that congress's House member and health
|
||||
committees. ``agi_k`` is empty until the SOI join."""
|
||||
"""Stored footprint joined to that congress's House member, health
|
||||
committees, and SOI income rolled by ZIP weight."""
|
||||
_exactly_one(bool(entity), bool(family), "pass exactly one of --entity or --family")
|
||||
level = "family" if family else "entity"
|
||||
key = family or entity
|
||||
@@ -1990,10 +1992,11 @@ def district_view(
|
||||
return
|
||||
for view in shown:
|
||||
seats = ",".join(view.health_committees) or "-"
|
||||
agi = f"{view.agi_k:g}" if view.agi_k is not None else "-"
|
||||
typer.echo(
|
||||
f"{view.district_id}\t{view.member}\t{view.party}\t{seats}\t"
|
||||
f"providers={view.n_providers:g}\tsites={view.n_sites:g}\t"
|
||||
f"services={view.services:g}\tpayment={view.payment:g}"
|
||||
f"services={view.services:g}\tpayment={view.payment:g}\tagi={agi}"
|
||||
)
|
||||
|
||||
|
||||
@@ -2541,6 +2544,56 @@ def revenue(
|
||||
typer.echo(f"stored: {n} revenue rows")
|
||||
|
||||
|
||||
def _echo_joint(label: str, values: dict[str, float | None]) -> None:
|
||||
parts = [label]
|
||||
for key, value in values.items():
|
||||
if value is None:
|
||||
continue
|
||||
parts.append(f"{key}={value:g}")
|
||||
typer.echo("\t".join(parts))
|
||||
|
||||
|
||||
@app.command()
|
||||
def joint(
|
||||
family: str = typer.Option("", "--family", help="A code-family key."),
|
||||
specialty: str = typer.Option("", "--specialty", help="A CMS specialty key."),
|
||||
from_year: int = typer.Option(2015, "--from"),
|
||||
to_year: int = typer.Option(2024, "--to"),
|
||||
monthly: bool = typer.Option(False, "--monthly", help="One row per month."),
|
||||
) -> None:
|
||||
"""Market series beside utilization, PSPS, OEWS, CES and FRED.
|
||||
|
||||
Pass either flag or both. PSPS allowed charges need both, because they
|
||||
sum the family's codes for one specialty.
|
||||
"""
|
||||
if not family and not specialty:
|
||||
typer.echo("pass --family or --specialty")
|
||||
raise typer.Exit(2)
|
||||
years = tuple(range(from_year, to_year + 1))
|
||||
con = _read()
|
||||
try:
|
||||
if monthly:
|
||||
rows = enrich_api.joint_monthly(
|
||||
con,
|
||||
family=family or None,
|
||||
specialty=specialty or None,
|
||||
years=years,
|
||||
)
|
||||
for row in rows:
|
||||
_echo_joint(f"{row.year}-{row.month:02d}", row.values)
|
||||
return
|
||||
annual = enrich_api.joint_annual(
|
||||
con,
|
||||
family=family or None,
|
||||
specialty=specialty or None,
|
||||
years=years,
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
for row in annual:
|
||||
_echo_joint(str(row.year), row.values)
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
465
src/market/enrich.py
Normal file
465
src/market/enrich.py
Normal file
@@ -0,0 +1,465 @@
|
||||
"""Readers that set market series beside BLS, FRED, PSPS and utilization.
|
||||
|
||||
No numpy or pandas at import time. A missing input is ``None``, never 0.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from typing import Any
|
||||
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
SPECIALTY_SOC: dict[str, tuple[str, ...]] = {
|
||||
"FAMILY PRACTICE": ("29-1215",),
|
||||
"NEUROLOGY": ("29-1217",),
|
||||
"INTERNAL MEDICINE": ("29-1216",),
|
||||
"OBSTETRICS GYNECOLOGY": ("29-1218",),
|
||||
"PEDIATRIC MEDICINE": ("29-1221",),
|
||||
"PATHOLOGY": ("29-1222",),
|
||||
"PSYCHIATRY": ("29-1223",),
|
||||
"DIAGNOSTIC RADIOLOGY": ("29-1224",),
|
||||
"ANESTHESIOLOGY": ("29-1211",),
|
||||
"OPHTHALMOLOGY": ("29-1241",),
|
||||
"CARDIOLOGY": ("29-1212",),
|
||||
"ORTHOPEDIC SURGERY": ("29-1242",),
|
||||
"DERMATOLOGY": ("29-1213",),
|
||||
"GENERAL SURGERY": ("29-1249",),
|
||||
"EMERGENCY MEDICINE": ("29-1214",),
|
||||
"PHYSICAL THERAPIST": ("29-1123",),
|
||||
"NURSE PRACTITIONER": ("29-1171",),
|
||||
"PHYSICIAN ASSISTANT": ("29-1071",),
|
||||
}
|
||||
|
||||
FAMILY_INDUSTRY: dict[str, tuple[str, ...]] = {
|
||||
"HOME-VISITS": ("621100", "621600"),
|
||||
}
|
||||
_DEFAULT_INDUSTRY = ("621100",)
|
||||
_TRI = ("XLV", "IHF", "SPY")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JointYear:
|
||||
"""One calendar year of the joint display."""
|
||||
|
||||
year: int
|
||||
values: dict[str, float | None]
|
||||
sources: dict[str, str]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JointMonth:
|
||||
"""One calendar month of the joint display."""
|
||||
|
||||
year: int
|
||||
month: int
|
||||
values: dict[str, float | None]
|
||||
sources: dict[str, str]
|
||||
|
||||
|
||||
def _call(fn, default):
|
||||
"""``fn()``, or *default* when the table it reads is not there."""
|
||||
try:
|
||||
return fn()
|
||||
except Exception as exc:
|
||||
if is_missing_table_error(exc):
|
||||
return default
|
||||
raise
|
||||
|
||||
|
||||
def annual_tri(con: Any, symbol: str) -> dict[int, float]:
|
||||
"""Mean of the symbol's monthly total-return index, by calendar year.
|
||||
|
||||
A non-``tri`` series for the same symbol is ignored.
|
||||
"""
|
||||
from market.tables import read_series
|
||||
|
||||
rows = _call(lambda: read_series(con, [symbol]), None)
|
||||
if rows is None:
|
||||
return {}
|
||||
buckets: dict[int, list[float]] = defaultdict(list)
|
||||
for row in rows:
|
||||
if row.series_id.endswith(":tri"):
|
||||
buckets[row.year].append(row.value)
|
||||
return {year: sum(vals) / len(vals) for year, vals in sorted(buckets.items())}
|
||||
|
||||
|
||||
def joint_annual(
|
||||
con: Any,
|
||||
*,
|
||||
family: str | None = None,
|
||||
specialty: str | None = None,
|
||||
years: Sequence[int],
|
||||
) -> list[JointYear]:
|
||||
"""One :class:`JointYear` per requested year.
|
||||
|
||||
``psps:<specialty>:allowed`` is the family's codes only, matched on
|
||||
:func:`pfs.commenters.specialty_key`. Specialty without a family leaves
|
||||
that column out and sets ``sources["psps"]`` to ``"needs a family"``.
|
||||
"""
|
||||
from market.tables import FRED_SERIES
|
||||
|
||||
wanted = tuple(int(year) for year in years)
|
||||
industries = _industries(family) if family else ()
|
||||
socs = SPECIALTY_SOC.get(specialty, ()) if specialty else ()
|
||||
util, util_keys = _utilization(con, family) if family else ({}, {})
|
||||
psps = _psps(con, family, specialty, wanted) if family and specialty else {}
|
||||
oews = _oews(con, socs, wanted)
|
||||
ces = _ces_annual(con, industries, wanted)
|
||||
fred = _fred_annual(con)
|
||||
out: list[JointYear] = []
|
||||
for year in wanted:
|
||||
values: dict[str, float | None] = {}
|
||||
sources: dict[str, str] = {}
|
||||
if family:
|
||||
_put_market(con, values, sources, "family", family, year)
|
||||
item = ",".join(sorted(util_keys.get(year, ())))
|
||||
_put(
|
||||
values,
|
||||
sources,
|
||||
f"util:{family}:n_services",
|
||||
util.get(year),
|
||||
f"pfs.utilization:{item}",
|
||||
)
|
||||
if specialty:
|
||||
_put_market(con, values, sources, "specialty", specialty, year)
|
||||
if family and specialty:
|
||||
total, item = psps.get(year, (None, ""))
|
||||
_put(
|
||||
values,
|
||||
sources,
|
||||
f"psps:{specialty}:allowed",
|
||||
total,
|
||||
f"pfs.utilization_specialty:{item}",
|
||||
)
|
||||
elif specialty:
|
||||
sources["psps"] = "needs a family"
|
||||
for soc in socs:
|
||||
median, item = oews.get((soc, year), (None, ""))
|
||||
_put(
|
||||
values,
|
||||
sources,
|
||||
f"oews:{soc}:a_median",
|
||||
median,
|
||||
f"bls.oews:{item}",
|
||||
)
|
||||
for industry in industries:
|
||||
employment, item = ces.get((industry, year), (None, ""))
|
||||
_put(
|
||||
values,
|
||||
sources,
|
||||
f"ces:{industry}:employment",
|
||||
employment,
|
||||
f"bls.ces:{item}",
|
||||
)
|
||||
for series_id in FRED_SERIES:
|
||||
value, item = fred.get((series_id, year), (None, ""))
|
||||
_put(values, sources, f"fred:{series_id}", value, f"econ.fred:{item}")
|
||||
out.append(JointYear(year, values, sources))
|
||||
return out
|
||||
|
||||
|
||||
def joint_monthly(
|
||||
con: Any,
|
||||
*,
|
||||
family: str | None = None,
|
||||
specialty: str | None = None,
|
||||
years: Sequence[int],
|
||||
) -> list[JointMonth]:
|
||||
"""Months in *years* that have a CES, FRED or benchmark total-return row.
|
||||
|
||||
CES is the family's industries, or offices of physicians (``621100``)
|
||||
when no family is named. *specialty* is accepted and unused: monthly
|
||||
columns are employment, prices and the FRED mean, not a wage or a BHAR.
|
||||
"""
|
||||
del specialty
|
||||
from market.tables import FRED_SERIES, read_fred, read_series
|
||||
|
||||
wanted = {int(year) for year in years}
|
||||
industries = _industries(family) if family else _DEFAULT_INDUSTRY
|
||||
ces_at = _ces_monthly(con, industries, wanted)
|
||||
fred_at: dict[tuple[str, int, int], list] = defaultdict(list)
|
||||
fred_rows = _call(lambda: read_fred(con), None) or []
|
||||
for row in fred_rows:
|
||||
parsed = _year_month(row.period)
|
||||
if parsed is None or parsed[0] not in wanted:
|
||||
continue
|
||||
fred_at[(row.series_id, parsed[0], parsed[1])].append(row)
|
||||
tri_at: dict[tuple[str, int, int], tuple[float, str]] = {}
|
||||
series_rows = _call(lambda: read_series(con, list(_TRI)), None) or []
|
||||
for row in series_rows:
|
||||
if not row.series_id.endswith(":tri") or row.year not in wanted:
|
||||
continue
|
||||
tri_at[(row.symbol, row.year, row.month)] = (row.value, row.item_key)
|
||||
stamps = {(year, month) for _industry, year, month in ces_at}
|
||||
stamps |= {(year, month) for _series, year, month in fred_at}
|
||||
stamps |= {(year, month) for _symbol, year, month in tri_at}
|
||||
out: list[JointMonth] = []
|
||||
for year, month in sorted(stamps):
|
||||
values: dict[str, float | None] = {}
|
||||
sources: dict[str, str] = {}
|
||||
for industry in industries:
|
||||
value, item = ces_at.get((industry, year, month), (None, ""))
|
||||
_put(
|
||||
values,
|
||||
sources,
|
||||
f"ces:{industry}:employment",
|
||||
value,
|
||||
f"bls.ces:{item}",
|
||||
)
|
||||
for series_id in FRED_SERIES:
|
||||
group = [
|
||||
row
|
||||
for row in fred_at.get((series_id, year, month), ())
|
||||
if row.value is not None
|
||||
]
|
||||
if group:
|
||||
mean = sum(row.value for row in group) / len(group)
|
||||
item = max(group, key=lambda row: row.period).item_key
|
||||
else:
|
||||
mean, item = None, ""
|
||||
_put(values, sources, f"fred:{series_id}", mean, f"econ.fred:{item}")
|
||||
for symbol in _TRI:
|
||||
value, item = tri_at.get((symbol, year, month), (None, ""))
|
||||
_put(
|
||||
values, sources, f"market:{symbol}:tri", value, f"market.series:{item}"
|
||||
)
|
||||
out.append(JointMonth(year, month, values, sources))
|
||||
return out
|
||||
|
||||
|
||||
def _put(
|
||||
values: dict[str, float | None],
|
||||
sources: dict[str, str],
|
||||
key: str,
|
||||
value: float | None,
|
||||
source: str,
|
||||
) -> None:
|
||||
values[key] = value
|
||||
sources[key] = source
|
||||
|
||||
|
||||
def _industries(family: str) -> tuple[str, ...]:
|
||||
return FAMILY_INDUSTRY.get(family, _DEFAULT_INDUSTRY)
|
||||
|
||||
|
||||
def _put_market(
|
||||
con: Any,
|
||||
values: dict[str, float | None],
|
||||
sources: dict[str, str],
|
||||
scope: str,
|
||||
target: str,
|
||||
year: int,
|
||||
) -> None:
|
||||
value, item = _market_bhar(con, scope, target, year)
|
||||
_put(
|
||||
values, sources, f"market:{scope}:{target}:bhar", value, f"market.series:{item}"
|
||||
)
|
||||
|
||||
|
||||
def _market_bhar(
|
||||
con: Any, scope: str, target: str, year: int
|
||||
) -> tuple[float | None, str]:
|
||||
"""Equal-weight portfolio BHAR against SPY. The cite is SPY's tri series."""
|
||||
from market.portfolio import members, portfolio_returns, year_bhar
|
||||
from market.tables import read_series
|
||||
from market.tiingo import read_returns
|
||||
|
||||
item = ""
|
||||
spy = _call(lambda: read_series(con, ["SPY"]), None) or []
|
||||
for row in spy:
|
||||
if row.series_id.endswith(":tri") and row.item_key:
|
||||
item = row.item_key
|
||||
break
|
||||
found = _call(
|
||||
lambda: members(con, scope=scope, target=target, on=date(year, 12, 31)),
|
||||
None,
|
||||
)
|
||||
if found is None:
|
||||
return None, item
|
||||
chosen, _gaps = found
|
||||
if not chosen:
|
||||
return None, item
|
||||
returns = _call(
|
||||
lambda: read_returns(
|
||||
con,
|
||||
[member.ticker for member in chosen] + ["SPY"],
|
||||
date(year, 1, 1),
|
||||
date(year, 12, 31),
|
||||
),
|
||||
None,
|
||||
)
|
||||
if returns is None:
|
||||
return None, item
|
||||
port = portfolio_returns(returns, chosen)
|
||||
return year_bhar(port, returns.get("SPY", {}), year), item
|
||||
|
||||
|
||||
def _family_codes(con: Any, family: str) -> list[str] | None:
|
||||
"""Member codes, or ``None`` when ``pfs.code_family`` is not on *con*."""
|
||||
found = _call(
|
||||
lambda: con.execute(
|
||||
"SELECT DISTINCT code FROM pfs.code_family WHERE key = ?",
|
||||
[family],
|
||||
).fetchall(),
|
||||
None,
|
||||
)
|
||||
if found is None:
|
||||
return None
|
||||
return [row[0] for row in found]
|
||||
|
||||
|
||||
def _utilization(con: Any, family: str) -> tuple[dict[int, float], dict[int, set[str]]]:
|
||||
from pfs.event_study import family_series
|
||||
|
||||
codes = _family_codes(con, family)
|
||||
if not codes:
|
||||
return {}, {}
|
||||
totals = _call(lambda: family_series(con, family), None)
|
||||
if totals is None:
|
||||
return {}, {}
|
||||
keys: dict[int, set[str]] = defaultdict(set)
|
||||
marks = ",".join("?" * len(codes))
|
||||
found = con.execute(
|
||||
"SELECT year, item_key FROM pfs.utilization "
|
||||
f"WHERE hcpcs IN ({marks}) AND item_key <> ''",
|
||||
[code.upper() for code in codes],
|
||||
).fetchall()
|
||||
for year, item in found:
|
||||
keys[int(year)].add(item)
|
||||
return totals, keys
|
||||
|
||||
|
||||
def _psps(
|
||||
con: Any, family: str, specialty: str, years: Sequence[int]
|
||||
) -> dict[int, tuple[float | None, str]]:
|
||||
from pfs.commenters import specialty_key
|
||||
from pfs.psps import read_by_specialty
|
||||
|
||||
codes = _family_codes(con, family)
|
||||
if not codes:
|
||||
return {}
|
||||
rows = _call(lambda: read_by_specialty(con, codes, years=years), None)
|
||||
if rows is None:
|
||||
return {}
|
||||
wanted = specialty_key(specialty)
|
||||
totals: dict[int, float] = {}
|
||||
keys: dict[int, set[str]] = defaultdict(set)
|
||||
seen: set[int] = set()
|
||||
for row in rows:
|
||||
if specialty_key(row.specialty) != wanted or row.allowed is None:
|
||||
continue
|
||||
seen.add(row.year)
|
||||
totals[row.year] = totals.get(row.year, 0.0) + float(row.allowed)
|
||||
if row.item_key:
|
||||
keys[row.year].add(row.item_key)
|
||||
return {
|
||||
year: (totals.get(year), ",".join(sorted(keys.get(year, ())))) for year in seen
|
||||
}
|
||||
|
||||
|
||||
def _oews(
|
||||
con: Any, socs: tuple[str, ...], years: Sequence[int]
|
||||
) -> dict[tuple[str, int], tuple[float | None, str]]:
|
||||
from bls.oews import read_occupations
|
||||
|
||||
if not socs:
|
||||
return {}
|
||||
rows = _call(lambda: read_occupations(con, socs, years=years), None)
|
||||
if rows is None:
|
||||
return {}
|
||||
out: dict[tuple[str, int], tuple[float | None, str]] = {}
|
||||
for row in rows:
|
||||
out[(row.occ_code, row.year)] = (row.a_median, row.item_key)
|
||||
return out
|
||||
|
||||
|
||||
def _employment_ids(industries: Sequence[str]) -> list[str]:
|
||||
from bls.ces import SERIES
|
||||
|
||||
ids: list[str] = []
|
||||
for industry in industries:
|
||||
for series in SERIES.values():
|
||||
if series.industry_code == industry and series.data_type == "employment":
|
||||
ids.append(series.series_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _ces_annual(
|
||||
con: Any, industries: Sequence[str], years: Sequence[int]
|
||||
) -> dict[tuple[str, int], tuple[float | None, str]]:
|
||||
from bls.ces import SERIES, read_series
|
||||
|
||||
ids = _employment_ids(industries)
|
||||
if not ids:
|
||||
return {}
|
||||
rows = _call(lambda: read_series(con, ids, annual=True), None)
|
||||
if rows is None:
|
||||
return {}
|
||||
wanted = set(years)
|
||||
out: dict[tuple[str, int], tuple[float | None, str]] = {}
|
||||
for row in rows:
|
||||
if row.year not in wanted:
|
||||
continue
|
||||
meta = SERIES[row.series_id]
|
||||
out[(meta.industry_code, row.year)] = (row.value, row.item_key)
|
||||
return out
|
||||
|
||||
|
||||
def _ces_monthly(
|
||||
con: Any, industries: Sequence[str], years: set[int]
|
||||
) -> dict[tuple[str, int, int], tuple[float | None, str]]:
|
||||
from bls.ces import SERIES, read_series
|
||||
|
||||
ids = _employment_ids(industries)
|
||||
rows = _call(lambda: read_series(con, ids, annual=False), None) if ids else []
|
||||
if rows is None:
|
||||
return {}
|
||||
out: dict[tuple[str, int, int], tuple[float | None, str]] = {}
|
||||
for row in rows:
|
||||
if row.year not in years or not 1 <= row.month <= 12:
|
||||
continue
|
||||
meta = SERIES[row.series_id]
|
||||
out[(meta.industry_code, row.year, row.month)] = (row.value, row.item_key)
|
||||
return out
|
||||
|
||||
|
||||
def _fred_annual(con: Any) -> dict[tuple[str, int], tuple[float | None, str]]:
|
||||
from market.tables import read_fred
|
||||
|
||||
rows = _call(lambda: read_fred(con), None)
|
||||
if rows is None:
|
||||
return {}
|
||||
grouped: dict[tuple[str, int], list] = defaultdict(list)
|
||||
for row in rows:
|
||||
year = row.year
|
||||
if year is None and len(row.period) >= 4 and row.period[:4].isdigit():
|
||||
year = int(row.period[:4])
|
||||
if year is None:
|
||||
continue
|
||||
grouped[(row.series_id, int(year))].append(row)
|
||||
out: dict[tuple[str, int], tuple[float | None, str]] = {}
|
||||
for key, group in grouped.items():
|
||||
usable = [row for row in group if row.value is not None]
|
||||
if not usable:
|
||||
out[key] = (None, "")
|
||||
continue
|
||||
latest = max(usable, key=lambda row: row.period)
|
||||
out[key] = (sum(row.value for row in usable) / len(usable), latest.item_key)
|
||||
return out
|
||||
|
||||
|
||||
def _year_month(period: str) -> tuple[int, int] | None:
|
||||
parts = period.split("-")
|
||||
if len(parts) < 2 or not (len(parts[0]) == 4 and parts[0].isdigit()):
|
||||
return None
|
||||
month_text = parts[1][:2]
|
||||
if not month_text.isdigit():
|
||||
return None
|
||||
month = int(month_text)
|
||||
if not 1 <= month <= 12:
|
||||
return None
|
||||
return int(parts[0]), month
|
||||
@@ -4,7 +4,8 @@ Sites come from the CMS All Owners snapshot of 2026-09-29 joined to
|
||||
enrollment ZIPs, and from NPPES organization rows. Family billers come from
|
||||
the 2024 Medicare physician-and-other-practitioners file. Amounts split
|
||||
across districts by :func:`market.districts.zip_weights`. An unmatched ZIP
|
||||
is kept as ``??``. ``agi_k`` stays empty until the SOI join.
|
||||
is kept as ``??``. ``agi_k`` is IRS SOI income rolled by ``zip_weights``,
|
||||
using the latest SOI year at or before the footprint year.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -24,6 +25,7 @@ from market.tables import (
|
||||
read_assignments,
|
||||
read_footprints,
|
||||
read_members,
|
||||
read_soi_zip,
|
||||
)
|
||||
from pfs.codetables import is_missing_table_error
|
||||
|
||||
@@ -73,8 +75,8 @@ class AllocItem:
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DistrictView:
|
||||
"""One district's footprint beside its House member. ``agi_k`` is filled
|
||||
by the SOI join."""
|
||||
"""One district's footprint beside its House member. ``agi_k`` is ZIP
|
||||
income rolled onto the district (thousands of dollars)."""
|
||||
|
||||
district_id: str
|
||||
bioguide_id: str
|
||||
@@ -334,6 +336,7 @@ def district_view(
|
||||
if row.congress != congress or row.committee_id not in HEALTH_COMMITTEES:
|
||||
continue
|
||||
committees.setdefault(row.bioguide_id, set()).add(row.committee_id)
|
||||
agi = _agi_by_district(con, congress, year)
|
||||
views: list[DistrictView] = []
|
||||
for district, (providers, sites, services, payment) in totals.items():
|
||||
member = members.get(district)
|
||||
@@ -349,13 +352,46 @@ def district_view(
|
||||
sites,
|
||||
services,
|
||||
payment,
|
||||
None,
|
||||
agi.get(district),
|
||||
)
|
||||
)
|
||||
views.sort(key=lambda v: (-v.payment, v.district_id))
|
||||
return views
|
||||
|
||||
|
||||
def _agi_by_district(con: Any, congress: int, year: int) -> dict[str, float]:
|
||||
"""SOI AGI (thousands) times ZIP weight. The latest SOI year at or
|
||||
before *year* is used: the footprint year is 2024 and SOI stops at 2023.
|
||||
|
||||
A non-positive weight adds nothing, so a zero-weight crosswalk stays
|
||||
``None`` rather than 0.
|
||||
"""
|
||||
try:
|
||||
chosen = con.execute(
|
||||
"SELECT max(year) FROM econ.soi_zip WHERE year <= ? AND agi_k IS NOT NULL",
|
||||
[year],
|
||||
).fetchone()[0]
|
||||
except Exception as exc:
|
||||
if is_missing_table_error(exc):
|
||||
return {}
|
||||
raise
|
||||
if chosen is None:
|
||||
return {}
|
||||
agi = {
|
||||
row.zip: row.agi_k
|
||||
for row in read_soi_zip(con, year=int(chosen))
|
||||
if row.agi_k is not None
|
||||
}
|
||||
weights = zip_weights(con, congress)
|
||||
out: dict[str, float] = {}
|
||||
for zipcode, amount in agi.items():
|
||||
for district, weight in weights.get(zipcode, ()):
|
||||
if weight <= 0:
|
||||
continue
|
||||
out[district] = out.get(district, 0.0) + float(amount) * float(weight)
|
||||
return out
|
||||
|
||||
|
||||
def _missing(exc: Exception) -> bool:
|
||||
return is_missing_table_error(exc)
|
||||
|
||||
|
||||
@@ -3563,3 +3563,81 @@ def test_revenue_cli_refusal_prints_gap(env, monkeypatch):
|
||||
assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output
|
||||
assert T.read_revenue(c, "0000001234") == []
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_joint_cli_prints(env):
|
||||
c, _, published, _ = env
|
||||
T.write_fred(c, [T.FredRow("CPIMEDSL", 2020, "2020-06-01", 9.0, "fk")])
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"joint",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--from",
|
||||
"2020",
|
||||
"--to",
|
||||
"2020",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2020" in res.output
|
||||
assert "fred:CPIMEDSL=9" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_joint_cli_monthly(env):
|
||||
from bls.ces import CesRow
|
||||
from bls.ces import write as write_ces
|
||||
|
||||
c, _, published, _ = env
|
||||
write_ces(
|
||||
c,
|
||||
[
|
||||
CesRow(
|
||||
"CES6562110001",
|
||||
"621100",
|
||||
"Offices of physicians",
|
||||
"employment",
|
||||
"S",
|
||||
2024,
|
||||
3,
|
||||
7.0,
|
||||
"ces-m",
|
||||
)
|
||||
],
|
||||
)
|
||||
T.write_fred(c, [T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m")])
|
||||
T.write_series(
|
||||
c,
|
||||
"tiingo:XLV:tri",
|
||||
[T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k")],
|
||||
)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"joint",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--from",
|
||||
"2024",
|
||||
"--to",
|
||||
"2024",
|
||||
"--monthly",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "2024-03" in res.output
|
||||
assert "ces:621100:employment=7" in res.output
|
||||
assert "fred:CPIMEDSL=1.5" in res.output
|
||||
assert "market:XLV:tri=120" in res.output
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_joint_cli_requires_family_or_specialty(env):
|
||||
res = runner.invoke(app, ["market", "joint"])
|
||||
assert res.exit_code == 2
|
||||
assert "family" in res.output
|
||||
assert "specialty" in res.output
|
||||
|
||||
431
tests/market/test_enrich.py
Normal file
431
tests/market/test_enrich.py
Normal file
@@ -0,0 +1,431 @@
|
||||
"""Joint annual and monthly readers beside BLS, FRED, PSPS and utilization."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, timedelta
|
||||
|
||||
import duckdb
|
||||
|
||||
from bls.ces import CesRow
|
||||
from bls.ces import write as write_ces
|
||||
from bls.oews import OewsRow
|
||||
from bls.oews import write as write_oews
|
||||
from market import enrich as E
|
||||
from market import tables as T
|
||||
from pfs.codetables import PspsRow, ensure_tables, write_psps
|
||||
from pfs.utilization import UtilizationRow
|
||||
from pfs.utilization import write as write_util
|
||||
|
||||
|
||||
def _days(n: int, start: date = date(2020, 1, 2)) -> list[date]:
|
||||
out: list[date] = []
|
||||
day = start
|
||||
while len(out) < n:
|
||||
if day.weekday() < 5:
|
||||
out.append(day)
|
||||
day += timedelta(days=1)
|
||||
return out
|
||||
|
||||
|
||||
def _oews(year: int, code: str, median: float, item_key: str) -> OewsRow:
|
||||
return OewsRow(
|
||||
year,
|
||||
code,
|
||||
"Family Medicine Physicians",
|
||||
"detailed",
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
median,
|
||||
None,
|
||||
None,
|
||||
"OEUN",
|
||||
item_key,
|
||||
)
|
||||
|
||||
|
||||
def _seed_joint(con) -> None:
|
||||
"""2020 inputs whose column values are fixed numbers."""
|
||||
T.ensure_tables(con)
|
||||
ensure_tables(con)
|
||||
con.execute(
|
||||
"CREATE TABLE IF NOT EXISTS pfs.code_family ("
|
||||
"key VARCHAR, name VARCHAR, code VARCHAR, role VARCHAR, since INTEGER, "
|
||||
"until INTEGER, item_key VARCHAR, p_id INTEGER, note VARCHAR)"
|
||||
)
|
||||
con.execute(
|
||||
"INSERT INTO pfs.code_family VALUES "
|
||||
"('HOME-VISITS', 'Home', '99349', 'member', 2015, NULL, 'fam', 0, ''), "
|
||||
"('HOME-VISITS', 'Home', '99350', 'member', 2015, NULL, 'fam', 0, '')"
|
||||
)
|
||||
write_util(
|
||||
con,
|
||||
2020,
|
||||
[
|
||||
UtilizationRow(
|
||||
2020,
|
||||
"99349",
|
||||
"",
|
||||
"N",
|
||||
"O",
|
||||
1,
|
||||
1,
|
||||
30.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"d",
|
||||
"util-k",
|
||||
),
|
||||
UtilizationRow(
|
||||
2020,
|
||||
"99350",
|
||||
"",
|
||||
"N",
|
||||
"O",
|
||||
1,
|
||||
1,
|
||||
50.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"d",
|
||||
"util-k",
|
||||
),
|
||||
UtilizationRow(
|
||||
2020,
|
||||
"11111",
|
||||
"",
|
||||
"N",
|
||||
"O",
|
||||
1,
|
||||
1,
|
||||
5.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"d",
|
||||
"util-other",
|
||||
),
|
||||
],
|
||||
)
|
||||
write_psps(
|
||||
con,
|
||||
2020,
|
||||
"99349",
|
||||
[PspsRow(2020, "99349", "08", "Family Practice", 1, 25.0, 1, "ds", "psps-k")],
|
||||
)
|
||||
write_psps(
|
||||
con,
|
||||
2020,
|
||||
"99350",
|
||||
[
|
||||
PspsRow(
|
||||
2020, "99350", "08", "Family Practice", 1, 100.0, 1, "ds", "psps-k"
|
||||
),
|
||||
PspsRow(
|
||||
2020, "99350", "08", "Family Practice", 1, None, 1, "ds", "psps-null"
|
||||
),
|
||||
PspsRow(2020, "99350", "09", "Family Practice", 1, 0.0, 1, "ds", ""),
|
||||
PspsRow(
|
||||
2020,
|
||||
"99350",
|
||||
"16",
|
||||
"Obstetrics/Gynecology",
|
||||
1,
|
||||
999.0,
|
||||
1,
|
||||
"ds",
|
||||
"psps-ob",
|
||||
),
|
||||
],
|
||||
)
|
||||
write_psps(
|
||||
con,
|
||||
2020,
|
||||
"11111",
|
||||
[PspsRow(2020, "11111", "08", "Family Practice", 1, 50.0, 1, "ds", "psps-k")],
|
||||
)
|
||||
write_oews(con, 2020, [_oews(2020, "29-1215", 211000.0, "oews-k")])
|
||||
write_ces(
|
||||
con,
|
||||
[
|
||||
CesRow(
|
||||
"CES6562110001",
|
||||
"621100",
|
||||
"Offices of physicians",
|
||||
"employment",
|
||||
"S",
|
||||
2020,
|
||||
1,
|
||||
10.0,
|
||||
"ces-k",
|
||||
),
|
||||
CesRow(
|
||||
"CES6562110001",
|
||||
"621100",
|
||||
"Offices of physicians",
|
||||
"employment",
|
||||
"S",
|
||||
2020,
|
||||
2,
|
||||
30.0,
|
||||
"ces-k",
|
||||
),
|
||||
],
|
||||
)
|
||||
T.write_fred(
|
||||
con,
|
||||
[
|
||||
T.FredRow("CPIMEDSL", 2020, "2020-01-01", 2.0, "fred-k"),
|
||||
T.FredRow("CPIMEDSL", 2020, "2020-02-01", 4.0, "fred-k"),
|
||||
T.FredRow("CPIMEDSL", 2020, "2020-03-01", None, "fred-k"),
|
||||
T.FredRow("CPIAUCSL", 2020, "2020-01-01", None, "none-k"),
|
||||
T.FredRow("DTB3", None, "2020-05-01", 1.0, "parsed-k"),
|
||||
T.FredRow("VIXCLS", None, "nope", 1.0, "bad-k"),
|
||||
],
|
||||
)
|
||||
T.write_entities(con, [T.EntityRow("E1", "Home Co", "company")])
|
||||
T.write_securities(
|
||||
con, "E1", [T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")]
|
||||
)
|
||||
T.write_links(
|
||||
con,
|
||||
[
|
||||
T.LinkRow(
|
||||
"E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x"
|
||||
),
|
||||
T.LinkRow(
|
||||
"E1",
|
||||
"specialty",
|
||||
"FAMILY PRACTICE",
|
||||
"curated",
|
||||
"curated",
|
||||
"item_key",
|
||||
"x",
|
||||
),
|
||||
],
|
||||
)
|
||||
days = _days(200)
|
||||
con.executemany(
|
||||
"INSERT INTO market.price_daily (ticker, day, ret) VALUES (?, ?, 0)",
|
||||
[(ticker, day) for ticker in ("HV", "SPY") for day in days],
|
||||
)
|
||||
T.write_series(
|
||||
con,
|
||||
"tiingo:SPY:tri",
|
||||
[T.MonthlyRow("tiingo:SPY:tri", "SPY", "tiingo", 2020, 12, 100.0, "spy-k")],
|
||||
)
|
||||
|
||||
|
||||
def test_joint_annual_columns_and_sources():
|
||||
con = duckdb.connect(":memory:")
|
||||
_seed_joint(con)
|
||||
rows = E.joint_annual(
|
||||
con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
|
||||
)
|
||||
con.close()
|
||||
assert [row.year for row in rows] == [2020]
|
||||
values, sources = rows[0].values, rows[0].sources
|
||||
assert values["market:family:HOME-VISITS:bhar"] == 0.0
|
||||
assert values["market:specialty:FAMILY PRACTICE:bhar"] == 0.0
|
||||
assert values["util:HOME-VISITS:n_services"] == 80.0
|
||||
assert values["psps:FAMILY PRACTICE:allowed"] == 125.0
|
||||
assert values["oews:29-1215:a_median"] == 211000.0
|
||||
assert values["ces:621100:employment"] == 20.0
|
||||
assert values["ces:621600:employment"] is None
|
||||
assert values["fred:CPIMEDSL"] == 3.0
|
||||
assert {key for key in values if key.startswith("fred:")} == {
|
||||
f"fred:{series}" for series in T.FRED_SERIES
|
||||
}
|
||||
assert sources["market:family:HOME-VISITS:bhar"] == "market.series:spy-k"
|
||||
assert sources["util:HOME-VISITS:n_services"] == "pfs.utilization:util-k"
|
||||
assert sources["psps:FAMILY PRACTICE:allowed"] == "pfs.utilization_specialty:psps-k"
|
||||
assert sources["oews:29-1215:a_median"] == "bls.oews:oews-k"
|
||||
assert sources["ces:621100:employment"] == "bls.ces:ces-k"
|
||||
assert sources["fred:CPIMEDSL"] == "econ.fred:fred-k"
|
||||
|
||||
|
||||
def test_joint_psps_uses_codes_and_specialty_key():
|
||||
con = duckdb.connect(":memory:")
|
||||
_seed_joint(con)
|
||||
rows = E.joint_annual(
|
||||
con, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
|
||||
)
|
||||
con.close()
|
||||
assert rows[0].values["psps:FAMILY PRACTICE:allowed"] == 125.0
|
||||
|
||||
|
||||
def test_joint_specialty_only_has_no_psps():
|
||||
con = duckdb.connect(":memory:")
|
||||
_seed_joint(con)
|
||||
rows = E.joint_annual(con, specialty="FAMILY PRACTICE", years=(2020,))
|
||||
con.close()
|
||||
assert not any(key.startswith("psps:") for key in rows[0].values)
|
||||
assert rows[0].sources["psps"] == "needs a family"
|
||||
|
||||
|
||||
def test_joint_missing_inputs_are_none_not_zero():
|
||||
con = duckdb.connect(":memory:")
|
||||
_seed_joint(con)
|
||||
rows = E.joint_annual(con, family="CCM", years=(2021,))
|
||||
assert rows[0].values["fred:CPIMEDSL"] is None
|
||||
assert rows[0].values["ces:621100:employment"] is None
|
||||
assert "ces:621600:employment" not in rows[0].values
|
||||
assert rows[0].values["util:CCM:n_services"] is None
|
||||
both = E.joint_annual(con, family="CCM", specialty="FAMILY PRACTICE", years=(2021,))
|
||||
assert both[0].values["psps:FAMILY PRACTICE:allowed"] is None
|
||||
con.close()
|
||||
|
||||
bare = duckdb.connect(":memory:")
|
||||
assert E.annual_tri(bare, "XLV") == {}
|
||||
assert E.joint_monthly(bare, years=(2020,)) == []
|
||||
bare_rows = E.joint_annual(
|
||||
bare, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
|
||||
)
|
||||
assert bare_rows[0].values["fred:CPIMEDSL"] is None
|
||||
assert bare_rows[0].values["util:HOME-VISITS:n_services"] is None
|
||||
assert bare_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None
|
||||
bare.close()
|
||||
|
||||
partial = duckdb.connect(":memory:")
|
||||
partial.execute("CREATE SCHEMA pfs")
|
||||
partial.execute("CREATE TABLE pfs.code_family (key VARCHAR, code VARCHAR)")
|
||||
partial.execute("INSERT INTO pfs.code_family VALUES ('HOME-VISITS', '99350')")
|
||||
partial_rows = E.joint_annual(
|
||||
partial, family="HOME-VISITS", specialty="FAMILY PRACTICE", years=(2020,)
|
||||
)
|
||||
assert partial_rows[0].values["util:HOME-VISITS:n_services"] is None
|
||||
assert partial_rows[0].values["psps:FAMILY PRACTICE:allowed"] is None
|
||||
partial.close()
|
||||
|
||||
priced = duckdb.connect(":memory:")
|
||||
T.ensure_tables(priced)
|
||||
T.write_entities(priced, [T.EntityRow("E1", "Home Co", "company")])
|
||||
T.write_securities(
|
||||
priced,
|
||||
"E1",
|
||||
[T.SecurityRow("E1", "HV", "XNAS", date(2020, 1, 1), None, "tiingo")],
|
||||
)
|
||||
T.write_links(
|
||||
priced,
|
||||
[
|
||||
T.LinkRow(
|
||||
"E1", "family", "HOME-VISITS", "curated", "curated", "item_key", "x"
|
||||
)
|
||||
],
|
||||
)
|
||||
priced.execute("DROP TABLE market.price_daily")
|
||||
priced_rows = E.joint_annual(priced, family="HOME-VISITS", years=(2020,))
|
||||
assert priced_rows[0].values["market:family:HOME-VISITS:bhar"] is None
|
||||
priced.close()
|
||||
|
||||
|
||||
def test_joint_monthly_alignment():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
write_ces(
|
||||
con,
|
||||
[
|
||||
CesRow(
|
||||
"CES6562110001",
|
||||
"621100",
|
||||
"Offices of physicians",
|
||||
"employment",
|
||||
"S",
|
||||
2024,
|
||||
3,
|
||||
7.0,
|
||||
"ces-m",
|
||||
),
|
||||
CesRow(
|
||||
"CES6562110001",
|
||||
"621100",
|
||||
"Offices of physicians",
|
||||
"employment",
|
||||
"S",
|
||||
2024,
|
||||
13,
|
||||
99.0,
|
||||
"ces-m",
|
||||
),
|
||||
],
|
||||
)
|
||||
T.write_fred(
|
||||
con,
|
||||
[
|
||||
T.FredRow("CPIMEDSL", 2024, "2024-03-01", 1.5, "fred-m"),
|
||||
T.FredRow("CPIMEDSL", 2024, "nope", 9.0, "bad"),
|
||||
T.FredRow("CPIMEDSL", 2019, "2019-03-01", 9.0, "old"),
|
||||
T.FredRow("VIXCLS", 2024, "2024-13-01", 9.0, "badm"),
|
||||
T.FredRow("USREC", 2024, "2024-xx", 9.0, "badm"),
|
||||
],
|
||||
)
|
||||
T.write_series(
|
||||
con,
|
||||
"tiingo:XLV:tri",
|
||||
[
|
||||
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 3, 120.0, "xlv-k"),
|
||||
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2019, 3, 1.0, "old"),
|
||||
],
|
||||
)
|
||||
T.write_series(
|
||||
con,
|
||||
"tiingo:XLV:close",
|
||||
[T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 3, 1.0, "c")],
|
||||
)
|
||||
months = E.joint_monthly(con, years=(2024,))
|
||||
con.close()
|
||||
march = next(row for row in months if row.year == 2024 and row.month == 3)
|
||||
assert march.values["ces:621100:employment"] == 7.0
|
||||
assert march.values["fred:CPIMEDSL"] == 1.5
|
||||
assert march.values["market:XLV:tri"] == 120.0
|
||||
|
||||
|
||||
def test_annual_tri():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_series(
|
||||
con,
|
||||
"tiingo:XLV:tri",
|
||||
[
|
||||
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 1, 100.0, "k"),
|
||||
T.MonthlyRow("tiingo:XLV:tri", "XLV", "tiingo", 2024, 2, 110.0, "k"),
|
||||
],
|
||||
)
|
||||
T.write_series(
|
||||
con,
|
||||
"tiingo:XLV:close",
|
||||
[T.MonthlyRow("tiingo:XLV:close", "XLV", "tiingo", 2024, 1, 999.0, "k")],
|
||||
)
|
||||
got = E.annual_tri(con, "XLV")
|
||||
con.close()
|
||||
assert got == {2024: 105.0}
|
||||
|
||||
|
||||
def test_specialty_soc_keys_are_specialty_keys():
|
||||
from pfs.commenters import specialty_key
|
||||
|
||||
assert E.FAMILY_INDUSTRY == {"HOME-VISITS": ("621100", "621600")}
|
||||
assert E.SPECIALTY_SOC
|
||||
for key, codes in E.SPECIALTY_SOC.items():
|
||||
assert key == specialty_key(key)
|
||||
assert codes
|
||||
for code in codes:
|
||||
assert re.fullmatch(r"\d{2}-\d{4}", code)
|
||||
@@ -212,6 +212,75 @@ def test_district_view_joins_member_and_committees():
|
||||
assert views[0].payment == 75
|
||||
|
||||
|
||||
def test_district_view_agi_from_soi():
|
||||
"""ZIP 48201's 2023 AGI rolls onto the 2024 footprint by district weight.
|
||||
|
||||
SOI has no 2024 file, so the latest year at or before the footprint year
|
||||
is the one that fills ``agi_k``.
|
||||
"""
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
T.write_footprints(
|
||||
con,
|
||||
[
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"MI-12",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"zip-weight",
|
||||
"prov-svc",
|
||||
),
|
||||
T.FootprintRow(
|
||||
"family",
|
||||
"HOME-VISITS",
|
||||
119,
|
||||
"MI-14",
|
||||
2024,
|
||||
1,
|
||||
0,
|
||||
1,
|
||||
1,
|
||||
"zip-weight",
|
||||
"prov-svc",
|
||||
),
|
||||
],
|
||||
)
|
||||
T.write_zip_districts(
|
||||
con,
|
||||
[
|
||||
T.ZipDistrictRow(119, "48201", "MI-12", 0.75, "hud-bus", "2026Q2", ""),
|
||||
T.ZipDistrictRow(119, "48201", "MI-13", 0.25, "hud-bus", "2026Q2", ""),
|
||||
T.ZipDistrictRow(119, "00000", "MI-14", 0.0, "hud-bus", "2026Q2", ""),
|
||||
],
|
||||
)
|
||||
T.write_soi_zip(
|
||||
con,
|
||||
[
|
||||
T.SoiZipRow(2023, "48201", "26", 10, 1000, "soi-k"),
|
||||
T.SoiZipRow(2023, "00000", "26", 1, 500, "soi-k"),
|
||||
T.SoiZipRow(2025, "48201", "26", 1, 9999, "later"),
|
||||
],
|
||||
)
|
||||
views = F.district_view(
|
||||
con, congress=119, level="family", key="HOME-VISITS", year=2024
|
||||
)
|
||||
by_id = {view.district_id: view for view in views}
|
||||
assert by_id["MI-12"].agi_k == 750.0
|
||||
assert by_id["MI-14"].agi_k is None
|
||||
con.execute("DROP TABLE econ.soi_zip")
|
||||
missing = F.district_view(
|
||||
con, congress=119, level="family", key="HOME-VISITS", year=2024
|
||||
)
|
||||
con.close()
|
||||
assert all(view.agi_k is None for view in missing)
|
||||
|
||||
|
||||
def test_enrichment_root_blank_npi_and_null_payment(tmp_path):
|
||||
assert F.enrichment_root().name == "lead_enrichment"
|
||||
assert (
|
||||
|
||||
@@ -20,6 +20,7 @@ MODULES = [
|
||||
"market.events",
|
||||
"market.abnormal",
|
||||
"market.portfolio",
|
||||
"market.enrich",
|
||||
]
|
||||
|
||||
_PROBE = """
|
||||
|
||||
Reference in New Issue
Block a user