feat(market): FRED (fredgraph), IRS SOI ZIP income and EDGAR revenue by period end, each cited
All checks were successful
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 2m21s
Deploy / changes (push) Successful in 15s
Infra CI / notebooks (push) Successful in 53s
CI / test (push) Successful in 4m33s
Infra CI / zotero (push) Successful in 20s
Infra CI / docs (push) Successful in 25s
Infra CI / bcda (push) Successful in 1m36s
Infra CI / api (push) Successful in 2m14s
Infra CI / mc (push) Successful in 27s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m11s
Deploy / api (push) Successful in 1m37s
Deploy / bcda (push) Successful in 1m52s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m14s
Deploy / report (push) Successful in 15s

This commit is contained in:
kert
2026-11-17 12:00:21 -05:00
parent 07a8944b61
commit d008735c77
10 changed files with 986 additions and 2 deletions

View File

@@ -8,6 +8,9 @@ uv run stack market footprint --entities --write
uv run stack market footprint --family HOME-VISITS --write
uv run stack market district-view --family HOME-VISITS --top 10
uv run stack market events --write # releases, meetings, tenures
uv run stack market fred --write # FRED fredgraph series
uv run stack market soi --write # IRS SOI ZIP income
uv run stack market revenue --write # EDGAR annual revenue
uv run stack market gaps # sources that could not be read
Every command fetches first, then opens one short DuckDB batch that only
@@ -43,17 +46,21 @@ from market.seed import (
validate_seed,
)
from market.tables import (
FRED_SERIES,
EntityRow,
FecCommitteeRow,
FecContributionRow,
FecTotalsRow,
FilingRow,
FredRow,
HitRow,
LdaFilingRow,
LdaIssueRow,
LinkRow,
PassageRow,
RevenueRow,
SecurityRow,
SoiZipRow,
clear_gap,
ensure_tables,
read_entities,
@@ -80,14 +87,17 @@ from market.tables import (
write_fec_totals,
write_filings,
write_footprints,
write_fred,
write_gap,
write_hits,
write_links,
write_members,
write_passages,
write_prices,
write_revenue,
write_securities,
write_series,
write_soi_zip,
write_tenure_returns,
write_zip_districts,
)
@@ -105,6 +115,9 @@ COMMITTEES_GAP = "legislators:committees"
BILLS_GAP = "congress:bills"
DISTRICTS_GAP = "census:districts"
HUD_GAP = "hud:hud"
FRED_GAP = "fred:fred"
SOI_GAP = "irs-soi:soi"
REVENUE_GAP = "sec-edgar:revenue"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -2233,6 +2246,301 @@ def tenure(
typer.echo(f"stored: {n} tenure returns")
def _cached_bytes(cl, url: str, path, source: str, pacer, *, ok_404: bool = False):
"""The hoarded body, or a fresh GET that is saved and never overwritten.
``None`` when the source returns 404 and *ok_404* is set.
"""
have = http.newest(path)
if have.is_file():
return have.read_bytes()
response = http.get(cl, url, pacer=pacer, source=source, ok_404=ok_404)
if response.status_code == 404:
return None
where = http.save(path, response.content)
http.manifest(where, str(response.request.url))
return where.read_bytes()
def _cite_file(
store: Any,
*,
title: str,
url: str,
year: int | None,
page_type: str,
website: str,
table: str,
source: str,
) -> str:
"""One bib Download for one source file. Returns its item key."""
from bib.item import Download
from bib.tag import Tag
item = Download(
title=title,
url=url,
file_urls=[url],
year=year,
page_type=page_type,
website_title=website,
)
return store.upsert(
item,
tags=[
Tag.module("market"),
Tag.table(table),
Tag.source(source),
],
)
def _revenue_ciks(con: Any, entity: list[str]) -> list[str]:
"""CIKs to pull. A passed ``--entity`` wins; otherwise the price universe,
skipping index benchmarks."""
if entity:
return [edgar.cik10(key) for key in entity]
seen: list[str] = []
for _ticker, key in tiingo.universe(con, used=set()):
if not str(key).isdigit():
continue
cik = edgar.cik10(key)
if cik not in seen:
seen.append(cik)
return seen
@app.command()
def fred(
series: list[str] = typer.Option(
[], "--series", help="Series id. Repeatable. Default: the eight study series."
),
write: bool = typer.Option(False, "--write", help="Store econ.fred."),
) -> None:
"""Keyless FRED fredgraph CSV. One bib Download per series file.
Clears ``fred:fred``."""
from econ.fred import FREDGRAPH, parse_fredgraph_csv
ids = series or list(FRED_SERIES)
pacer = http.Pacer(0.2)
cl = http.client("fred")
fetched: list[tuple[str, str, tuple]] = []
try:
try:
for series_id in ids:
url = FREDGRAPH.format(series=series_id)
body = _cached_bytes(
cl, url, http.hoard("fred", f"{series_id}.csv"), "fred", pacer
)
try:
parsed = parse_fredgraph_csv(body.decode("utf-8-sig"), series_id)
except ValueError as exc:
raise http.Refused("fred", 200, str(exc)) from None
fetched.append((series_id, url, parsed))
typer.echo(f"{series_id}: {len(parsed)} rows")
except http.Refused as exc:
_gap(FRED_GAP, str(exc), write=write)
return
finally:
cl.close()
if not write:
typer.echo("dry run: pass --write")
return
store = _store()
pending: list[FredRow] = []
for series_id, url, parsed in fetched:
key = _cite_file(
store,
title=f"FRED {series_id}",
url=url,
year=None,
page_type="fredgraph",
website="Federal Reserve Bank of St. Louis",
table="econ.fred",
source="fred",
)
pending.extend(
FredRow(row.series_id, row.year, row.period, row.value, key)
for row in parsed
)
from cms.ingest_log import log_ingest
with _batch() as con:
n = write_fred(con, pending)
clear_gap(con, FRED_GAP)
log_ingest(
con,
module="econ.fred",
table_name="econ.fred",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} fred rows")
@app.command()
def soi(
year: list[int] = typer.Option(
[], "--year", help="Tax year. Repeatable. Default: 2011 through 2023."
),
write: bool = typer.Option(False, "--write", help="Store econ.soi_zip."),
) -> None:
"""IRS SOI ZIP-code income. One bib Download per year file.
Clears ``irs-soi:soi``."""
from econ.irs_soi import SOI_YEARS, SOI_ZIP, parse_soi_zip
years = year or list(SOI_YEARS)
pacer = http.Pacer(0.2)
cl = http.client("irs-soi")
fetched: list[tuple[int, str, tuple]] = []
try:
try:
for tax_year in years:
yy = f"{tax_year % 100:02d}"
url = SOI_ZIP.format(yy=yy)
body = _cached_bytes(
cl,
url,
http.hoard("irs-soi", f"{yy}zpallagi.csv"),
"irs-soi",
pacer,
)
try:
parsed = parse_soi_zip(body.decode("utf-8-sig"), tax_year)
except ValueError as exc:
raise http.Refused("irs-soi", 200, str(exc)) from None
fetched.append((tax_year, url, parsed))
typer.echo(f"{tax_year}: {len(parsed)} zips")
except http.Refused as exc:
_gap(SOI_GAP, str(exc), write=write)
return
finally:
cl.close()
if not write:
typer.echo("dry run: pass --write")
return
store = _store()
pending: list[SoiZipRow] = []
for tax_year, url, parsed in fetched:
key = _cite_file(
store,
title=f"IRS SOI ZIP code income, {tax_year}",
url=url,
year=tax_year,
page_type="zip-income",
website="Internal Revenue Service",
table="econ.soi_zip",
source="irs-soi",
)
pending.extend(
SoiZipRow(
row.year,
row.zip,
row.state_fips,
row.n_returns,
row.agi_k,
key,
)
for row in parsed
)
from cms.ingest_log import log_ingest
with _batch() as con:
n = write_soi_zip(con, pending)
clear_gap(con, SOI_GAP)
log_ingest(
con,
module="econ.irs_soi",
table_name="econ.soi_zip",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} soi rows")
@app.command()
def revenue(
entity: list[str] = typer.Option(
[],
"--entity",
help="Entity key. Repeatable. Default: entities in the price universe.",
),
write: bool = typer.Option(False, "--write", help="Store market.revenue."),
) -> None:
"""Annual 10-K revenue from SEC companyfacts, keyed on the period end.
One bib Download per companyfacts file. Clears ``sec-edgar:revenue``."""
import json
con = _read()
try:
ciks = _revenue_ciks(con, entity)
finally:
con.close()
pacer = edgar.new_pacer()
cl = http.client("sec-edgar")
fetched: list[tuple[str, str, list]] = []
try:
try:
for cik in ciks:
url = edgar.COMPANYFACTS.format(cik=cik)
body = _cached_bytes(
cl,
url,
http.hoard("sec", "companyfacts", f"CIK{cik}.json"),
"sec-edgar",
pacer,
ok_404=True,
)
if body is None:
typer.echo(f"{cik}: not found")
continue
try:
facts = json.loads(body)
except ValueError as exc:
raise http.Refused("sec-edgar", 200, str(exc)) from None
parsed = edgar.revenue_rows(facts, item_key="")
fetched.append((cik, url, parsed))
typer.echo(f"{cik}: {len(parsed)} periods")
except http.Refused as exc:
_gap(REVENUE_GAP, str(exc), write=write)
return
finally:
cl.close()
if not write:
typer.echo("dry run: pass --write")
return
store = _store()
pending: list[RevenueRow] = []
for cik, url, parsed in fetched:
key = _cite_file(
store,
title=f"SEC companyfacts {cik}",
url=url,
year=None,
page_type="companyfacts",
website="U.S. Securities and Exchange Commission",
table="market.revenue",
source="sec-edgar",
)
pending.extend(dataclasses.replace(row, item_key=key) for row in parsed)
from cms.ingest_log import log_ingest
with _batch() as con:
n = write_revenue(con, pending)
clear_gap(con, REVENUE_GAP)
log_ingest(
con,
module="market.edgar",
table_name="market.revenue",
rows=n,
source_file=str(http.hoard(http.MANIFEST)),
)
_publish()
typer.echo(f"stored: {n} revenue rows")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

View File

@@ -2,9 +2,13 @@
from __future__ import annotations
import csv
import io
from collections.abc import Mapping
from dataclasses import dataclass
FREDGRAPH = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series}"
@dataclass(frozen=True)
class SeriesRow:
@@ -43,3 +47,27 @@ def parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]:
)
)
return tuple(rows)
def parse_fredgraph_csv(text: str, series_id: str) -> tuple[SeriesRow, ...]:
"""One row per fredgraph observation.
The header is ``observation_date,<ID>`` or ``DATE,<ID>``. A ``.`` or
blank is missing. Any other header raises ``ValueError`` (FRED returns
an HTML page with status 200 for an unknown id).
"""
reader = csv.reader(io.StringIO(text.lstrip("\ufeff")))
header = next(reader, None)
if (
not header
or len(header) < 2
or header[0].strip() not in {"observation_date", "DATE"}
or header[1].strip() != series_id
):
raise ValueError(f"unexpected header: {header}")
rows: list[SeriesRow] = []
for rec in reader:
period = rec[0].strip()
year = int(period[:4]) if len(period) >= 4 and period[:4].isdigit() else None
rows.append(SeriesRow(series_id, year, period, _value(rec[1])))
return tuple(rows)

View File

@@ -1,4 +1,4 @@
"""IRS SOI county income. The caller supplies the CSV. No download."""
"""IRS SOI income. The caller supplies the CSV. No download."""
from __future__ import annotations
@@ -7,6 +7,10 @@ import io
from dataclasses import dataclass
_COLUMNS = ("fips", "year", "agi")
SOI_ZIP = "https://www.irs.gov/pub/irs-soi/{yy}zpallagi.csv"
#: Tax years whose ``zpallagi.csv`` carries ``N1`` and ``A00100``. 2023 is
#: the newest ZIP file published as of 2026-11-17; 2024 is not up yet.
SOI_YEARS = tuple(range(2011, 2024))
@dataclass(frozen=True)
@@ -40,3 +44,56 @@ def parse_county_income(csv_text: str) -> tuple[CountyIncome, ...]:
)
)
return tuple(rows)
@dataclass(frozen=True)
class ZipIncome:
year: int
zip: str
state_fips: str
n_returns: int | None
agi_k: int | None
def _sum_int(values: list[str]) -> int:
total = 0
for raw in values:
text = raw.strip()
if text in {"", "."}:
continue
total += int(float(text))
return total
def parse_soi_zip(csv_text: str, year: int) -> tuple[ZipIncome, ...]:
"""Sum ``N1`` and ``A00100`` over the AGI stubs of one ZIP.
``agi_stub`` 0 is the total row and is not added again. Zipcodes
``00000`` and ``99999`` are dropped. ZIPs are zero-padded to 5 digits.
"""
reader = csv.DictReader(io.StringIO(csv_text.lstrip("\ufeff")))
if reader.fieldnames:
reader.fieldnames = [name.strip().upper() for name in reader.fieldnames]
buckets: dict[tuple[str, str], list[dict[str, str]]] = {}
for rec in reader:
stub = (rec.get("AGI_STUB") or "").strip()
if stub in {"", "0"}:
continue
zipcode = (rec.get("ZIPCODE") or "").strip().split(".")[0].zfill(5)
if zipcode in {"00000", "99999"}:
continue
state = (rec.get("STATEFIPS") or "").strip().zfill(2)
buckets.setdefault((state, zipcode), []).append(rec)
rows: list[ZipIncome] = []
for state, zipcode in sorted(buckets):
group = buckets[(state, zipcode)]
rows.append(
ZipIncome(
year,
zipcode,
state,
_sum_int([(row.get("N1") or "") for row in group]),
_sum_int([(row.get("A00100") or "") for row in group]),
)
)
return tuple(rows)

View File

@@ -14,7 +14,7 @@ import html as _html
import json
import re
import xml.etree.ElementTree as ET
from collections.abc import Callable, Iterable, Iterator, Sequence
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from pathlib import Path
from typing import Any
@@ -28,6 +28,7 @@ from market.tables import (
HitRow,
LinkRow,
PassageRow,
RevenueRow,
SecurityRow,
)
@@ -38,6 +39,7 @@ SIC_ATOM = (
"&type=10-K&owner=include&count=100&start={start}&output=atom"
)
COMPANY_PAGE = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}"
COMPANYFACTS = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json"
ARCHIVE_INDEX = (
"https://www.sec.gov/Archives/edgar/data/{cik}/{nodash}/{adsh}-index.htm"
)
@@ -991,3 +993,70 @@ def cite_filing(store: Any, hit: HitRow, path: Path) -> str:
)
store.attach_file(key, path, title=path.name)
return key
_REVENUE_CONCEPTS = (
"Revenues",
"RevenueFromContractWithCustomerExcludingAssessedTax",
)
_REVENUE_FORMS = frozenset({"10-K", "10-K/A"})
def _annual_revenue(
fact: Mapping[str, Any],
) -> tuple[dt.date, dt.date, dt.date] | None:
"""``(start, end, filed)`` for an annual 10-K duration, else ``None``.
The span is ``end - start`` in days, 350 through 380. The filing's
``fy`` is never the period.
"""
if fact.get("form") not in _REVENUE_FORMS:
return None
start = _date(str(fact.get("start") or ""))
end = _date(str(fact.get("end") or ""))
filed = _date(str(fact.get("filed") or ""))
if start is None or end is None or filed is None:
return None
if not 350 <= (end - start).days <= 380:
return None
return start, end, filed
def revenue_rows(facts: Mapping[str, Any], *, item_key: str) -> list[RevenueRow]:
"""Annual revenue keyed on the period's ``end`` (D1).
``us-gaap:Revenues`` wins when it has an annual 10-K; otherwise
``RevenueFromContractWithCustomerExcludingAssessedTax``. The latest
``filed`` per ``end`` wins. ``frame`` is kept when the fact has one.
"""
cik = cik10(facts.get("cik") or 0)
gaap = (facts.get("facts") or {}).get("us-gaap") or {}
chosen: list[tuple[str, Mapping[str, Any], dt.date, dt.date, dt.date]] = []
for name in _REVENUE_CONCEPTS:
units = ((gaap.get(name) or {}).get("units") or {}).get("USD") or []
annual = []
for fact in units:
span = _annual_revenue(fact)
if span is not None:
annual.append((name, fact, *span))
if annual:
chosen = annual
break
best: dict[dt.date, RevenueRow] = {}
for name, fact, start, end, filed in chosen:
row = RevenueRow(
cik=cik,
period_start=start,
period_end=end,
frame=str(fact.get("frame") or ""),
form=str(fact.get("form") or ""),
concept=f"us-gaap:{name}",
value=float(fact["val"]),
filed=filed,
adsh=str(fact.get("accn") or ""),
item_key=item_key,
)
old = best.get(end)
if old is None or row.filed >= old.filed:
best[end] = row
return [best[key] for key in sorted(best)]

View File

@@ -138,9 +138,32 @@ CREATE TABLE IF NOT EXISTS market.abnormal_return (
);
CREATE TABLE IF NOT EXISTS market.tenure_return (
key VARCHAR, year INTEGER, on_ruc BOOLEAN, bhar DOUBLE, n_members INTEGER
);
CREATE SCHEMA IF NOT EXISTS econ;
CREATE TABLE IF NOT EXISTS econ.fred (
series_id VARCHAR, year INTEGER, period VARCHAR, value DOUBLE, item_key VARCHAR
);
CREATE TABLE IF NOT EXISTS econ.soi_zip (
year INTEGER, zip VARCHAR, state_fips VARCHAR, n_returns INTEGER,
agi_k INTEGER, item_key VARCHAR
);
CREATE TABLE IF NOT EXISTS market.revenue (
cik VARCHAR, period_start DATE, period_end DATE, frame VARCHAR, form VARCHAR,
concept VARCHAR, value DOUBLE, filed DATE, adsh VARCHAR, item_key VARCHAR
)
"""
FRED_SERIES = (
"CPIAUCSL",
"CPIMEDSL",
"CUSR0000SAM2",
"PCU621111621111",
"PCU621610621610",
"VIXCLS",
"USREC",
"DTB3",
)
def ensure_tables(con: Any) -> None:
for stmt in _DDL.strip().split(";"):
@@ -586,6 +609,45 @@ class TenureRow:
n_members: int
@dataclass(frozen=True)
class FredRow:
"""One FRED observation (``econ.fred``)."""
series_id: str
year: int | None
period: str
value: float | None
item_key: str
@dataclass(frozen=True)
class SoiZipRow:
"""One ZIP-year of IRS SOI returns and AGI (``econ.soi_zip``)."""
year: int
zip: str
state_fips: str
n_returns: int | None
agi_k: int | None
item_key: str
@dataclass(frozen=True)
class RevenueRow:
"""One annual reported revenue (``market.revenue``), keyed on period end."""
cik: str
period_start: date
period_end: date
frame: str
form: str
concept: str
value: float
filed: date
adsh: str
item_key: str
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
a statement per row, far too slow for filing lists)."""
@@ -1279,6 +1341,67 @@ def read_tenure_returns(con: Any, key: str | None = None) -> list[TenureRow]:
return sorted((TenureRow(*r) for r in rows), key=lambda r: (r.key, r.year))
def _replace_keys(con: Any, table: str, column: str, keys: Sequence[Any]) -> None:
con.execute(
f"DELETE FROM {table} WHERE {column} IN ({', '.join('?' * len(keys))})",
list(keys),
)
def write_fred(con: Any, rows: Sequence[FredRow]) -> int:
"""Replace the series ids in *rows*. An empty batch deletes nothing."""
ensure_tables(con)
if not rows:
return 0
_replace_keys(con, "econ.fred", "series_id", sorted({r.series_id for r in rows}))
return _insert(con, "econ.fred", rows)
def read_fred(con: Any, series_id: str | None = None) -> list[FredRow]:
where, params = "", []
if series_id is not None:
where, params = "series_id = ?", [series_id]
rows = _select(con, FredRow, "econ.fred", where, params)
return sorted((FredRow(*r) for r in rows), key=lambda r: (r.series_id, r.period))
def write_soi_zip(con: Any, rows: Sequence[SoiZipRow]) -> int:
"""Replace the years in *rows*. An empty batch deletes nothing."""
ensure_tables(con)
if not rows:
return 0
_replace_keys(con, "econ.soi_zip", "year", sorted({r.year for r in rows}))
return _insert(con, "econ.soi_zip", rows)
def read_soi_zip(con: Any, year: int | None = None) -> list[SoiZipRow]:
where, params = "", []
if year is not None:
where, params = "year = ?", [year]
rows = _select(con, SoiZipRow, "econ.soi_zip", where, params)
return sorted((SoiZipRow(*r) for r in rows), key=lambda r: (r.year, r.zip))
def write_revenue(con: Any, rows: Sequence[RevenueRow]) -> int:
"""Replace the CIKs in *rows*. An empty batch deletes nothing."""
ensure_tables(con)
if not rows:
return 0
_replace_keys(con, "market.revenue", "cik", sorted({r.cik for r in rows}))
return _insert(con, "market.revenue", rows)
def read_revenue(con: Any, cik: str | None = None) -> list[RevenueRow]:
where, params = "", []
if cik is not None:
where, params = "cik = ?", [cik]
rows = _select(con, RevenueRow, "market.revenue", where, params)
return sorted(
(RevenueRow(*r) for r in rows),
key=lambda r: (r.cik, r.period_end, r.concept),
)
# ── districts ────────────────────────────────────────────────────────

View File

@@ -3421,3 +3421,145 @@ def test_tenure_cli_write(env, monkeypatch):
assert any(r.key == "CARDIOLOGY" and r.bhar == 0.2 for r in rows)
assert all(r.key != "ONCOLOGY" for r in rows)
assert published == [True]
_FRED_CSV = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
_SOI_CSV = (
"STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n"
"25,MA,1001,1,10,100\n"
"25,MA,1001,2,20,200\n"
)
def _mock_market_http(monkeypatch, bodies: dict[str, bytes]):
seen: list[httpx.Request] = []
state = {"status": 200}
def handler(request: httpx.Request) -> httpx.Response:
seen.append(request)
if state["status"] != 200:
return httpx.Response(state["status"], text="no")
for needle, body in bodies.items():
if needle in str(request.url):
return httpx.Response(200, content=body)
return httpx.Response(404, text="missing")
def client(source, key=None):
return httpx.Client(transport=httpx.MockTransport(handler))
monkeypatch.setattr(http, "client", client)
return seen, state
def test_fred_cli_offline_write(env, monkeypatch):
c, store, published, hoard = env
T.ensure_tables(c)
T.write_gap(c, "fred:fred", "old")
seen, _state = _mock_market_http(monkeypatch, {"fredgraph.csv": _FRED_CSV.encode()})
res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"])
assert res.exit_code == 0, res.output
assert "stored: 2 fred rows" in res.output
rows = T.read_fred(c, "CPIMEDSL")
assert [row.value for row in rows] == [13.2, None]
assert rows[0].item_key
assert len({row.item_key for row in rows}) == 1
assert T.read_gaps(c) == []
assert published == [True]
assert len(seen) == 1
assert seen[0].url.host == "fred.stlouisfed.org"
urls = [item.url or "" for item in store.list_items()]
assert urls == ["https://fred.stlouisfed.org/graph/fredgraph.csv?id=CPIMEDSL"]
assert rows[0].item_key == store.list_items()[0].key
assert (hoard / "fred" / "CPIMEDSL.csv").is_file()
assert "fredgraph.csv" in (hoard / "MANIFEST.jsonl").read_text()
def test_fred_cli_refusal_prints_gap(env, monkeypatch):
c, _, published, _ = env
T.ensure_tables(c)
_seen, state = _mock_market_http(monkeypatch, {})
state["status"] = 403
res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"])
assert res.exit_code == 0, res.output
assert "gap: fred:fred: fred: HTTP 403" in res.output
assert T.read_fred(c, "CPIMEDSL") == []
assert published == [True]
def test_soi_cli_offline_write(env, monkeypatch):
c, store, published, hoard = env
T.ensure_tables(c)
T.write_gap(c, "irs-soi:soi", "old")
seen, _state = _mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()})
res = runner.invoke(app, ["market", "soi", "--year", "2022", "--write"])
assert res.exit_code == 0, res.output
assert "stored: 1 soi rows" in res.output
rows = T.read_soi_zip(c, 2022)
assert len(rows) == 1
assert rows[0].zip == "01001"
assert rows[0].n_returns == 30
assert rows[0].agi_k == 300
assert rows[0].item_key
assert rows[0].item_key == store.list_items()[0].key
assert T.read_gaps(c) == []
assert published == [True]
assert seen[0].url.host == "www.irs.gov"
assert "22zpallagi.csv" in str(seen[0].url)
assert (hoard / "irs-soi" / "22zpallagi.csv").is_file()
def test_soi_cli_no_write(env, monkeypatch):
c, _, published, _ = env
T.ensure_tables(c)
_mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()})
res = runner.invoke(app, ["market", "soi", "--year", "2022"])
assert res.exit_code == 0, res.output
assert "dry run: pass --write" in res.output
assert "stored:" not in res.output
assert T.read_soi_zip(c, 2022) == []
assert published == []
def test_revenue_cli_offline_write(env, monkeypatch):
c, store, published, hoard = env
T.ensure_tables(c)
T.write_gap(c, "sec-edgar:revenue", "old")
body = (FX / "companyfacts_snippet.json").read_bytes()
seen, _state = _mock_market_http(monkeypatch, {"companyfacts": body})
res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"])
assert res.exit_code == 0, res.output
assert "stored:" in res.output
rows = T.read_revenue(c, "0000001234")
by_end = {row.period_end: row for row in rows}
assert by_end[date(2022, 12, 31)].value == 1050
assert by_end[date(2022, 12, 31)].item_key == store.list_items()[0].key
assert T.read_gaps(c) == []
assert published == [True]
assert seen[0].url.host == "data.sec.gov"
assert "CIK0000001234.json" in str(seen[0].url)
assert (hoard / "sec" / "companyfacts" / "CIK0000001234.json").is_file()
monkeypatch.setattr(
"market.tiingo.universe",
lambda con, **_kwargs: [
("AAA", "0000001234"),
("BBB", "0000001234"),
("SPY", "index:SPY"),
],
)
again = runner.invoke(app, ["market", "revenue", "--write"])
assert again.exit_code == 0, again.output
assert len(seen) == 1
assert T.read_revenue(c, "0000001234")
assert published == [True, True]
def test_revenue_cli_refusal_prints_gap(env, monkeypatch):
c, _, published, _ = env
T.ensure_tables(c)
_seen, state = _mock_market_http(monkeypatch, {})
state["status"] = 403
res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output
assert T.read_revenue(c, "0000001234") == []
assert published == [True]

View File

@@ -35,3 +35,49 @@ def test_blank_agi_is_none():
def test_missing_column_raises():
with pytest.raises(ValueError):
parse_county_income("fips,year\n01001,2022\n")
def test_parse_fredgraph_csv():
from econ.fred import SeriesRow, parse_fredgraph_csv
text = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
assert parse_fredgraph_csv(text, "CPIMEDSL") == (
SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2),
SeriesRow("CPIMEDSL", 2026, "2026-10-01", None),
)
def test_fredgraph_old_header():
from econ.fred import SeriesRow, parse_fredgraph_csv
text = "DATE,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
assert parse_fredgraph_csv(text, "CPIMEDSL") == (
SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2),
SeriesRow("CPIMEDSL", 2026, "2026-10-01", None),
)
def test_fredgraph_html_rejected():
from econ.fred import parse_fredgraph_csv
with pytest.raises(ValueError):
parse_fredgraph_csv("<!DOCTYPE html><html></html>\n", "CPIMEDSL")
def test_soi_zip_sums_agi_stubs():
from econ.irs_soi import ZipIncome, parse_soi_zip
text = (
"STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n"
"25,MA,1001,0,9999,9999\n"
"25,MA,1001,1,10,100\n"
"25,MA,1001,2,20,200\n"
"25,MA,1001,3,30,300\n"
"25,MA,1001,4,40,400\n"
"25,MA,1001,5,50,500\n"
"25,MA,1001,6,60,600\n"
"25,MA,1001,7,,\n"
"25,MA,00000,1,9,9\n"
"25,MA,99999,1,8,8\n"
)
assert parse_soi_zip(text, 2022) == (ZipIncome(2022, "01001", "25", 210, 2100),)

View File

@@ -0,0 +1,104 @@
{
"cik": 1234,
"entityName": "Example Health",
"facts": {
"us-gaap": {
"Revenues": {
"units": {
"USD": [
{
"start": "2022-01-01",
"end": "2022-12-31",
"val": 1050,
"accn": "0000001234-25-000001",
"fy": 2024,
"fp": "FY",
"form": "10-K",
"filed": "2025-02-20",
"frame": "CY2022"
},
{
"end": "2022-12-31",
"val": 1,
"accn": "0000001234-23-000008",
"fy": 2023,
"fp": "FY",
"form": "10-K",
"filed": "2023-02-15"
},
{
"start": "2023-10-01",
"end": "2023-12-31",
"val": 300,
"accn": "0000001234-24-000010",
"fy": 2023,
"fp": "Q4",
"form": "10-K",
"filed": "2024-02-15"
},
{
"start": "2021-01-01",
"end": "2021-12-31",
"val": 777,
"accn": "0000001234-21-000099",
"fy": 2021,
"fp": "FY",
"form": "10-Q",
"filed": "2021-11-01",
"frame": "CY2021"
},
{
"start": "2020-01-01",
"end": "2020-12-31",
"val": 800,
"accn": "0000001234-21-000002",
"fy": 2020,
"fp": "FY",
"form": "10-K/A",
"filed": "2021-06-01"
},
{
"start": "2023-01-01",
"end": "2023-12-31",
"val": 1100,
"accn": "0000001234-24-000001",
"fy": 2023,
"fp": "FY",
"form": "10-K",
"filed": "2024-02-15",
"frame": "CY2023"
},
{
"start": "2022-01-01",
"end": "2022-12-31",
"val": 1000,
"accn": "0000001234-23-000001",
"fy": 2023,
"fp": "FY",
"form": "10-K",
"filed": "2023-02-15",
"frame": "CY2022"
}
]
}
},
"RevenueFromContractWithCustomerExcludingAssessedTax": {
"units": {
"USD": [
{
"start": "2022-01-01",
"end": "2022-12-31",
"val": 1,
"accn": "0000001234-25-000009",
"fy": 2024,
"fp": "FY",
"form": "10-K",
"filed": "2025-02-20",
"frame": "CY2022"
}
]
}
}
}
}
}

View File

@@ -0,0 +1,65 @@
"""EDGAR companyfacts revenue, keyed on the period end (D1)."""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
FX = Path(__file__).parent / "fixtures" / "companyfacts_snippet.json"
def test_companyfacts_revenue_keys_period_end():
from market.edgar import revenue_rows
rows = revenue_rows(json.loads(FX.read_text()), item_key="K")
by_end = {row.period_end: row for row in rows}
comparative = by_end[date(2022, 12, 31)]
assert comparative.period_start == date(2022, 1, 1)
assert comparative.value == 1050
assert comparative.frame == "CY2022"
assert comparative.form == "10-K"
assert comparative.concept == "us-gaap:Revenues"
assert comparative.filed == date(2025, 2, 20)
assert comparative.adsh == "0000001234-25-000001"
assert comparative.cik == "0000001234"
assert comparative.item_key == "K"
assert comparative.period_end.year == 2022
assert by_end[date(2023, 12, 31)].value == 1100
assert by_end[date(2020, 12, 31)].form == "10-K/A"
assert by_end[date(2020, 12, 31)].frame == ""
assert all(row.value not in {1000, 300, 777, 1} for row in rows)
assert all(row.period_end.year != 2024 for row in rows)
fallback = {
"cik": 1234,
"facts": {
"us-gaap": {
"RevenueFromContractWithCustomerExcludingAssessedTax": {
"units": {
"USD": [
{
"start": "2021-01-01",
"end": "2021-12-31",
"val": 900,
"accn": "0000001234-22-000001",
"fy": 2022,
"fp": "FY",
"form": "10-K",
"filed": "2022-02-15",
"frame": "CY2021",
}
]
}
}
}
},
}
alt = revenue_rows(fallback, item_key="K2")
assert len(alt) == 1
assert alt[0].period_end == date(2021, 12, 31)
assert alt[0].value == 900
assert alt[0].concept == (
"us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax"
)
assert alt[0].item_key == "K2"

View File

@@ -58,6 +58,7 @@ def test_ensure_tables_idempotent():
"event",
"abnormal_return",
"tenure_return",
"revenue",
}
@@ -296,6 +297,20 @@ CAR = T.CarRow(
date(2026, 11, 17),
)
TENURE = T.TenureRow("FAMILY PRACTICE", 2024, True, 0.1, 4)
FRED = T.FredRow("CPIMEDSL", 1947, "1947-01-01", 13.2, "FREDKEY")
SOI_ZIP = T.SoiZipRow(2022, "01001", "25", 210, 2100, "SOIKEY")
REVENUE = T.RevenueRow(
"0000001234",
date(2022, 1, 1),
date(2022, 12, 31),
"CY2022",
"10-K",
"us-gaap:Revenues",
1050.0,
date(2025, 2, 20),
"0000001234-25-000001",
"REVKEY",
)
ROUNDTRIP = {
"lda_filing": (
@@ -420,6 +435,21 @@ ROUNDTRIP = {
lambda c, r: T.write_tenure_returns(c, [r]),
lambda c: T.read_tenure_returns(c, "FAMILY PRACTICE"),
),
"fred": (
FRED,
lambda c, r: T.write_fred(c, [r]),
lambda c: T.read_fred(c, "CPIMEDSL"),
),
"soi_zip": (
SOI_ZIP,
lambda c, r: T.write_soi_zip(c, [r]),
lambda c: T.read_soi_zip(c, 2022),
),
"revenue": (
REVENUE,
lambda c, r: T.write_revenue(c, [r]),
lambda c: T.read_revenue(c, "0000001234"),
),
}
@@ -452,12 +482,24 @@ def test_write_empty_is_noop(con):
assert T.write_events(con, []) == 0
assert T.write_cars(con, "family", "HOME-VISITS", "market", []) == 0
assert T.write_tenure_returns(con, []) == 0
assert T.write_fred(con, []) == 0
assert T.write_soi_zip(con, []) == 0
assert T.write_revenue(con, []) == 0
T.write_cars(con, CAR.scope, CAR.target, CAR.model, [CAR])
assert T.write_cars(con, CAR.scope, CAR.target, CAR.model, []) == 0
assert T.read_cars(con, "family", "HOME-VISITS") == [CAR]
T.write_tenure_returns(con, [TENURE])
assert T.write_tenure_returns(con, []) == 0
assert T.read_tenure_returns(con) == [TENURE]
T.write_fred(con, [FRED])
assert T.write_fred(con, []) == 0
assert T.read_fred(con, "CPIMEDSL") == [FRED]
T.write_soi_zip(con, [SOI_ZIP])
assert T.write_soi_zip(con, []) == 0
assert T.read_soi_zip(con, 2022) == [SOI_ZIP]
T.write_revenue(con, [REVENUE])
assert T.write_revenue(con, []) == 0
assert T.read_revenue(con, "0000001234") == [REVENUE]
def test_lda_readers_filter_by_entity(con):