diff --git a/src/cli/market.py b/src/cli/market.py index 85586ca..f3f9d00 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -8,6 +8,9 @@ uv run stack market footprint --entities --write uv run stack market footprint --family HOME-VISITS --write uv run stack market district-view --family HOME-VISITS --top 10 uv run stack market events --write # releases, meetings, tenures +uv run stack market fred --write # FRED fredgraph series +uv run stack market soi --write # IRS SOI ZIP income +uv run stack market revenue --write # EDGAR annual revenue uv run stack market gaps # sources that could not be read Every command fetches first, then opens one short DuckDB batch that only @@ -43,17 +46,21 @@ from market.seed import ( validate_seed, ) from market.tables import ( + FRED_SERIES, EntityRow, FecCommitteeRow, FecContributionRow, FecTotalsRow, FilingRow, + FredRow, HitRow, LdaFilingRow, LdaIssueRow, LinkRow, PassageRow, + RevenueRow, SecurityRow, + SoiZipRow, clear_gap, ensure_tables, read_entities, @@ -80,14 +87,17 @@ from market.tables import ( write_fec_totals, write_filings, write_footprints, + write_fred, write_gap, write_hits, write_links, write_members, write_passages, write_prices, + write_revenue, write_securities, write_series, + write_soi_zip, write_tenure_returns, write_zip_districts, ) @@ -105,6 +115,9 @@ COMMITTEES_GAP = "legislators:committees" BILLS_GAP = "congress:bills" DISTRICTS_GAP = "census:districts" HUD_GAP = "hud:hud" +FRED_GAP = "fred:fred" +SOI_GAP = "irs-soi:soi" +REVENUE_GAP = "sec-edgar:revenue" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -2233,6 +2246,301 @@ def tenure( typer.echo(f"stored: {n} tenure returns") +def _cached_bytes(cl, url: str, path, source: str, pacer, *, ok_404: bool = False): + """The hoarded body, or a fresh GET that is saved and never overwritten. + + ``None`` when the source returns 404 and *ok_404* is set. + """ + have = http.newest(path) + if have.is_file(): + return have.read_bytes() + response = http.get(cl, url, pacer=pacer, source=source, ok_404=ok_404) + if response.status_code == 404: + return None + where = http.save(path, response.content) + http.manifest(where, str(response.request.url)) + return where.read_bytes() + + +def _cite_file( + store: Any, + *, + title: str, + url: str, + year: int | None, + page_type: str, + website: str, + table: str, + source: str, +) -> str: + """One bib Download for one source file. Returns its item key.""" + from bib.item import Download + from bib.tag import Tag + + item = Download( + title=title, + url=url, + file_urls=[url], + year=year, + page_type=page_type, + website_title=website, + ) + return store.upsert( + item, + tags=[ + Tag.module("market"), + Tag.table(table), + Tag.source(source), + ], + ) + + +def _revenue_ciks(con: Any, entity: list[str]) -> list[str]: + """CIKs to pull. A passed ``--entity`` wins; otherwise the price universe, + skipping index benchmarks.""" + if entity: + return [edgar.cik10(key) for key in entity] + seen: list[str] = [] + for _ticker, key in tiingo.universe(con, used=set()): + if not str(key).isdigit(): + continue + cik = edgar.cik10(key) + if cik not in seen: + seen.append(cik) + return seen + + +@app.command() +def fred( + series: list[str] = typer.Option( + [], "--series", help="Series id. Repeatable. Default: the eight study series." + ), + write: bool = typer.Option(False, "--write", help="Store econ.fred."), +) -> None: + """Keyless FRED fredgraph CSV. One bib Download per series file. + Clears ``fred:fred``.""" + from econ.fred import FREDGRAPH, parse_fredgraph_csv + + ids = series or list(FRED_SERIES) + pacer = http.Pacer(0.2) + cl = http.client("fred") + fetched: list[tuple[str, str, tuple]] = [] + try: + try: + for series_id in ids: + url = FREDGRAPH.format(series=series_id) + body = _cached_bytes( + cl, url, http.hoard("fred", f"{series_id}.csv"), "fred", pacer + ) + try: + parsed = parse_fredgraph_csv(body.decode("utf-8-sig"), series_id) + except ValueError as exc: + raise http.Refused("fred", 200, str(exc)) from None + fetched.append((series_id, url, parsed)) + typer.echo(f"{series_id}: {len(parsed)} rows") + except http.Refused as exc: + _gap(FRED_GAP, str(exc), write=write) + return + finally: + cl.close() + if not write: + typer.echo("dry run: pass --write") + return + store = _store() + pending: list[FredRow] = [] + for series_id, url, parsed in fetched: + key = _cite_file( + store, + title=f"FRED {series_id}", + url=url, + year=None, + page_type="fredgraph", + website="Federal Reserve Bank of St. Louis", + table="econ.fred", + source="fred", + ) + pending.extend( + FredRow(row.series_id, row.year, row.period, row.value, key) + for row in parsed + ) + from cms.ingest_log import log_ingest + + with _batch() as con: + n = write_fred(con, pending) + clear_gap(con, FRED_GAP) + log_ingest( + con, + module="econ.fred", + table_name="econ.fred", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} fred rows") + + +@app.command() +def soi( + year: list[int] = typer.Option( + [], "--year", help="Tax year. Repeatable. Default: 2011 through 2023." + ), + write: bool = typer.Option(False, "--write", help="Store econ.soi_zip."), +) -> None: + """IRS SOI ZIP-code income. One bib Download per year file. + Clears ``irs-soi:soi``.""" + from econ.irs_soi import SOI_YEARS, SOI_ZIP, parse_soi_zip + + years = year or list(SOI_YEARS) + pacer = http.Pacer(0.2) + cl = http.client("irs-soi") + fetched: list[tuple[int, str, tuple]] = [] + try: + try: + for tax_year in years: + yy = f"{tax_year % 100:02d}" + url = SOI_ZIP.format(yy=yy) + body = _cached_bytes( + cl, + url, + http.hoard("irs-soi", f"{yy}zpallagi.csv"), + "irs-soi", + pacer, + ) + try: + parsed = parse_soi_zip(body.decode("utf-8-sig"), tax_year) + except ValueError as exc: + raise http.Refused("irs-soi", 200, str(exc)) from None + fetched.append((tax_year, url, parsed)) + typer.echo(f"{tax_year}: {len(parsed)} zips") + except http.Refused as exc: + _gap(SOI_GAP, str(exc), write=write) + return + finally: + cl.close() + if not write: + typer.echo("dry run: pass --write") + return + store = _store() + pending: list[SoiZipRow] = [] + for tax_year, url, parsed in fetched: + key = _cite_file( + store, + title=f"IRS SOI ZIP code income, {tax_year}", + url=url, + year=tax_year, + page_type="zip-income", + website="Internal Revenue Service", + table="econ.soi_zip", + source="irs-soi", + ) + pending.extend( + SoiZipRow( + row.year, + row.zip, + row.state_fips, + row.n_returns, + row.agi_k, + key, + ) + for row in parsed + ) + from cms.ingest_log import log_ingest + + with _batch() as con: + n = write_soi_zip(con, pending) + clear_gap(con, SOI_GAP) + log_ingest( + con, + module="econ.irs_soi", + table_name="econ.soi_zip", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} soi rows") + + +@app.command() +def revenue( + entity: list[str] = typer.Option( + [], + "--entity", + help="Entity key. Repeatable. Default: entities in the price universe.", + ), + write: bool = typer.Option(False, "--write", help="Store market.revenue."), +) -> None: + """Annual 10-K revenue from SEC companyfacts, keyed on the period end. + One bib Download per companyfacts file. Clears ``sec-edgar:revenue``.""" + import json + + con = _read() + try: + ciks = _revenue_ciks(con, entity) + finally: + con.close() + pacer = edgar.new_pacer() + cl = http.client("sec-edgar") + fetched: list[tuple[str, str, list]] = [] + try: + try: + for cik in ciks: + url = edgar.COMPANYFACTS.format(cik=cik) + body = _cached_bytes( + cl, + url, + http.hoard("sec", "companyfacts", f"CIK{cik}.json"), + "sec-edgar", + pacer, + ok_404=True, + ) + if body is None: + typer.echo(f"{cik}: not found") + continue + try: + facts = json.loads(body) + except ValueError as exc: + raise http.Refused("sec-edgar", 200, str(exc)) from None + parsed = edgar.revenue_rows(facts, item_key="") + fetched.append((cik, url, parsed)) + typer.echo(f"{cik}: {len(parsed)} periods") + except http.Refused as exc: + _gap(REVENUE_GAP, str(exc), write=write) + return + finally: + cl.close() + if not write: + typer.echo("dry run: pass --write") + return + store = _store() + pending: list[RevenueRow] = [] + for cik, url, parsed in fetched: + key = _cite_file( + store, + title=f"SEC companyfacts {cik}", + url=url, + year=None, + page_type="companyfacts", + website="U.S. Securities and Exchange Commission", + table="market.revenue", + source="sec-edgar", + ) + pending.extend(dataclasses.replace(row, item_key=key) for row in parsed) + from cms.ingest_log import log_ingest + + with _batch() as con: + n = write_revenue(con, pending) + clear_gap(con, REVENUE_GAP) + log_ingest( + con, + module="market.edgar", + table_name="market.revenue", + rows=n, + source_file=str(http.hoard(http.MANIFEST)), + ) + _publish() + typer.echo(f"stored: {n} revenue rows") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/econ/fred.py b/src/econ/fred.py index 7774836..f02fd1a 100644 --- a/src/econ/fred.py +++ b/src/econ/fred.py @@ -2,9 +2,13 @@ from __future__ import annotations +import csv +import io from collections.abc import Mapping from dataclasses import dataclass +FREDGRAPH = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series}" + @dataclass(frozen=True) class SeriesRow: @@ -43,3 +47,27 @@ def parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]: ) ) return tuple(rows) + + +def parse_fredgraph_csv(text: str, series_id: str) -> tuple[SeriesRow, ...]: + """One row per fredgraph observation. + + The header is ``observation_date,`` or ``DATE,``. A ``.`` or + blank is missing. Any other header raises ``ValueError`` (FRED returns + an HTML page with status 200 for an unknown id). + """ + reader = csv.reader(io.StringIO(text.lstrip("\ufeff"))) + header = next(reader, None) + if ( + not header + or len(header) < 2 + or header[0].strip() not in {"observation_date", "DATE"} + or header[1].strip() != series_id + ): + raise ValueError(f"unexpected header: {header}") + rows: list[SeriesRow] = [] + for rec in reader: + period = rec[0].strip() + year = int(period[:4]) if len(period) >= 4 and period[:4].isdigit() else None + rows.append(SeriesRow(series_id, year, period, _value(rec[1]))) + return tuple(rows) diff --git a/src/econ/irs_soi.py b/src/econ/irs_soi.py index 833e45d..7a9fcab 100644 --- a/src/econ/irs_soi.py +++ b/src/econ/irs_soi.py @@ -1,4 +1,4 @@ -"""IRS SOI county income. The caller supplies the CSV. No download.""" +"""IRS SOI income. The caller supplies the CSV. No download.""" from __future__ import annotations @@ -7,6 +7,10 @@ import io from dataclasses import dataclass _COLUMNS = ("fips", "year", "agi") +SOI_ZIP = "https://www.irs.gov/pub/irs-soi/{yy}zpallagi.csv" +#: Tax years whose ``zpallagi.csv`` carries ``N1`` and ``A00100``. 2023 is +#: the newest ZIP file published as of 2026-11-17; 2024 is not up yet. +SOI_YEARS = tuple(range(2011, 2024)) @dataclass(frozen=True) @@ -40,3 +44,56 @@ def parse_county_income(csv_text: str) -> tuple[CountyIncome, ...]: ) ) return tuple(rows) + + +@dataclass(frozen=True) +class ZipIncome: + year: int + zip: str + state_fips: str + n_returns: int | None + agi_k: int | None + + +def _sum_int(values: list[str]) -> int: + total = 0 + for raw in values: + text = raw.strip() + if text in {"", "."}: + continue + total += int(float(text)) + return total + + +def parse_soi_zip(csv_text: str, year: int) -> tuple[ZipIncome, ...]: + """Sum ``N1`` and ``A00100`` over the AGI stubs of one ZIP. + + ``agi_stub`` 0 is the total row and is not added again. Zipcodes + ``00000`` and ``99999`` are dropped. ZIPs are zero-padded to 5 digits. + """ + reader = csv.DictReader(io.StringIO(csv_text.lstrip("\ufeff"))) + if reader.fieldnames: + reader.fieldnames = [name.strip().upper() for name in reader.fieldnames] + buckets: dict[tuple[str, str], list[dict[str, str]]] = {} + for rec in reader: + stub = (rec.get("AGI_STUB") or "").strip() + if stub in {"", "0"}: + continue + zipcode = (rec.get("ZIPCODE") or "").strip().split(".")[0].zfill(5) + if zipcode in {"00000", "99999"}: + continue + state = (rec.get("STATEFIPS") or "").strip().zfill(2) + buckets.setdefault((state, zipcode), []).append(rec) + rows: list[ZipIncome] = [] + for state, zipcode in sorted(buckets): + group = buckets[(state, zipcode)] + rows.append( + ZipIncome( + year, + zipcode, + state, + _sum_int([(row.get("N1") or "") for row in group]), + _sum_int([(row.get("A00100") or "") for row in group]), + ) + ) + return tuple(rows) diff --git a/src/market/edgar.py b/src/market/edgar.py index 0d04faa..24e952c 100644 --- a/src/market/edgar.py +++ b/src/market/edgar.py @@ -14,7 +14,7 @@ import html as _html import json import re import xml.etree.ElementTree as ET -from collections.abc import Callable, Iterable, Iterator, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from pathlib import Path from typing import Any @@ -28,6 +28,7 @@ from market.tables import ( HitRow, LinkRow, PassageRow, + RevenueRow, SecurityRow, ) @@ -38,6 +39,7 @@ SIC_ATOM = ( "&type=10-K&owner=include&count=100&start={start}&output=atom" ) COMPANY_PAGE = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}" +COMPANYFACTS = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json" ARCHIVE_INDEX = ( "https://www.sec.gov/Archives/edgar/data/{cik}/{nodash}/{adsh}-index.htm" ) @@ -991,3 +993,70 @@ def cite_filing(store: Any, hit: HitRow, path: Path) -> str: ) store.attach_file(key, path, title=path.name) return key + + +_REVENUE_CONCEPTS = ( + "Revenues", + "RevenueFromContractWithCustomerExcludingAssessedTax", +) +_REVENUE_FORMS = frozenset({"10-K", "10-K/A"}) + + +def _annual_revenue( + fact: Mapping[str, Any], +) -> tuple[dt.date, dt.date, dt.date] | None: + """``(start, end, filed)`` for an annual 10-K duration, else ``None``. + + The span is ``end - start`` in days, 350 through 380. The filing's + ``fy`` is never the period. + """ + if fact.get("form") not in _REVENUE_FORMS: + return None + start = _date(str(fact.get("start") or "")) + end = _date(str(fact.get("end") or "")) + filed = _date(str(fact.get("filed") or "")) + if start is None or end is None or filed is None: + return None + if not 350 <= (end - start).days <= 380: + return None + return start, end, filed + + +def revenue_rows(facts: Mapping[str, Any], *, item_key: str) -> list[RevenueRow]: + """Annual revenue keyed on the period's ``end`` (D1). + + ``us-gaap:Revenues`` wins when it has an annual 10-K; otherwise + ``RevenueFromContractWithCustomerExcludingAssessedTax``. The latest + ``filed`` per ``end`` wins. ``frame`` is kept when the fact has one. + """ + cik = cik10(facts.get("cik") or 0) + gaap = (facts.get("facts") or {}).get("us-gaap") or {} + chosen: list[tuple[str, Mapping[str, Any], dt.date, dt.date, dt.date]] = [] + for name in _REVENUE_CONCEPTS: + units = ((gaap.get(name) or {}).get("units") or {}).get("USD") or [] + annual = [] + for fact in units: + span = _annual_revenue(fact) + if span is not None: + annual.append((name, fact, *span)) + if annual: + chosen = annual + break + best: dict[dt.date, RevenueRow] = {} + for name, fact, start, end, filed in chosen: + row = RevenueRow( + cik=cik, + period_start=start, + period_end=end, + frame=str(fact.get("frame") or ""), + form=str(fact.get("form") or ""), + concept=f"us-gaap:{name}", + value=float(fact["val"]), + filed=filed, + adsh=str(fact.get("accn") or ""), + item_key=item_key, + ) + old = best.get(end) + if old is None or row.filed >= old.filed: + best[end] = row + return [best[key] for key in sorted(best)] diff --git a/src/market/tables.py b/src/market/tables.py index 9292f31..16777ff 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -138,9 +138,32 @@ CREATE TABLE IF NOT EXISTS market.abnormal_return ( ); CREATE TABLE IF NOT EXISTS market.tenure_return ( key VARCHAR, year INTEGER, on_ruc BOOLEAN, bhar DOUBLE, n_members INTEGER +); +CREATE SCHEMA IF NOT EXISTS econ; +CREATE TABLE IF NOT EXISTS econ.fred ( + series_id VARCHAR, year INTEGER, period VARCHAR, value DOUBLE, item_key VARCHAR +); +CREATE TABLE IF NOT EXISTS econ.soi_zip ( + year INTEGER, zip VARCHAR, state_fips VARCHAR, n_returns INTEGER, + agi_k INTEGER, item_key VARCHAR +); +CREATE TABLE IF NOT EXISTS market.revenue ( + cik VARCHAR, period_start DATE, period_end DATE, frame VARCHAR, form VARCHAR, + concept VARCHAR, value DOUBLE, filed DATE, adsh VARCHAR, item_key VARCHAR ) """ +FRED_SERIES = ( + "CPIAUCSL", + "CPIMEDSL", + "CUSR0000SAM2", + "PCU621111621111", + "PCU621610621610", + "VIXCLS", + "USREC", + "DTB3", +) + def ensure_tables(con: Any) -> None: for stmt in _DDL.strip().split(";"): @@ -586,6 +609,45 @@ class TenureRow: n_members: int +@dataclass(frozen=True) +class FredRow: + """One FRED observation (``econ.fred``).""" + + series_id: str + year: int | None + period: str + value: float | None + item_key: str + + +@dataclass(frozen=True) +class SoiZipRow: + """One ZIP-year of IRS SOI returns and AGI (``econ.soi_zip``).""" + + year: int + zip: str + state_fips: str + n_returns: int | None + agi_k: int | None + item_key: str + + +@dataclass(frozen=True) +class RevenueRow: + """One annual reported revenue (``market.revenue``), keyed on period end.""" + + cik: str + period_start: date + period_end: date + frame: str + form: str + concept: str + value: float + filed: date + adsh: str + item_key: str + + def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int: """Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is a statement per row, far too slow for filing lists).""" @@ -1279,6 +1341,67 @@ def read_tenure_returns(con: Any, key: str | None = None) -> list[TenureRow]: return sorted((TenureRow(*r) for r in rows), key=lambda r: (r.key, r.year)) +def _replace_keys(con: Any, table: str, column: str, keys: Sequence[Any]) -> None: + con.execute( + f"DELETE FROM {table} WHERE {column} IN ({', '.join('?' * len(keys))})", + list(keys), + ) + + +def write_fred(con: Any, rows: Sequence[FredRow]) -> int: + """Replace the series ids in *rows*. An empty batch deletes nothing.""" + ensure_tables(con) + if not rows: + return 0 + _replace_keys(con, "econ.fred", "series_id", sorted({r.series_id for r in rows})) + return _insert(con, "econ.fred", rows) + + +def read_fred(con: Any, series_id: str | None = None) -> list[FredRow]: + where, params = "", [] + if series_id is not None: + where, params = "series_id = ?", [series_id] + rows = _select(con, FredRow, "econ.fred", where, params) + return sorted((FredRow(*r) for r in rows), key=lambda r: (r.series_id, r.period)) + + +def write_soi_zip(con: Any, rows: Sequence[SoiZipRow]) -> int: + """Replace the years in *rows*. An empty batch deletes nothing.""" + ensure_tables(con) + if not rows: + return 0 + _replace_keys(con, "econ.soi_zip", "year", sorted({r.year for r in rows})) + return _insert(con, "econ.soi_zip", rows) + + +def read_soi_zip(con: Any, year: int | None = None) -> list[SoiZipRow]: + where, params = "", [] + if year is not None: + where, params = "year = ?", [year] + rows = _select(con, SoiZipRow, "econ.soi_zip", where, params) + return sorted((SoiZipRow(*r) for r in rows), key=lambda r: (r.year, r.zip)) + + +def write_revenue(con: Any, rows: Sequence[RevenueRow]) -> int: + """Replace the CIKs in *rows*. An empty batch deletes nothing.""" + ensure_tables(con) + if not rows: + return 0 + _replace_keys(con, "market.revenue", "cik", sorted({r.cik for r in rows})) + return _insert(con, "market.revenue", rows) + + +def read_revenue(con: Any, cik: str | None = None) -> list[RevenueRow]: + where, params = "", [] + if cik is not None: + where, params = "cik = ?", [cik] + rows = _select(con, RevenueRow, "market.revenue", where, params) + return sorted( + (RevenueRow(*r) for r in rows), + key=lambda r: (r.cik, r.period_end, r.concept), + ) + + # ── districts ──────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 3668e16..0f7b223 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -3421,3 +3421,145 @@ def test_tenure_cli_write(env, monkeypatch): assert any(r.key == "CARDIOLOGY" and r.bhar == 0.2 for r in rows) assert all(r.key != "ONCOLOGY" for r in rows) assert published == [True] + + +_FRED_CSV = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n" +_SOI_CSV = ( + "STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n" + "25,MA,1001,1,10,100\n" + "25,MA,1001,2,20,200\n" +) + + +def _mock_market_http(monkeypatch, bodies: dict[str, bytes]): + seen: list[httpx.Request] = [] + state = {"status": 200} + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request) + if state["status"] != 200: + return httpx.Response(state["status"], text="no") + for needle, body in bodies.items(): + if needle in str(request.url): + return httpx.Response(200, content=body) + return httpx.Response(404, text="missing") + + def client(source, key=None): + return httpx.Client(transport=httpx.MockTransport(handler)) + + monkeypatch.setattr(http, "client", client) + return seen, state + + +def test_fred_cli_offline_write(env, monkeypatch): + c, store, published, hoard = env + T.ensure_tables(c) + T.write_gap(c, "fred:fred", "old") + seen, _state = _mock_market_http(monkeypatch, {"fredgraph.csv": _FRED_CSV.encode()}) + res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"]) + assert res.exit_code == 0, res.output + assert "stored: 2 fred rows" in res.output + rows = T.read_fred(c, "CPIMEDSL") + assert [row.value for row in rows] == [13.2, None] + assert rows[0].item_key + assert len({row.item_key for row in rows}) == 1 + assert T.read_gaps(c) == [] + assert published == [True] + assert len(seen) == 1 + assert seen[0].url.host == "fred.stlouisfed.org" + urls = [item.url or "" for item in store.list_items()] + assert urls == ["https://fred.stlouisfed.org/graph/fredgraph.csv?id=CPIMEDSL"] + assert rows[0].item_key == store.list_items()[0].key + assert (hoard / "fred" / "CPIMEDSL.csv").is_file() + assert "fredgraph.csv" in (hoard / "MANIFEST.jsonl").read_text() + + +def test_fred_cli_refusal_prints_gap(env, monkeypatch): + c, _, published, _ = env + T.ensure_tables(c) + _seen, state = _mock_market_http(monkeypatch, {}) + state["status"] = 403 + res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"]) + assert res.exit_code == 0, res.output + assert "gap: fred:fred: fred: HTTP 403" in res.output + assert T.read_fred(c, "CPIMEDSL") == [] + assert published == [True] + + +def test_soi_cli_offline_write(env, monkeypatch): + c, store, published, hoard = env + T.ensure_tables(c) + T.write_gap(c, "irs-soi:soi", "old") + seen, _state = _mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()}) + res = runner.invoke(app, ["market", "soi", "--year", "2022", "--write"]) + assert res.exit_code == 0, res.output + assert "stored: 1 soi rows" in res.output + rows = T.read_soi_zip(c, 2022) + assert len(rows) == 1 + assert rows[0].zip == "01001" + assert rows[0].n_returns == 30 + assert rows[0].agi_k == 300 + assert rows[0].item_key + assert rows[0].item_key == store.list_items()[0].key + assert T.read_gaps(c) == [] + assert published == [True] + assert seen[0].url.host == "www.irs.gov" + assert "22zpallagi.csv" in str(seen[0].url) + assert (hoard / "irs-soi" / "22zpallagi.csv").is_file() + + +def test_soi_cli_no_write(env, monkeypatch): + c, _, published, _ = env + T.ensure_tables(c) + _mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()}) + res = runner.invoke(app, ["market", "soi", "--year", "2022"]) + assert res.exit_code == 0, res.output + assert "dry run: pass --write" in res.output + assert "stored:" not in res.output + assert T.read_soi_zip(c, 2022) == [] + assert published == [] + + +def test_revenue_cli_offline_write(env, monkeypatch): + c, store, published, hoard = env + T.ensure_tables(c) + T.write_gap(c, "sec-edgar:revenue", "old") + body = (FX / "companyfacts_snippet.json").read_bytes() + seen, _state = _mock_market_http(monkeypatch, {"companyfacts": body}) + res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"]) + assert res.exit_code == 0, res.output + assert "stored:" in res.output + rows = T.read_revenue(c, "0000001234") + by_end = {row.period_end: row for row in rows} + assert by_end[date(2022, 12, 31)].value == 1050 + assert by_end[date(2022, 12, 31)].item_key == store.list_items()[0].key + assert T.read_gaps(c) == [] + assert published == [True] + assert seen[0].url.host == "data.sec.gov" + assert "CIK0000001234.json" in str(seen[0].url) + assert (hoard / "sec" / "companyfacts" / "CIK0000001234.json").is_file() + monkeypatch.setattr( + "market.tiingo.universe", + lambda con, **_kwargs: [ + ("AAA", "0000001234"), + ("BBB", "0000001234"), + ("SPY", "index:SPY"), + ], + ) + again = runner.invoke(app, ["market", "revenue", "--write"]) + assert again.exit_code == 0, again.output + assert len(seen) == 1 + assert T.read_revenue(c, "0000001234") + assert published == [True, True] + + +def test_revenue_cli_refusal_prints_gap(env, monkeypatch): + c, _, published, _ = env + T.ensure_tables(c) + _seen, state = _mock_market_http(monkeypatch, {}) + state["status"] = 403 + res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"]) + assert res.exit_code == 0, res.output + assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output + assert T.read_revenue(c, "0000001234") == [] + assert published == [True] diff --git a/tests/econ/test_fred_irs.py b/tests/econ/test_fred_irs.py index f12dea8..78e0aab 100644 --- a/tests/econ/test_fred_irs.py +++ b/tests/econ/test_fred_irs.py @@ -35,3 +35,49 @@ def test_blank_agi_is_none(): def test_missing_column_raises(): with pytest.raises(ValueError): parse_county_income("fips,year\n01001,2022\n") + + +def test_parse_fredgraph_csv(): + from econ.fred import SeriesRow, parse_fredgraph_csv + + text = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n" + assert parse_fredgraph_csv(text, "CPIMEDSL") == ( + SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2), + SeriesRow("CPIMEDSL", 2026, "2026-10-01", None), + ) + + +def test_fredgraph_old_header(): + from econ.fred import SeriesRow, parse_fredgraph_csv + + text = "DATE,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n" + assert parse_fredgraph_csv(text, "CPIMEDSL") == ( + SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2), + SeriesRow("CPIMEDSL", 2026, "2026-10-01", None), + ) + + +def test_fredgraph_html_rejected(): + from econ.fred import parse_fredgraph_csv + + with pytest.raises(ValueError): + parse_fredgraph_csv("\n", "CPIMEDSL") + + +def test_soi_zip_sums_agi_stubs(): + from econ.irs_soi import ZipIncome, parse_soi_zip + + text = ( + "STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n" + "25,MA,1001,0,9999,9999\n" + "25,MA,1001,1,10,100\n" + "25,MA,1001,2,20,200\n" + "25,MA,1001,3,30,300\n" + "25,MA,1001,4,40,400\n" + "25,MA,1001,5,50,500\n" + "25,MA,1001,6,60,600\n" + "25,MA,1001,7,,\n" + "25,MA,00000,1,9,9\n" + "25,MA,99999,1,8,8\n" + ) + assert parse_soi_zip(text, 2022) == (ZipIncome(2022, "01001", "25", 210, 2100),) diff --git a/tests/market/fixtures/companyfacts_snippet.json b/tests/market/fixtures/companyfacts_snippet.json new file mode 100644 index 0000000..ccf819d --- /dev/null +++ b/tests/market/fixtures/companyfacts_snippet.json @@ -0,0 +1,104 @@ +{ + "cik": 1234, + "entityName": "Example Health", + "facts": { + "us-gaap": { + "Revenues": { + "units": { + "USD": [ + { + "start": "2022-01-01", + "end": "2022-12-31", + "val": 1050, + "accn": "0000001234-25-000001", + "fy": 2024, + "fp": "FY", + "form": "10-K", + "filed": "2025-02-20", + "frame": "CY2022" + }, + { + "end": "2022-12-31", + "val": 1, + "accn": "0000001234-23-000008", + "fy": 2023, + "fp": "FY", + "form": "10-K", + "filed": "2023-02-15" + }, + { + "start": "2023-10-01", + "end": "2023-12-31", + "val": 300, + "accn": "0000001234-24-000010", + "fy": 2023, + "fp": "Q4", + "form": "10-K", + "filed": "2024-02-15" + }, + { + "start": "2021-01-01", + "end": "2021-12-31", + "val": 777, + "accn": "0000001234-21-000099", + "fy": 2021, + "fp": "FY", + "form": "10-Q", + "filed": "2021-11-01", + "frame": "CY2021" + }, + { + "start": "2020-01-01", + "end": "2020-12-31", + "val": 800, + "accn": "0000001234-21-000002", + "fy": 2020, + "fp": "FY", + "form": "10-K/A", + "filed": "2021-06-01" + }, + { + "start": "2023-01-01", + "end": "2023-12-31", + "val": 1100, + "accn": "0000001234-24-000001", + "fy": 2023, + "fp": "FY", + "form": "10-K", + "filed": "2024-02-15", + "frame": "CY2023" + }, + { + "start": "2022-01-01", + "end": "2022-12-31", + "val": 1000, + "accn": "0000001234-23-000001", + "fy": 2023, + "fp": "FY", + "form": "10-K", + "filed": "2023-02-15", + "frame": "CY2022" + } + ] + } + }, + "RevenueFromContractWithCustomerExcludingAssessedTax": { + "units": { + "USD": [ + { + "start": "2022-01-01", + "end": "2022-12-31", + "val": 1, + "accn": "0000001234-25-000009", + "fy": 2024, + "fp": "FY", + "form": "10-K", + "filed": "2025-02-20", + "frame": "CY2022" + } + ] + } + } + } + } +} diff --git a/tests/market/test_enrich_sources.py b/tests/market/test_enrich_sources.py new file mode 100644 index 0000000..53b2810 --- /dev/null +++ b/tests/market/test_enrich_sources.py @@ -0,0 +1,65 @@ +"""EDGAR companyfacts revenue, keyed on the period end (D1).""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +FX = Path(__file__).parent / "fixtures" / "companyfacts_snippet.json" + + +def test_companyfacts_revenue_keys_period_end(): + from market.edgar import revenue_rows + + rows = revenue_rows(json.loads(FX.read_text()), item_key="K") + by_end = {row.period_end: row for row in rows} + comparative = by_end[date(2022, 12, 31)] + assert comparative.period_start == date(2022, 1, 1) + assert comparative.value == 1050 + assert comparative.frame == "CY2022" + assert comparative.form == "10-K" + assert comparative.concept == "us-gaap:Revenues" + assert comparative.filed == date(2025, 2, 20) + assert comparative.adsh == "0000001234-25-000001" + assert comparative.cik == "0000001234" + assert comparative.item_key == "K" + assert comparative.period_end.year == 2022 + assert by_end[date(2023, 12, 31)].value == 1100 + assert by_end[date(2020, 12, 31)].form == "10-K/A" + assert by_end[date(2020, 12, 31)].frame == "" + assert all(row.value not in {1000, 300, 777, 1} for row in rows) + assert all(row.period_end.year != 2024 for row in rows) + + fallback = { + "cik": 1234, + "facts": { + "us-gaap": { + "RevenueFromContractWithCustomerExcludingAssessedTax": { + "units": { + "USD": [ + { + "start": "2021-01-01", + "end": "2021-12-31", + "val": 900, + "accn": "0000001234-22-000001", + "fy": 2022, + "fp": "FY", + "form": "10-K", + "filed": "2022-02-15", + "frame": "CY2021", + } + ] + } + } + } + }, + } + alt = revenue_rows(fallback, item_key="K2") + assert len(alt) == 1 + assert alt[0].period_end == date(2021, 12, 31) + assert alt[0].value == 900 + assert alt[0].concept == ( + "us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax" + ) + assert alt[0].item_key == "K2" diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 34fff51..9652b32 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -58,6 +58,7 @@ def test_ensure_tables_idempotent(): "event", "abnormal_return", "tenure_return", + "revenue", } @@ -296,6 +297,20 @@ CAR = T.CarRow( date(2026, 11, 17), ) TENURE = T.TenureRow("FAMILY PRACTICE", 2024, True, 0.1, 4) +FRED = T.FredRow("CPIMEDSL", 1947, "1947-01-01", 13.2, "FREDKEY") +SOI_ZIP = T.SoiZipRow(2022, "01001", "25", 210, 2100, "SOIKEY") +REVENUE = T.RevenueRow( + "0000001234", + date(2022, 1, 1), + date(2022, 12, 31), + "CY2022", + "10-K", + "us-gaap:Revenues", + 1050.0, + date(2025, 2, 20), + "0000001234-25-000001", + "REVKEY", +) ROUNDTRIP = { "lda_filing": ( @@ -420,6 +435,21 @@ ROUNDTRIP = { lambda c, r: T.write_tenure_returns(c, [r]), lambda c: T.read_tenure_returns(c, "FAMILY PRACTICE"), ), + "fred": ( + FRED, + lambda c, r: T.write_fred(c, [r]), + lambda c: T.read_fred(c, "CPIMEDSL"), + ), + "soi_zip": ( + SOI_ZIP, + lambda c, r: T.write_soi_zip(c, [r]), + lambda c: T.read_soi_zip(c, 2022), + ), + "revenue": ( + REVENUE, + lambda c, r: T.write_revenue(c, [r]), + lambda c: T.read_revenue(c, "0000001234"), + ), } @@ -452,12 +482,24 @@ def test_write_empty_is_noop(con): assert T.write_events(con, []) == 0 assert T.write_cars(con, "family", "HOME-VISITS", "market", []) == 0 assert T.write_tenure_returns(con, []) == 0 + assert T.write_fred(con, []) == 0 + assert T.write_soi_zip(con, []) == 0 + assert T.write_revenue(con, []) == 0 T.write_cars(con, CAR.scope, CAR.target, CAR.model, [CAR]) assert T.write_cars(con, CAR.scope, CAR.target, CAR.model, []) == 0 assert T.read_cars(con, "family", "HOME-VISITS") == [CAR] T.write_tenure_returns(con, [TENURE]) assert T.write_tenure_returns(con, []) == 0 assert T.read_tenure_returns(con) == [TENURE] + T.write_fred(con, [FRED]) + assert T.write_fred(con, []) == 0 + assert T.read_fred(con, "CPIMEDSL") == [FRED] + T.write_soi_zip(con, [SOI_ZIP]) + assert T.write_soi_zip(con, []) == 0 + assert T.read_soi_zip(con, 2022) == [SOI_ZIP] + T.write_revenue(con, [REVENUE]) + assert T.write_revenue(con, []) == 0 + assert T.read_revenue(con, "0000001234") == [REVENUE] def test_lda_readers_filter_by_entity(con):