feat(market): FRED (fredgraph), IRS SOI ZIP income and EDGAR revenue by period end, each cited
All checks were successful
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 2m21s
Deploy / changes (push) Successful in 15s
Infra CI / notebooks (push) Successful in 53s
CI / test (push) Successful in 4m33s
Infra CI / zotero (push) Successful in 20s
Infra CI / docs (push) Successful in 25s
Infra CI / bcda (push) Successful in 1m36s
Infra CI / api (push) Successful in 2m14s
Infra CI / mc (push) Successful in 27s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m11s
Deploy / api (push) Successful in 1m37s
Deploy / bcda (push) Successful in 1m52s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m14s
Deploy / report (push) Successful in 15s
All checks were successful
CI / lint (push) Successful in 40s
CI / notebooks-smoke (push) Successful in 2m21s
Deploy / changes (push) Successful in 15s
Infra CI / notebooks (push) Successful in 53s
CI / test (push) Successful in 4m33s
Infra CI / zotero (push) Successful in 20s
Infra CI / docs (push) Successful in 25s
Infra CI / bcda (push) Successful in 1m36s
Infra CI / api (push) Successful in 2m14s
Infra CI / mc (push) Successful in 27s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Infra CI / llm (push) Successful in 1m11s
Deploy / api (push) Successful in 1m37s
Deploy / bcda (push) Successful in 1m52s
Deploy / mc (push) Has been skipped
Deploy / llm (push) Successful in 1m14s
Deploy / report (push) Successful in 15s
This commit is contained in:
@@ -8,6 +8,9 @@ uv run stack market footprint --entities --write
|
||||
uv run stack market footprint --family HOME-VISITS --write
|
||||
uv run stack market district-view --family HOME-VISITS --top 10
|
||||
uv run stack market events --write # releases, meetings, tenures
|
||||
uv run stack market fred --write # FRED fredgraph series
|
||||
uv run stack market soi --write # IRS SOI ZIP income
|
||||
uv run stack market revenue --write # EDGAR annual revenue
|
||||
uv run stack market gaps # sources that could not be read
|
||||
|
||||
Every command fetches first, then opens one short DuckDB batch that only
|
||||
@@ -43,17 +46,21 @@ from market.seed import (
|
||||
validate_seed,
|
||||
)
|
||||
from market.tables import (
|
||||
FRED_SERIES,
|
||||
EntityRow,
|
||||
FecCommitteeRow,
|
||||
FecContributionRow,
|
||||
FecTotalsRow,
|
||||
FilingRow,
|
||||
FredRow,
|
||||
HitRow,
|
||||
LdaFilingRow,
|
||||
LdaIssueRow,
|
||||
LinkRow,
|
||||
PassageRow,
|
||||
RevenueRow,
|
||||
SecurityRow,
|
||||
SoiZipRow,
|
||||
clear_gap,
|
||||
ensure_tables,
|
||||
read_entities,
|
||||
@@ -80,14 +87,17 @@ from market.tables import (
|
||||
write_fec_totals,
|
||||
write_filings,
|
||||
write_footprints,
|
||||
write_fred,
|
||||
write_gap,
|
||||
write_hits,
|
||||
write_links,
|
||||
write_members,
|
||||
write_passages,
|
||||
write_prices,
|
||||
write_revenue,
|
||||
write_securities,
|
||||
write_series,
|
||||
write_soi_zip,
|
||||
write_tenure_returns,
|
||||
write_zip_districts,
|
||||
)
|
||||
@@ -105,6 +115,9 @@ COMMITTEES_GAP = "legislators:committees"
|
||||
BILLS_GAP = "congress:bills"
|
||||
DISTRICTS_GAP = "census:districts"
|
||||
HUD_GAP = "hud:hud"
|
||||
FRED_GAP = "fred:fred"
|
||||
SOI_GAP = "irs-soi:soi"
|
||||
REVENUE_GAP = "sec-edgar:revenue"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -2233,6 +2246,301 @@ def tenure(
|
||||
typer.echo(f"stored: {n} tenure returns")
|
||||
|
||||
|
||||
def _cached_bytes(cl, url: str, path, source: str, pacer, *, ok_404: bool = False):
|
||||
"""The hoarded body, or a fresh GET that is saved and never overwritten.
|
||||
|
||||
``None`` when the source returns 404 and *ok_404* is set.
|
||||
"""
|
||||
have = http.newest(path)
|
||||
if have.is_file():
|
||||
return have.read_bytes()
|
||||
response = http.get(cl, url, pacer=pacer, source=source, ok_404=ok_404)
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
where = http.save(path, response.content)
|
||||
http.manifest(where, str(response.request.url))
|
||||
return where.read_bytes()
|
||||
|
||||
|
||||
def _cite_file(
|
||||
store: Any,
|
||||
*,
|
||||
title: str,
|
||||
url: str,
|
||||
year: int | None,
|
||||
page_type: str,
|
||||
website: str,
|
||||
table: str,
|
||||
source: str,
|
||||
) -> str:
|
||||
"""One bib Download for one source file. Returns its item key."""
|
||||
from bib.item import Download
|
||||
from bib.tag import Tag
|
||||
|
||||
item = Download(
|
||||
title=title,
|
||||
url=url,
|
||||
file_urls=[url],
|
||||
year=year,
|
||||
page_type=page_type,
|
||||
website_title=website,
|
||||
)
|
||||
return store.upsert(
|
||||
item,
|
||||
tags=[
|
||||
Tag.module("market"),
|
||||
Tag.table(table),
|
||||
Tag.source(source),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _revenue_ciks(con: Any, entity: list[str]) -> list[str]:
|
||||
"""CIKs to pull. A passed ``--entity`` wins; otherwise the price universe,
|
||||
skipping index benchmarks."""
|
||||
if entity:
|
||||
return [edgar.cik10(key) for key in entity]
|
||||
seen: list[str] = []
|
||||
for _ticker, key in tiingo.universe(con, used=set()):
|
||||
if not str(key).isdigit():
|
||||
continue
|
||||
cik = edgar.cik10(key)
|
||||
if cik not in seen:
|
||||
seen.append(cik)
|
||||
return seen
|
||||
|
||||
|
||||
@app.command()
|
||||
def fred(
|
||||
series: list[str] = typer.Option(
|
||||
[], "--series", help="Series id. Repeatable. Default: the eight study series."
|
||||
),
|
||||
write: bool = typer.Option(False, "--write", help="Store econ.fred."),
|
||||
) -> None:
|
||||
"""Keyless FRED fredgraph CSV. One bib Download per series file.
|
||||
Clears ``fred:fred``."""
|
||||
from econ.fred import FREDGRAPH, parse_fredgraph_csv
|
||||
|
||||
ids = series or list(FRED_SERIES)
|
||||
pacer = http.Pacer(0.2)
|
||||
cl = http.client("fred")
|
||||
fetched: list[tuple[str, str, tuple]] = []
|
||||
try:
|
||||
try:
|
||||
for series_id in ids:
|
||||
url = FREDGRAPH.format(series=series_id)
|
||||
body = _cached_bytes(
|
||||
cl, url, http.hoard("fred", f"{series_id}.csv"), "fred", pacer
|
||||
)
|
||||
try:
|
||||
parsed = parse_fredgraph_csv(body.decode("utf-8-sig"), series_id)
|
||||
except ValueError as exc:
|
||||
raise http.Refused("fred", 200, str(exc)) from None
|
||||
fetched.append((series_id, url, parsed))
|
||||
typer.echo(f"{series_id}: {len(parsed)} rows")
|
||||
except http.Refused as exc:
|
||||
_gap(FRED_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write")
|
||||
return
|
||||
store = _store()
|
||||
pending: list[FredRow] = []
|
||||
for series_id, url, parsed in fetched:
|
||||
key = _cite_file(
|
||||
store,
|
||||
title=f"FRED {series_id}",
|
||||
url=url,
|
||||
year=None,
|
||||
page_type="fredgraph",
|
||||
website="Federal Reserve Bank of St. Louis",
|
||||
table="econ.fred",
|
||||
source="fred",
|
||||
)
|
||||
pending.extend(
|
||||
FredRow(row.series_id, row.year, row.period, row.value, key)
|
||||
for row in parsed
|
||||
)
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
n = write_fred(con, pending)
|
||||
clear_gap(con, FRED_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="econ.fred",
|
||||
table_name="econ.fred",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} fred rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def soi(
|
||||
year: list[int] = typer.Option(
|
||||
[], "--year", help="Tax year. Repeatable. Default: 2011 through 2023."
|
||||
),
|
||||
write: bool = typer.Option(False, "--write", help="Store econ.soi_zip."),
|
||||
) -> None:
|
||||
"""IRS SOI ZIP-code income. One bib Download per year file.
|
||||
Clears ``irs-soi:soi``."""
|
||||
from econ.irs_soi import SOI_YEARS, SOI_ZIP, parse_soi_zip
|
||||
|
||||
years = year or list(SOI_YEARS)
|
||||
pacer = http.Pacer(0.2)
|
||||
cl = http.client("irs-soi")
|
||||
fetched: list[tuple[int, str, tuple]] = []
|
||||
try:
|
||||
try:
|
||||
for tax_year in years:
|
||||
yy = f"{tax_year % 100:02d}"
|
||||
url = SOI_ZIP.format(yy=yy)
|
||||
body = _cached_bytes(
|
||||
cl,
|
||||
url,
|
||||
http.hoard("irs-soi", f"{yy}zpallagi.csv"),
|
||||
"irs-soi",
|
||||
pacer,
|
||||
)
|
||||
try:
|
||||
parsed = parse_soi_zip(body.decode("utf-8-sig"), tax_year)
|
||||
except ValueError as exc:
|
||||
raise http.Refused("irs-soi", 200, str(exc)) from None
|
||||
fetched.append((tax_year, url, parsed))
|
||||
typer.echo(f"{tax_year}: {len(parsed)} zips")
|
||||
except http.Refused as exc:
|
||||
_gap(SOI_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write")
|
||||
return
|
||||
store = _store()
|
||||
pending: list[SoiZipRow] = []
|
||||
for tax_year, url, parsed in fetched:
|
||||
key = _cite_file(
|
||||
store,
|
||||
title=f"IRS SOI ZIP code income, {tax_year}",
|
||||
url=url,
|
||||
year=tax_year,
|
||||
page_type="zip-income",
|
||||
website="Internal Revenue Service",
|
||||
table="econ.soi_zip",
|
||||
source="irs-soi",
|
||||
)
|
||||
pending.extend(
|
||||
SoiZipRow(
|
||||
row.year,
|
||||
row.zip,
|
||||
row.state_fips,
|
||||
row.n_returns,
|
||||
row.agi_k,
|
||||
key,
|
||||
)
|
||||
for row in parsed
|
||||
)
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
n = write_soi_zip(con, pending)
|
||||
clear_gap(con, SOI_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="econ.irs_soi",
|
||||
table_name="econ.soi_zip",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} soi rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def revenue(
|
||||
entity: list[str] = typer.Option(
|
||||
[],
|
||||
"--entity",
|
||||
help="Entity key. Repeatable. Default: entities in the price universe.",
|
||||
),
|
||||
write: bool = typer.Option(False, "--write", help="Store market.revenue."),
|
||||
) -> None:
|
||||
"""Annual 10-K revenue from SEC companyfacts, keyed on the period end.
|
||||
One bib Download per companyfacts file. Clears ``sec-edgar:revenue``."""
|
||||
import json
|
||||
|
||||
con = _read()
|
||||
try:
|
||||
ciks = _revenue_ciks(con, entity)
|
||||
finally:
|
||||
con.close()
|
||||
pacer = edgar.new_pacer()
|
||||
cl = http.client("sec-edgar")
|
||||
fetched: list[tuple[str, str, list]] = []
|
||||
try:
|
||||
try:
|
||||
for cik in ciks:
|
||||
url = edgar.COMPANYFACTS.format(cik=cik)
|
||||
body = _cached_bytes(
|
||||
cl,
|
||||
url,
|
||||
http.hoard("sec", "companyfacts", f"CIK{cik}.json"),
|
||||
"sec-edgar",
|
||||
pacer,
|
||||
ok_404=True,
|
||||
)
|
||||
if body is None:
|
||||
typer.echo(f"{cik}: not found")
|
||||
continue
|
||||
try:
|
||||
facts = json.loads(body)
|
||||
except ValueError as exc:
|
||||
raise http.Refused("sec-edgar", 200, str(exc)) from None
|
||||
parsed = edgar.revenue_rows(facts, item_key="")
|
||||
fetched.append((cik, url, parsed))
|
||||
typer.echo(f"{cik}: {len(parsed)} periods")
|
||||
except http.Refused as exc:
|
||||
_gap(REVENUE_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write")
|
||||
return
|
||||
store = _store()
|
||||
pending: list[RevenueRow] = []
|
||||
for cik, url, parsed in fetched:
|
||||
key = _cite_file(
|
||||
store,
|
||||
title=f"SEC companyfacts {cik}",
|
||||
url=url,
|
||||
year=None,
|
||||
page_type="companyfacts",
|
||||
website="U.S. Securities and Exchange Commission",
|
||||
table="market.revenue",
|
||||
source="sec-edgar",
|
||||
)
|
||||
pending.extend(dataclasses.replace(row, item_key=key) for row in parsed)
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
with _batch() as con:
|
||||
n = write_revenue(con, pending)
|
||||
clear_gap(con, REVENUE_GAP)
|
||||
log_ingest(
|
||||
con,
|
||||
module="market.edgar",
|
||||
table_name="market.revenue",
|
||||
rows=n,
|
||||
source_file=str(http.hoard(http.MANIFEST)),
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: {n} revenue rows")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
@@ -2,9 +2,13 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
|
||||
FREDGRAPH = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SeriesRow:
|
||||
@@ -43,3 +47,27 @@ def parse_observations(payload: Mapping[str, object]) -> tuple[SeriesRow, ...]:
|
||||
)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
def parse_fredgraph_csv(text: str, series_id: str) -> tuple[SeriesRow, ...]:
|
||||
"""One row per fredgraph observation.
|
||||
|
||||
The header is ``observation_date,<ID>`` or ``DATE,<ID>``. A ``.`` or
|
||||
blank is missing. Any other header raises ``ValueError`` (FRED returns
|
||||
an HTML page with status 200 for an unknown id).
|
||||
"""
|
||||
reader = csv.reader(io.StringIO(text.lstrip("\ufeff")))
|
||||
header = next(reader, None)
|
||||
if (
|
||||
not header
|
||||
or len(header) < 2
|
||||
or header[0].strip() not in {"observation_date", "DATE"}
|
||||
or header[1].strip() != series_id
|
||||
):
|
||||
raise ValueError(f"unexpected header: {header}")
|
||||
rows: list[SeriesRow] = []
|
||||
for rec in reader:
|
||||
period = rec[0].strip()
|
||||
year = int(period[:4]) if len(period) >= 4 and period[:4].isdigit() else None
|
||||
rows.append(SeriesRow(series_id, year, period, _value(rec[1])))
|
||||
return tuple(rows)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""IRS SOI county income. The caller supplies the CSV. No download."""
|
||||
"""IRS SOI income. The caller supplies the CSV. No download."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -7,6 +7,10 @@ import io
|
||||
from dataclasses import dataclass
|
||||
|
||||
_COLUMNS = ("fips", "year", "agi")
|
||||
SOI_ZIP = "https://www.irs.gov/pub/irs-soi/{yy}zpallagi.csv"
|
||||
#: Tax years whose ``zpallagi.csv`` carries ``N1`` and ``A00100``. 2023 is
|
||||
#: the newest ZIP file published as of 2026-11-17; 2024 is not up yet.
|
||||
SOI_YEARS = tuple(range(2011, 2024))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -40,3 +44,56 @@ def parse_county_income(csv_text: str) -> tuple[CountyIncome, ...]:
|
||||
)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ZipIncome:
|
||||
year: int
|
||||
zip: str
|
||||
state_fips: str
|
||||
n_returns: int | None
|
||||
agi_k: int | None
|
||||
|
||||
|
||||
def _sum_int(values: list[str]) -> int:
|
||||
total = 0
|
||||
for raw in values:
|
||||
text = raw.strip()
|
||||
if text in {"", "."}:
|
||||
continue
|
||||
total += int(float(text))
|
||||
return total
|
||||
|
||||
|
||||
def parse_soi_zip(csv_text: str, year: int) -> tuple[ZipIncome, ...]:
|
||||
"""Sum ``N1`` and ``A00100`` over the AGI stubs of one ZIP.
|
||||
|
||||
``agi_stub`` 0 is the total row and is not added again. Zipcodes
|
||||
``00000`` and ``99999`` are dropped. ZIPs are zero-padded to 5 digits.
|
||||
"""
|
||||
reader = csv.DictReader(io.StringIO(csv_text.lstrip("\ufeff")))
|
||||
if reader.fieldnames:
|
||||
reader.fieldnames = [name.strip().upper() for name in reader.fieldnames]
|
||||
buckets: dict[tuple[str, str], list[dict[str, str]]] = {}
|
||||
for rec in reader:
|
||||
stub = (rec.get("AGI_STUB") or "").strip()
|
||||
if stub in {"", "0"}:
|
||||
continue
|
||||
zipcode = (rec.get("ZIPCODE") or "").strip().split(".")[0].zfill(5)
|
||||
if zipcode in {"00000", "99999"}:
|
||||
continue
|
||||
state = (rec.get("STATEFIPS") or "").strip().zfill(2)
|
||||
buckets.setdefault((state, zipcode), []).append(rec)
|
||||
rows: list[ZipIncome] = []
|
||||
for state, zipcode in sorted(buckets):
|
||||
group = buckets[(state, zipcode)]
|
||||
rows.append(
|
||||
ZipIncome(
|
||||
year,
|
||||
zipcode,
|
||||
state,
|
||||
_sum_int([(row.get("N1") or "") for row in group]),
|
||||
_sum_int([(row.get("A00100") or "") for row in group]),
|
||||
)
|
||||
)
|
||||
return tuple(rows)
|
||||
|
||||
@@ -14,7 +14,7 @@ import html as _html
|
||||
import json
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -28,6 +28,7 @@ from market.tables import (
|
||||
HitRow,
|
||||
LinkRow,
|
||||
PassageRow,
|
||||
RevenueRow,
|
||||
SecurityRow,
|
||||
)
|
||||
|
||||
@@ -38,6 +39,7 @@ SIC_ATOM = (
|
||||
"&type=10-K&owner=include&count=100&start={start}&output=atom"
|
||||
)
|
||||
COMPANY_PAGE = "https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK={cik}"
|
||||
COMPANYFACTS = "https://data.sec.gov/api/xbrl/companyfacts/CIK{cik}.json"
|
||||
ARCHIVE_INDEX = (
|
||||
"https://www.sec.gov/Archives/edgar/data/{cik}/{nodash}/{adsh}-index.htm"
|
||||
)
|
||||
@@ -991,3 +993,70 @@ def cite_filing(store: Any, hit: HitRow, path: Path) -> str:
|
||||
)
|
||||
store.attach_file(key, path, title=path.name)
|
||||
return key
|
||||
|
||||
|
||||
_REVENUE_CONCEPTS = (
|
||||
"Revenues",
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax",
|
||||
)
|
||||
_REVENUE_FORMS = frozenset({"10-K", "10-K/A"})
|
||||
|
||||
|
||||
def _annual_revenue(
|
||||
fact: Mapping[str, Any],
|
||||
) -> tuple[dt.date, dt.date, dt.date] | None:
|
||||
"""``(start, end, filed)`` for an annual 10-K duration, else ``None``.
|
||||
|
||||
The span is ``end - start`` in days, 350 through 380. The filing's
|
||||
``fy`` is never the period.
|
||||
"""
|
||||
if fact.get("form") not in _REVENUE_FORMS:
|
||||
return None
|
||||
start = _date(str(fact.get("start") or ""))
|
||||
end = _date(str(fact.get("end") or ""))
|
||||
filed = _date(str(fact.get("filed") or ""))
|
||||
if start is None or end is None or filed is None:
|
||||
return None
|
||||
if not 350 <= (end - start).days <= 380:
|
||||
return None
|
||||
return start, end, filed
|
||||
|
||||
|
||||
def revenue_rows(facts: Mapping[str, Any], *, item_key: str) -> list[RevenueRow]:
|
||||
"""Annual revenue keyed on the period's ``end`` (D1).
|
||||
|
||||
``us-gaap:Revenues`` wins when it has an annual 10-K; otherwise
|
||||
``RevenueFromContractWithCustomerExcludingAssessedTax``. The latest
|
||||
``filed`` per ``end`` wins. ``frame`` is kept when the fact has one.
|
||||
"""
|
||||
cik = cik10(facts.get("cik") or 0)
|
||||
gaap = (facts.get("facts") or {}).get("us-gaap") or {}
|
||||
chosen: list[tuple[str, Mapping[str, Any], dt.date, dt.date, dt.date]] = []
|
||||
for name in _REVENUE_CONCEPTS:
|
||||
units = ((gaap.get(name) or {}).get("units") or {}).get("USD") or []
|
||||
annual = []
|
||||
for fact in units:
|
||||
span = _annual_revenue(fact)
|
||||
if span is not None:
|
||||
annual.append((name, fact, *span))
|
||||
if annual:
|
||||
chosen = annual
|
||||
break
|
||||
best: dict[dt.date, RevenueRow] = {}
|
||||
for name, fact, start, end, filed in chosen:
|
||||
row = RevenueRow(
|
||||
cik=cik,
|
||||
period_start=start,
|
||||
period_end=end,
|
||||
frame=str(fact.get("frame") or ""),
|
||||
form=str(fact.get("form") or ""),
|
||||
concept=f"us-gaap:{name}",
|
||||
value=float(fact["val"]),
|
||||
filed=filed,
|
||||
adsh=str(fact.get("accn") or ""),
|
||||
item_key=item_key,
|
||||
)
|
||||
old = best.get(end)
|
||||
if old is None or row.filed >= old.filed:
|
||||
best[end] = row
|
||||
return [best[key] for key in sorted(best)]
|
||||
|
||||
@@ -138,9 +138,32 @@ CREATE TABLE IF NOT EXISTS market.abnormal_return (
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.tenure_return (
|
||||
key VARCHAR, year INTEGER, on_ruc BOOLEAN, bhar DOUBLE, n_members INTEGER
|
||||
);
|
||||
CREATE SCHEMA IF NOT EXISTS econ;
|
||||
CREATE TABLE IF NOT EXISTS econ.fred (
|
||||
series_id VARCHAR, year INTEGER, period VARCHAR, value DOUBLE, item_key VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS econ.soi_zip (
|
||||
year INTEGER, zip VARCHAR, state_fips VARCHAR, n_returns INTEGER,
|
||||
agi_k INTEGER, item_key VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.revenue (
|
||||
cik VARCHAR, period_start DATE, period_end DATE, frame VARCHAR, form VARCHAR,
|
||||
concept VARCHAR, value DOUBLE, filed DATE, adsh VARCHAR, item_key VARCHAR
|
||||
)
|
||||
"""
|
||||
|
||||
FRED_SERIES = (
|
||||
"CPIAUCSL",
|
||||
"CPIMEDSL",
|
||||
"CUSR0000SAM2",
|
||||
"PCU621111621111",
|
||||
"PCU621610621610",
|
||||
"VIXCLS",
|
||||
"USREC",
|
||||
"DTB3",
|
||||
)
|
||||
|
||||
|
||||
def ensure_tables(con: Any) -> None:
|
||||
for stmt in _DDL.strip().split(";"):
|
||||
@@ -586,6 +609,45 @@ class TenureRow:
|
||||
n_members: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FredRow:
|
||||
"""One FRED observation (``econ.fred``)."""
|
||||
|
||||
series_id: str
|
||||
year: int | None
|
||||
period: str
|
||||
value: float | None
|
||||
item_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SoiZipRow:
|
||||
"""One ZIP-year of IRS SOI returns and AGI (``econ.soi_zip``)."""
|
||||
|
||||
year: int
|
||||
zip: str
|
||||
state_fips: str
|
||||
n_returns: int | None
|
||||
agi_k: int | None
|
||||
item_key: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RevenueRow:
|
||||
"""One annual reported revenue (``market.revenue``), keyed on period end."""
|
||||
|
||||
cik: str
|
||||
period_start: date
|
||||
period_end: date
|
||||
frame: str
|
||||
form: str
|
||||
concept: str
|
||||
value: float
|
||||
filed: date
|
||||
adsh: str
|
||||
item_key: str
|
||||
|
||||
|
||||
def _insert(con: Any, table: str, rows: Sequence[Any], chunk: int = 400) -> int:
|
||||
"""Insert *rows* with multi-row ``VALUES`` statements (``executemany`` is
|
||||
a statement per row, far too slow for filing lists)."""
|
||||
@@ -1279,6 +1341,67 @@ def read_tenure_returns(con: Any, key: str | None = None) -> list[TenureRow]:
|
||||
return sorted((TenureRow(*r) for r in rows), key=lambda r: (r.key, r.year))
|
||||
|
||||
|
||||
def _replace_keys(con: Any, table: str, column: str, keys: Sequence[Any]) -> None:
|
||||
con.execute(
|
||||
f"DELETE FROM {table} WHERE {column} IN ({', '.join('?' * len(keys))})",
|
||||
list(keys),
|
||||
)
|
||||
|
||||
|
||||
def write_fred(con: Any, rows: Sequence[FredRow]) -> int:
|
||||
"""Replace the series ids in *rows*. An empty batch deletes nothing."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_keys(con, "econ.fred", "series_id", sorted({r.series_id for r in rows}))
|
||||
return _insert(con, "econ.fred", rows)
|
||||
|
||||
|
||||
def read_fred(con: Any, series_id: str | None = None) -> list[FredRow]:
|
||||
where, params = "", []
|
||||
if series_id is not None:
|
||||
where, params = "series_id = ?", [series_id]
|
||||
rows = _select(con, FredRow, "econ.fred", where, params)
|
||||
return sorted((FredRow(*r) for r in rows), key=lambda r: (r.series_id, r.period))
|
||||
|
||||
|
||||
def write_soi_zip(con: Any, rows: Sequence[SoiZipRow]) -> int:
|
||||
"""Replace the years in *rows*. An empty batch deletes nothing."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_keys(con, "econ.soi_zip", "year", sorted({r.year for r in rows}))
|
||||
return _insert(con, "econ.soi_zip", rows)
|
||||
|
||||
|
||||
def read_soi_zip(con: Any, year: int | None = None) -> list[SoiZipRow]:
|
||||
where, params = "", []
|
||||
if year is not None:
|
||||
where, params = "year = ?", [year]
|
||||
rows = _select(con, SoiZipRow, "econ.soi_zip", where, params)
|
||||
return sorted((SoiZipRow(*r) for r in rows), key=lambda r: (r.year, r.zip))
|
||||
|
||||
|
||||
def write_revenue(con: Any, rows: Sequence[RevenueRow]) -> int:
|
||||
"""Replace the CIKs in *rows*. An empty batch deletes nothing."""
|
||||
ensure_tables(con)
|
||||
if not rows:
|
||||
return 0
|
||||
_replace_keys(con, "market.revenue", "cik", sorted({r.cik for r in rows}))
|
||||
return _insert(con, "market.revenue", rows)
|
||||
|
||||
|
||||
def read_revenue(con: Any, cik: str | None = None) -> list[RevenueRow]:
|
||||
where, params = "", []
|
||||
if cik is not None:
|
||||
where, params = "cik = ?", [cik]
|
||||
rows = _select(con, RevenueRow, "market.revenue", where, params)
|
||||
return sorted(
|
||||
(RevenueRow(*r) for r in rows),
|
||||
key=lambda r: (r.cik, r.period_end, r.concept),
|
||||
)
|
||||
|
||||
|
||||
# ── districts ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -3421,3 +3421,145 @@ def test_tenure_cli_write(env, monkeypatch):
|
||||
assert any(r.key == "CARDIOLOGY" and r.bhar == 0.2 for r in rows)
|
||||
assert all(r.key != "ONCOLOGY" for r in rows)
|
||||
assert published == [True]
|
||||
|
||||
|
||||
_FRED_CSV = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
|
||||
_SOI_CSV = (
|
||||
"STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n"
|
||||
"25,MA,1001,1,10,100\n"
|
||||
"25,MA,1001,2,20,200\n"
|
||||
)
|
||||
|
||||
|
||||
def _mock_market_http(monkeypatch, bodies: dict[str, bytes]):
|
||||
seen: list[httpx.Request] = []
|
||||
state = {"status": 200}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request)
|
||||
if state["status"] != 200:
|
||||
return httpx.Response(state["status"], text="no")
|
||||
for needle, body in bodies.items():
|
||||
if needle in str(request.url):
|
||||
return httpx.Response(200, content=body)
|
||||
return httpx.Response(404, text="missing")
|
||||
|
||||
def client(source, key=None):
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
monkeypatch.setattr(http, "client", client)
|
||||
return seen, state
|
||||
|
||||
|
||||
def test_fred_cli_offline_write(env, monkeypatch):
|
||||
c, store, published, hoard = env
|
||||
T.ensure_tables(c)
|
||||
T.write_gap(c, "fred:fred", "old")
|
||||
seen, _state = _mock_market_http(monkeypatch, {"fredgraph.csv": _FRED_CSV.encode()})
|
||||
res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored: 2 fred rows" in res.output
|
||||
rows = T.read_fred(c, "CPIMEDSL")
|
||||
assert [row.value for row in rows] == [13.2, None]
|
||||
assert rows[0].item_key
|
||||
assert len({row.item_key for row in rows}) == 1
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert len(seen) == 1
|
||||
assert seen[0].url.host == "fred.stlouisfed.org"
|
||||
urls = [item.url or "" for item in store.list_items()]
|
||||
assert urls == ["https://fred.stlouisfed.org/graph/fredgraph.csv?id=CPIMEDSL"]
|
||||
assert rows[0].item_key == store.list_items()[0].key
|
||||
assert (hoard / "fred" / "CPIMEDSL.csv").is_file()
|
||||
assert "fredgraph.csv" in (hoard / "MANIFEST.jsonl").read_text()
|
||||
|
||||
|
||||
def test_fred_cli_refusal_prints_gap(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
T.ensure_tables(c)
|
||||
_seen, state = _mock_market_http(monkeypatch, {})
|
||||
state["status"] = 403
|
||||
res = runner.invoke(app, ["market", "fred", "--series", "CPIMEDSL", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: fred:fred: fred: HTTP 403" in res.output
|
||||
assert T.read_fred(c, "CPIMEDSL") == []
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_soi_cli_offline_write(env, monkeypatch):
|
||||
c, store, published, hoard = env
|
||||
T.ensure_tables(c)
|
||||
T.write_gap(c, "irs-soi:soi", "old")
|
||||
seen, _state = _mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()})
|
||||
res = runner.invoke(app, ["market", "soi", "--year", "2022", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored: 1 soi rows" in res.output
|
||||
rows = T.read_soi_zip(c, 2022)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].zip == "01001"
|
||||
assert rows[0].n_returns == 30
|
||||
assert rows[0].agi_k == 300
|
||||
assert rows[0].item_key
|
||||
assert rows[0].item_key == store.list_items()[0].key
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert seen[0].url.host == "www.irs.gov"
|
||||
assert "22zpallagi.csv" in str(seen[0].url)
|
||||
assert (hoard / "irs-soi" / "22zpallagi.csv").is_file()
|
||||
|
||||
|
||||
def test_soi_cli_no_write(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
T.ensure_tables(c)
|
||||
_mock_market_http(monkeypatch, {"zpallagi.csv": _SOI_CSV.encode()})
|
||||
res = runner.invoke(app, ["market", "soi", "--year", "2022"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "dry run: pass --write" in res.output
|
||||
assert "stored:" not in res.output
|
||||
assert T.read_soi_zip(c, 2022) == []
|
||||
assert published == []
|
||||
|
||||
|
||||
def test_revenue_cli_offline_write(env, monkeypatch):
|
||||
c, store, published, hoard = env
|
||||
T.ensure_tables(c)
|
||||
T.write_gap(c, "sec-edgar:revenue", "old")
|
||||
body = (FX / "companyfacts_snippet.json").read_bytes()
|
||||
seen, _state = _mock_market_http(monkeypatch, {"companyfacts": body})
|
||||
res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "stored:" in res.output
|
||||
rows = T.read_revenue(c, "0000001234")
|
||||
by_end = {row.period_end: row for row in rows}
|
||||
assert by_end[date(2022, 12, 31)].value == 1050
|
||||
assert by_end[date(2022, 12, 31)].item_key == store.list_items()[0].key
|
||||
assert T.read_gaps(c) == []
|
||||
assert published == [True]
|
||||
assert seen[0].url.host == "data.sec.gov"
|
||||
assert "CIK0000001234.json" in str(seen[0].url)
|
||||
assert (hoard / "sec" / "companyfacts" / "CIK0000001234.json").is_file()
|
||||
monkeypatch.setattr(
|
||||
"market.tiingo.universe",
|
||||
lambda con, **_kwargs: [
|
||||
("AAA", "0000001234"),
|
||||
("BBB", "0000001234"),
|
||||
("SPY", "index:SPY"),
|
||||
],
|
||||
)
|
||||
again = runner.invoke(app, ["market", "revenue", "--write"])
|
||||
assert again.exit_code == 0, again.output
|
||||
assert len(seen) == 1
|
||||
assert T.read_revenue(c, "0000001234")
|
||||
assert published == [True, True]
|
||||
|
||||
|
||||
def test_revenue_cli_refusal_prints_gap(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
T.ensure_tables(c)
|
||||
_seen, state = _mock_market_http(monkeypatch, {})
|
||||
state["status"] = 403
|
||||
res = runner.invoke(app, ["market", "revenue", "--entity", "0000001234", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: sec-edgar:revenue: sec-edgar: HTTP 403" in res.output
|
||||
assert T.read_revenue(c, "0000001234") == []
|
||||
assert published == [True]
|
||||
|
||||
@@ -35,3 +35,49 @@ def test_blank_agi_is_none():
|
||||
def test_missing_column_raises():
|
||||
with pytest.raises(ValueError):
|
||||
parse_county_income("fips,year\n01001,2022\n")
|
||||
|
||||
|
||||
def test_parse_fredgraph_csv():
|
||||
from econ.fred import SeriesRow, parse_fredgraph_csv
|
||||
|
||||
text = "observation_date,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
|
||||
assert parse_fredgraph_csv(text, "CPIMEDSL") == (
|
||||
SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2),
|
||||
SeriesRow("CPIMEDSL", 2026, "2026-10-01", None),
|
||||
)
|
||||
|
||||
|
||||
def test_fredgraph_old_header():
|
||||
from econ.fred import SeriesRow, parse_fredgraph_csv
|
||||
|
||||
text = "DATE,CPIMEDSL\n1947-01-01,13.200\n2026-10-01,.\n"
|
||||
assert parse_fredgraph_csv(text, "CPIMEDSL") == (
|
||||
SeriesRow("CPIMEDSL", 1947, "1947-01-01", 13.2),
|
||||
SeriesRow("CPIMEDSL", 2026, "2026-10-01", None),
|
||||
)
|
||||
|
||||
|
||||
def test_fredgraph_html_rejected():
|
||||
from econ.fred import parse_fredgraph_csv
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
parse_fredgraph_csv("<!DOCTYPE html><html></html>\n", "CPIMEDSL")
|
||||
|
||||
|
||||
def test_soi_zip_sums_agi_stubs():
|
||||
from econ.irs_soi import ZipIncome, parse_soi_zip
|
||||
|
||||
text = (
|
||||
"STATEFIPS,STATE,ZIPCODE,AGI_STUB,N1,A00100\n"
|
||||
"25,MA,1001,0,9999,9999\n"
|
||||
"25,MA,1001,1,10,100\n"
|
||||
"25,MA,1001,2,20,200\n"
|
||||
"25,MA,1001,3,30,300\n"
|
||||
"25,MA,1001,4,40,400\n"
|
||||
"25,MA,1001,5,50,500\n"
|
||||
"25,MA,1001,6,60,600\n"
|
||||
"25,MA,1001,7,,\n"
|
||||
"25,MA,00000,1,9,9\n"
|
||||
"25,MA,99999,1,8,8\n"
|
||||
)
|
||||
assert parse_soi_zip(text, 2022) == (ZipIncome(2022, "01001", "25", 210, 2100),)
|
||||
|
||||
104
tests/market/fixtures/companyfacts_snippet.json
Normal file
104
tests/market/fixtures/companyfacts_snippet.json
Normal file
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"cik": 1234,
|
||||
"entityName": "Example Health",
|
||||
"facts": {
|
||||
"us-gaap": {
|
||||
"Revenues": {
|
||||
"units": {
|
||||
"USD": [
|
||||
{
|
||||
"start": "2022-01-01",
|
||||
"end": "2022-12-31",
|
||||
"val": 1050,
|
||||
"accn": "0000001234-25-000001",
|
||||
"fy": 2024,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2025-02-20",
|
||||
"frame": "CY2022"
|
||||
},
|
||||
{
|
||||
"end": "2022-12-31",
|
||||
"val": 1,
|
||||
"accn": "0000001234-23-000008",
|
||||
"fy": 2023,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2023-02-15"
|
||||
},
|
||||
{
|
||||
"start": "2023-10-01",
|
||||
"end": "2023-12-31",
|
||||
"val": 300,
|
||||
"accn": "0000001234-24-000010",
|
||||
"fy": 2023,
|
||||
"fp": "Q4",
|
||||
"form": "10-K",
|
||||
"filed": "2024-02-15"
|
||||
},
|
||||
{
|
||||
"start": "2021-01-01",
|
||||
"end": "2021-12-31",
|
||||
"val": 777,
|
||||
"accn": "0000001234-21-000099",
|
||||
"fy": 2021,
|
||||
"fp": "FY",
|
||||
"form": "10-Q",
|
||||
"filed": "2021-11-01",
|
||||
"frame": "CY2021"
|
||||
},
|
||||
{
|
||||
"start": "2020-01-01",
|
||||
"end": "2020-12-31",
|
||||
"val": 800,
|
||||
"accn": "0000001234-21-000002",
|
||||
"fy": 2020,
|
||||
"fp": "FY",
|
||||
"form": "10-K/A",
|
||||
"filed": "2021-06-01"
|
||||
},
|
||||
{
|
||||
"start": "2023-01-01",
|
||||
"end": "2023-12-31",
|
||||
"val": 1100,
|
||||
"accn": "0000001234-24-000001",
|
||||
"fy": 2023,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2024-02-15",
|
||||
"frame": "CY2023"
|
||||
},
|
||||
{
|
||||
"start": "2022-01-01",
|
||||
"end": "2022-12-31",
|
||||
"val": 1000,
|
||||
"accn": "0000001234-23-000001",
|
||||
"fy": 2023,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2023-02-15",
|
||||
"frame": "CY2022"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax": {
|
||||
"units": {
|
||||
"USD": [
|
||||
{
|
||||
"start": "2022-01-01",
|
||||
"end": "2022-12-31",
|
||||
"val": 1,
|
||||
"accn": "0000001234-25-000009",
|
||||
"fy": 2024,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2025-02-20",
|
||||
"frame": "CY2022"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
65
tests/market/test_enrich_sources.py
Normal file
65
tests/market/test_enrich_sources.py
Normal file
@@ -0,0 +1,65 @@
|
||||
"""EDGAR companyfacts revenue, keyed on the period end (D1)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
FX = Path(__file__).parent / "fixtures" / "companyfacts_snippet.json"
|
||||
|
||||
|
||||
def test_companyfacts_revenue_keys_period_end():
|
||||
from market.edgar import revenue_rows
|
||||
|
||||
rows = revenue_rows(json.loads(FX.read_text()), item_key="K")
|
||||
by_end = {row.period_end: row for row in rows}
|
||||
comparative = by_end[date(2022, 12, 31)]
|
||||
assert comparative.period_start == date(2022, 1, 1)
|
||||
assert comparative.value == 1050
|
||||
assert comparative.frame == "CY2022"
|
||||
assert comparative.form == "10-K"
|
||||
assert comparative.concept == "us-gaap:Revenues"
|
||||
assert comparative.filed == date(2025, 2, 20)
|
||||
assert comparative.adsh == "0000001234-25-000001"
|
||||
assert comparative.cik == "0000001234"
|
||||
assert comparative.item_key == "K"
|
||||
assert comparative.period_end.year == 2022
|
||||
assert by_end[date(2023, 12, 31)].value == 1100
|
||||
assert by_end[date(2020, 12, 31)].form == "10-K/A"
|
||||
assert by_end[date(2020, 12, 31)].frame == ""
|
||||
assert all(row.value not in {1000, 300, 777, 1} for row in rows)
|
||||
assert all(row.period_end.year != 2024 for row in rows)
|
||||
|
||||
fallback = {
|
||||
"cik": 1234,
|
||||
"facts": {
|
||||
"us-gaap": {
|
||||
"RevenueFromContractWithCustomerExcludingAssessedTax": {
|
||||
"units": {
|
||||
"USD": [
|
||||
{
|
||||
"start": "2021-01-01",
|
||||
"end": "2021-12-31",
|
||||
"val": 900,
|
||||
"accn": "0000001234-22-000001",
|
||||
"fy": 2022,
|
||||
"fp": "FY",
|
||||
"form": "10-K",
|
||||
"filed": "2022-02-15",
|
||||
"frame": "CY2021",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
alt = revenue_rows(fallback, item_key="K2")
|
||||
assert len(alt) == 1
|
||||
assert alt[0].period_end == date(2021, 12, 31)
|
||||
assert alt[0].value == 900
|
||||
assert alt[0].concept == (
|
||||
"us-gaap:RevenueFromContractWithCustomerExcludingAssessedTax"
|
||||
)
|
||||
assert alt[0].item_key == "K2"
|
||||
@@ -58,6 +58,7 @@ def test_ensure_tables_idempotent():
|
||||
"event",
|
||||
"abnormal_return",
|
||||
"tenure_return",
|
||||
"revenue",
|
||||
}
|
||||
|
||||
|
||||
@@ -296,6 +297,20 @@ CAR = T.CarRow(
|
||||
date(2026, 11, 17),
|
||||
)
|
||||
TENURE = T.TenureRow("FAMILY PRACTICE", 2024, True, 0.1, 4)
|
||||
FRED = T.FredRow("CPIMEDSL", 1947, "1947-01-01", 13.2, "FREDKEY")
|
||||
SOI_ZIP = T.SoiZipRow(2022, "01001", "25", 210, 2100, "SOIKEY")
|
||||
REVENUE = T.RevenueRow(
|
||||
"0000001234",
|
||||
date(2022, 1, 1),
|
||||
date(2022, 12, 31),
|
||||
"CY2022",
|
||||
"10-K",
|
||||
"us-gaap:Revenues",
|
||||
1050.0,
|
||||
date(2025, 2, 20),
|
||||
"0000001234-25-000001",
|
||||
"REVKEY",
|
||||
)
|
||||
|
||||
ROUNDTRIP = {
|
||||
"lda_filing": (
|
||||
@@ -420,6 +435,21 @@ ROUNDTRIP = {
|
||||
lambda c, r: T.write_tenure_returns(c, [r]),
|
||||
lambda c: T.read_tenure_returns(c, "FAMILY PRACTICE"),
|
||||
),
|
||||
"fred": (
|
||||
FRED,
|
||||
lambda c, r: T.write_fred(c, [r]),
|
||||
lambda c: T.read_fred(c, "CPIMEDSL"),
|
||||
),
|
||||
"soi_zip": (
|
||||
SOI_ZIP,
|
||||
lambda c, r: T.write_soi_zip(c, [r]),
|
||||
lambda c: T.read_soi_zip(c, 2022),
|
||||
),
|
||||
"revenue": (
|
||||
REVENUE,
|
||||
lambda c, r: T.write_revenue(c, [r]),
|
||||
lambda c: T.read_revenue(c, "0000001234"),
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
@@ -452,12 +482,24 @@ def test_write_empty_is_noop(con):
|
||||
assert T.write_events(con, []) == 0
|
||||
assert T.write_cars(con, "family", "HOME-VISITS", "market", []) == 0
|
||||
assert T.write_tenure_returns(con, []) == 0
|
||||
assert T.write_fred(con, []) == 0
|
||||
assert T.write_soi_zip(con, []) == 0
|
||||
assert T.write_revenue(con, []) == 0
|
||||
T.write_cars(con, CAR.scope, CAR.target, CAR.model, [CAR])
|
||||
assert T.write_cars(con, CAR.scope, CAR.target, CAR.model, []) == 0
|
||||
assert T.read_cars(con, "family", "HOME-VISITS") == [CAR]
|
||||
T.write_tenure_returns(con, [TENURE])
|
||||
assert T.write_tenure_returns(con, []) == 0
|
||||
assert T.read_tenure_returns(con) == [TENURE]
|
||||
T.write_fred(con, [FRED])
|
||||
assert T.write_fred(con, []) == 0
|
||||
assert T.read_fred(con, "CPIMEDSL") == [FRED]
|
||||
T.write_soi_zip(con, [SOI_ZIP])
|
||||
assert T.write_soi_zip(con, []) == 0
|
||||
assert T.read_soi_zip(con, 2022) == [SOI_ZIP]
|
||||
T.write_revenue(con, [REVENUE])
|
||||
assert T.write_revenue(con, []) == 0
|
||||
assert T.read_revenue(con, "0000001234") == [REVENUE]
|
||||
|
||||
|
||||
def test_lda_readers_filter_by_entity(con):
|
||||
|
||||
Reference in New Issue
Block a user