feat(market): EDGAR full-text exposure passages per program and family, cited and AMA-guarded
Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled
This commit is contained in:
@@ -13,9 +13,11 @@ from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import datetime as dt
|
||||
import time
|
||||
from collections import Counter
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import typer
|
||||
|
||||
from market import edgar, http, tiingo
|
||||
@@ -30,18 +32,26 @@ from market.seed import (
|
||||
from market.tables import (
|
||||
EntityRow,
|
||||
FilingRow,
|
||||
HitRow,
|
||||
LinkRow,
|
||||
PassageRow,
|
||||
SecurityRow,
|
||||
clear_gap,
|
||||
ensure_tables,
|
||||
read_entities,
|
||||
read_gaps,
|
||||
read_hits,
|
||||
read_links,
|
||||
read_passages,
|
||||
read_prices,
|
||||
retire_links,
|
||||
retire_organizations,
|
||||
write_entities,
|
||||
write_filings,
|
||||
write_gap,
|
||||
write_hits,
|
||||
write_links,
|
||||
write_passages,
|
||||
write_prices,
|
||||
write_securities,
|
||||
write_series,
|
||||
@@ -52,6 +62,7 @@ app = typer.Typer(no_args_is_help=True)
|
||||
|
||||
ENTITIES_GAP = "sec-edgar:entities"
|
||||
PRICES_GAP = "tiingo:prices"
|
||||
FTS_GAP = "sec-edgar:fts"
|
||||
|
||||
|
||||
# ── indirections the tests monkeypatch ───────────────────────────────
|
||||
@@ -79,6 +90,17 @@ def _publish() -> None:
|
||||
typer.echo(f"replica → {publish_replica('aco')}")
|
||||
|
||||
|
||||
def _asker() -> Any:
|
||||
"""The local JSON extractor on ``cfg.instruct_model`` (Ollama through the
|
||||
host pool; never a cloud model)."""
|
||||
from llm import config as llm_config
|
||||
from llm.pool import HostPool
|
||||
from llm.structured import json_extractor
|
||||
|
||||
cfg = llm_config.load()
|
||||
return json_extractor(cfg, HostPool.from_config(cfg))
|
||||
|
||||
|
||||
# ── gaps ─────────────────────────────────────────────────────────────
|
||||
def _gap(source: str, note: str, *, write: bool) -> None:
|
||||
"""Print ``gap: <source>: <note>``; with *write* record it (and nothing
|
||||
@@ -570,6 +592,434 @@ def prices(
|
||||
typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows")
|
||||
|
||||
|
||||
# ── full-text search ─────────────────────────────────────────────────
|
||||
def _fts_scope(
|
||||
families: list[str], programs: list[str]
|
||||
) -> list[tuple[str, str, tuple[str, ...]]]:
|
||||
"""``(scope, target, terms)`` for the requested programs and families (all
|
||||
programs when none is named)."""
|
||||
out: list[tuple[str, str, tuple[str, ...]]] = []
|
||||
if not families and not programs:
|
||||
programs = list(edgar.PROGRAM_TERMS)
|
||||
for key in programs:
|
||||
if key not in edgar.PROGRAM_TERMS:
|
||||
raise typer.BadParameter(f"unknown program {key!r}")
|
||||
out.append(("program", key, edgar.PROGRAM_TERMS[key]))
|
||||
if families:
|
||||
from pfs.families import FAMILIES, refresh_from
|
||||
|
||||
con = _read()
|
||||
try:
|
||||
refresh_from(con)
|
||||
finally:
|
||||
con.close()
|
||||
for key in families:
|
||||
if key not in FAMILIES:
|
||||
raise typer.BadParameter(f"unknown family {key!r}")
|
||||
out.append(("family", key, edgar.family_terms(FAMILIES[key])))
|
||||
return out
|
||||
|
||||
|
||||
def _ama_index() -> edgar.AmaIndex:
|
||||
"""The 8-word runs of every licensed CPT descriptor, all editions."""
|
||||
con = _read()
|
||||
try:
|
||||
return edgar.cpt_grams(
|
||||
d
|
||||
for (d,) in con.execute("SELECT descriptor FROM pfs.cpt_code").fetchall()
|
||||
if d
|
||||
)
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRow]]:
|
||||
return (
|
||||
_or_empty(read_hits, con_read, []),
|
||||
_or_empty(read_passages, con_read, []),
|
||||
_or_empty(read_entities, con_read, []),
|
||||
)
|
||||
|
||||
|
||||
def _classify(
|
||||
kept: list[PassageRow], limit: int
|
||||
) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]:
|
||||
"""Label *kept* with the local model, resumably (``pfs.runlog``). Returns
|
||||
the ledger outputs by unit, the run counts and the model name."""
|
||||
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
|
||||
|
||||
ask = _asker()
|
||||
model = getattr(ask, "model", "local")
|
||||
version = edgar.EXPOSURE_PROMPT_VERSION
|
||||
log = RunLog(ledger_path("market-exposure", "all"))
|
||||
|
||||
def unit(p: PassageRow) -> Any:
|
||||
def run() -> dict | None:
|
||||
label = edgar.classify_passage(p.passage, p.term, ask)
|
||||
return {"label": label} if label else None
|
||||
|
||||
return run
|
||||
|
||||
units = [
|
||||
(f"{p.passage_id}|{p.term}", input_hash(p.passage, p.term), unit(p))
|
||||
for p in kept
|
||||
]
|
||||
counts = run_units(units, log, model=model, version=version, limit=limit)
|
||||
return log.outputs(model, version), counts, model
|
||||
|
||||
|
||||
def _verified(
|
||||
links: list[LinkRow], passage_rows: dict[Any, PassageRow]
|
||||
) -> tuple[list[LinkRow], int]:
|
||||
"""M12: keep a link only when its passage occurs verbatim in the hoarded
|
||||
filing (re-read from disk) and is at most 400 characters."""
|
||||
by_id = {p.passage_id: p for p in passage_rows.values()}
|
||||
texts: dict[str, str | None] = {}
|
||||
kept: list[LinkRow] = []
|
||||
for lk in links:
|
||||
p = by_id.get(lk.passage_id)
|
||||
if p is None:
|
||||
continue
|
||||
if (p.adsh, p.filename) not in texts:
|
||||
path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename))
|
||||
texts[(p.adsh, p.filename)] = (
|
||||
edgar.read_doc(path) if path.is_file() else None
|
||||
)
|
||||
text = texts[(p.adsh, p.filename)]
|
||||
if text is not None and len(p.passage) <= 400 and p.passage in text:
|
||||
kept.append(lk)
|
||||
return kept, len(links) - len(kept)
|
||||
|
||||
|
||||
def _new_entities(
|
||||
ciks: list[str], seeds: dict[str, Any], issues: list[str]
|
||||
) -> list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]]:
|
||||
"""An entity, its securities and its sweep-form filings for each CIK the
|
||||
registry does not know yet (submissions are hoarded)."""
|
||||
if not ciks:
|
||||
return []
|
||||
out: list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]] = []
|
||||
pacer = edgar.new_pacer()
|
||||
with http.client(edgar.SOURCE) as cl:
|
||||
tickers = edgar.fetch_tickers(cl, pacer=pacer)
|
||||
for cik in ciks:
|
||||
sub = edgar.fetch_submissions(cl, f"CIK{cik}.json", pacer=pacer)
|
||||
if sub is None:
|
||||
issues.append(f"CIK{cik}: no submissions, no entity")
|
||||
continue
|
||||
seed = seeds.get(cik)
|
||||
filings = [
|
||||
f
|
||||
for f in edgar.filings_from_submissions(sub)
|
||||
if f.form in edgar.SWEEP_FORMS
|
||||
]
|
||||
out.append(
|
||||
(
|
||||
edgar.entity_from_submissions(sub, seed),
|
||||
edgar.securities_for(sub, seed, tickers.get(cik, [])),
|
||||
filings,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _gap_note(issues: list[str]) -> str:
|
||||
shown = "; ".join(issues[:15])
|
||||
more = f"; ... {len(issues) - 15} more" if len(issues) > 15 else ""
|
||||
return f"{len(issues)} piece(s) missing: {shown}{more}"
|
||||
|
||||
|
||||
@app.command()
|
||||
def fts(
|
||||
family: list[str] = typer.Option([], "--family", help="A family key (repeatable)."),
|
||||
program: list[str] = typer.Option(
|
||||
[], "--program", help="A program key (repeatable; all when none given)."
|
||||
),
|
||||
forms: list[str] = typer.Option(
|
||||
["10-K"], "--forms", help="Filing form to search (repeatable)."
|
||||
),
|
||||
from_year: int = typer.Option(edgar.FTS_FIRST_YEAR, "--from"),
|
||||
to_year: int = typer.Option(0, "--to", help="Default: this year."),
|
||||
limit: int = typer.Option(
|
||||
0, "--limit", help="At most N new filing downloads and model calls."
|
||||
),
|
||||
max_minutes: float = typer.Option(
|
||||
0.0, "--max-minutes", help="Stop fetching after this long (rerun resumes)."
|
||||
),
|
||||
refresh: bool = typer.Option(False, "--refresh", help="Download again."),
|
||||
dry_run: bool = typer.Option(False, "--dry-run", help="List the queries only."),
|
||||
classify: bool = typer.Option(
|
||||
False, "--classify", help="Label passages with the local model."
|
||||
),
|
||||
write: bool = typer.Option(False, "--write", help="Store hits, passages, links."),
|
||||
) -> None:
|
||||
"""EDGAR full-text search for program and family terms in 10-K filings:
|
||||
hits (market.filing_hit), verbatim passages of at most 400 characters
|
||||
that pass the AMA guard (market.filing_passage) and ``fts`` links
|
||||
(market.entity_link) for health-SIC and seed companies. A link needs a
|
||||
kept passage. Everything fetched is hoarded; a rerun resumes."""
|
||||
to_year = to_year or dt.date.today().year
|
||||
scope = _fts_scope(family, program)
|
||||
owners: dict[str, list[tuple[str, str]]] = {}
|
||||
for sc, target, terms in scope:
|
||||
for t in terms:
|
||||
owners.setdefault(t, []).append((sc, target))
|
||||
queries = [
|
||||
(t, f, y) for t in owners for f in forms for y in range(from_year, to_year + 1)
|
||||
]
|
||||
if dry_run:
|
||||
for sc, target, terms in scope:
|
||||
n = len(terms) * len(forms) * (to_year - from_year + 1)
|
||||
typer.echo(f"{sc} {target}: {len(terms)} terms, {n} queries")
|
||||
cached = sum(edgar.fts_cached(*q) for q in queries)
|
||||
typer.echo(
|
||||
f"{len(queries)} queries ({cached} already hoarded), "
|
||||
f"{edgar.FTS_PAGE} hits a page, at most 10 requests a second"
|
||||
)
|
||||
return
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
def out_of_time() -> bool:
|
||||
return bool(max_minutes) and time.monotonic() - started > max_minutes * 60
|
||||
|
||||
pacer = edgar.new_pacer()
|
||||
prior_hits: list[HitRow] = []
|
||||
prior_passages: list[PassageRow] = []
|
||||
prior_entities: list[EntityRow] = []
|
||||
con = _read()
|
||||
try:
|
||||
prior_hits, prior_passages, prior_entities = _prior(con)
|
||||
finally:
|
||||
con.close()
|
||||
companies, _orgs = load_seed()
|
||||
seeds = {c.cik: c for c in companies}
|
||||
seed_ciks = set(seeds)
|
||||
health = set(edgar.HEALTH_SICS)
|
||||
known = {e.key for e in prior_entities}
|
||||
|
||||
new_hits: dict[tuple[str, str, str, str, str], HitRow] = {}
|
||||
new_passages: list[PassageRow] = []
|
||||
issues: list[str] = []
|
||||
withheld = 0
|
||||
pending = 0
|
||||
incomplete = False
|
||||
cl = http.client(edgar.SOURCE)
|
||||
# a stalled connection should fail in seconds, not minutes; the retry
|
||||
# opens a new one
|
||||
cl.timeout = httpx.Timeout(30.0)
|
||||
try:
|
||||
for term, form, year in queries:
|
||||
if out_of_time():
|
||||
incomplete = True
|
||||
break
|
||||
n = 0
|
||||
for _offset, payload in edgar.fetch_fts(
|
||||
cl, term, form, year, pacer=pacer, refresh=refresh
|
||||
):
|
||||
for sc, target in owners[term]:
|
||||
for h in edgar.parse_hits(
|
||||
payload, term=term, scope=sc, target=target
|
||||
):
|
||||
new_hits[(h.term, h.scope, h.target, h.adsh, h.filename)] = h
|
||||
n += 1
|
||||
typer.echo(f"{term} {form} {year}: {n} hits")
|
||||
|
||||
def eligible(h: HitRow) -> bool:
|
||||
return h.cik in seed_ciks or bool(set(h.sics.split(",")) & health)
|
||||
|
||||
docs: dict[tuple[str, str, str], list[HitRow]] = {}
|
||||
for h in new_hits.values():
|
||||
if eligible(h):
|
||||
docs.setdefault((h.cik, h.adsh, h.filename), []).append(h)
|
||||
ama = _ama_index() if docs else edgar.cpt_grams([])
|
||||
fetched = 0
|
||||
order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k))
|
||||
for cik, adsh, filename in order:
|
||||
hs = docs[(cik, adsh, filename)]
|
||||
if not filename.lower().endswith(edgar.READABLE):
|
||||
issues.append(f"{adsh}:{filename}: format not readable as text")
|
||||
continue
|
||||
path = http.hoard("sec", "docs", cik, adsh, filename)
|
||||
have = http.newest(path).is_file()
|
||||
if not have and ((limit and fetched >= limit) or out_of_time()):
|
||||
pending += 1
|
||||
continue
|
||||
try:
|
||||
got = edgar.fetch_doc(cl, hs[0], pacer=pacer, refresh=refresh)
|
||||
except http.Refused as exc:
|
||||
if exc.status in (0, 500, 502, 503, 504):
|
||||
# one flaky document: skip it, a rerun retries
|
||||
issues.append(f"{adsh}:{filename}: {exc}")
|
||||
continue
|
||||
raise
|
||||
if got is None:
|
||||
issues.append(f"{adsh}:{filename}: document not found")
|
||||
continue
|
||||
fetched += 0 if have else 1
|
||||
text = edgar.read_doc(got)
|
||||
for h in {x.term: x for x in hs}.values():
|
||||
kept, held = edgar.safe_passages(text, h.term, ama)
|
||||
withheld += held
|
||||
for start, quote in kept:
|
||||
new_passages.append(
|
||||
PassageRow(
|
||||
f"{adsh}:{filename}:{start}",
|
||||
cik,
|
||||
adsh,
|
||||
filename,
|
||||
h.term,
|
||||
start,
|
||||
quote,
|
||||
)
|
||||
)
|
||||
if pending:
|
||||
incomplete = True
|
||||
issues.append(f"{pending} filing(s) not downloaded yet (limit or time)")
|
||||
except http.Refused as exc:
|
||||
_gap(FTS_GAP, str(exc), write=write)
|
||||
return
|
||||
finally:
|
||||
cl.close()
|
||||
|
||||
# the union of this run and what earlier runs stored
|
||||
hits = {(h.term, h.scope, h.target, h.adsh, h.filename): h for h in prior_hits}
|
||||
hits.update(new_hits)
|
||||
redone = {(p.adsh, p.filename, p.term) for p in new_passages}
|
||||
prior_by_id = {(p.passage_id, p.term): p for p in prior_passages}
|
||||
merged: dict[tuple[str, str], PassageRow] = {
|
||||
k: p
|
||||
for k, p in prior_by_id.items()
|
||||
if (p.adsh, p.filename, p.term) not in redone
|
||||
}
|
||||
for p in new_passages:
|
||||
old = prior_by_id.get((p.passage_id, p.term))
|
||||
if old and old.passage == p.passage:
|
||||
p = dataclasses.replace(
|
||||
p,
|
||||
label=old.label,
|
||||
label_version=old.label_version,
|
||||
item_key=old.item_key,
|
||||
)
|
||||
merged[(p.passage_id, p.term)] = p
|
||||
|
||||
if classify:
|
||||
filed = {(h.adsh, h.filename): h.file_date for h in hits.values()}
|
||||
todo = sorted(
|
||||
(p for p in merged.values() if p.term in owners),
|
||||
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
|
||||
)
|
||||
outputs, counts, model = _classify(todo, limit)
|
||||
typer.echo(f"classified with {model}: {counts}")
|
||||
for k, p in list(merged.items()):
|
||||
got = outputs.get(f"{p.passage_id}|{p.term}")
|
||||
if got:
|
||||
merged[k] = dataclasses.replace(
|
||||
p,
|
||||
label=got["label"],
|
||||
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
|
||||
)
|
||||
entities = list(prior_entities)
|
||||
all_hits = list(hits.values())
|
||||
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
|
||||
linkable = [
|
||||
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
|
||||
]
|
||||
links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
|
||||
mine = {(sc, t) for sc, t, _ in scope}
|
||||
links = [lk for lk in links if (lk.scope, lk.target) in mine]
|
||||
links, unverified = _verified(links, merged)
|
||||
if unverified:
|
||||
issues.append(f"{unverified} link(s) dropped: quote not found in the hoard")
|
||||
try:
|
||||
new_entities = _new_entities(
|
||||
sorted({lk.entity_key for lk in links} - known), seeds, issues
|
||||
)
|
||||
except http.Refused as exc:
|
||||
_gap(FTS_GAP, str(exc), write=write)
|
||||
return
|
||||
have = known | {e.key for e, _, _ in new_entities}
|
||||
links = [lk for lk in links if lk.entity_key in have]
|
||||
by_target = Counter(lk.target for lk in links)
|
||||
typer.echo(
|
||||
f"hits {len(new_hits)} (stored {len(all_hits)}); passages kept "
|
||||
f"{len(new_passages)}, withheld by the AMA guard {withheld}; new entities "
|
||||
f"{len(new_entities)}; fts links {len(links)} "
|
||||
f"({', '.join(f'{k} {n}' for k, n in sorted(by_target.items())) or 'none'})"
|
||||
)
|
||||
for i in issues[:20]:
|
||||
typer.echo(f"gap: {FTS_GAP}: {i}")
|
||||
if len(issues) > 20:
|
||||
typer.echo(f"gap: {FTS_GAP}: ... {len(issues) - 20} more")
|
||||
if not write:
|
||||
typer.echo("dry run: pass --write to store")
|
||||
return
|
||||
if incomplete:
|
||||
typer.echo("incomplete (limit or time): nothing written; rerun to resume")
|
||||
return
|
||||
|
||||
store = _store()
|
||||
docs_of = {(h.adsh, h.filename): h for h in all_hits}
|
||||
item_of: dict[tuple[str, str], str] = {}
|
||||
for lk in links:
|
||||
adsh, _, filename = lk.cite_ref.partition(":")
|
||||
if (adsh, filename) not in item_of:
|
||||
h = docs_of[(adsh, filename)]
|
||||
item_of[(adsh, filename)] = edgar.cite_filing(
|
||||
store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename))
|
||||
)
|
||||
links = [
|
||||
dataclasses.replace(
|
||||
lk,
|
||||
item_key=item_of[tuple(lk.cite_ref.split(":", 1))], # type: ignore[arg-type]
|
||||
)
|
||||
for lk in links
|
||||
]
|
||||
stamped = [
|
||||
dataclasses.replace(p, item_key=item_of[(p.adsh, p.filename)])
|
||||
if (p.adsh, p.filename) in item_of and not p.item_key
|
||||
else p
|
||||
for p in merged.values()
|
||||
]
|
||||
|
||||
from cms.ingest_log import log_ingest
|
||||
|
||||
sic_rows = [r for e, _, _ in new_entities for r in edgar.sic_links(e)]
|
||||
with _batch() as con:
|
||||
ensure_tables(con)
|
||||
write_entities(con, [e for e, _, _ in new_entities])
|
||||
for e, secs, filings in new_entities:
|
||||
write_securities(con, e.key, secs)
|
||||
write_filings(con, e.cik, filings)
|
||||
write_links(con, sic_rows)
|
||||
write_hits(con, list(new_hits.values()))
|
||||
write_passages(con, stamped)
|
||||
write_links(con, links)
|
||||
stored = {(lk.entity_key, lk.scope, lk.target) for lk in links}
|
||||
old_fts = [lk for lk in read_links(con, basis="fts")]
|
||||
keep = stored | {
|
||||
(lk.entity_key, lk.scope, lk.target)
|
||||
for lk in old_fts
|
||||
if (lk.scope, lk.target) not in mine
|
||||
}
|
||||
retired = retire_links(con, "fts", {lk.entity_key for lk in old_fts}, keep)
|
||||
if issues:
|
||||
write_gap(con, FTS_GAP, _gap_note(issues))
|
||||
else:
|
||||
clear_gap(con, FTS_GAP)
|
||||
src = str(http.hoard(http.MANIFEST))
|
||||
for table, n in (
|
||||
("market.filing_hit", len(new_hits)),
|
||||
("market.filing_passage", len(stamped)),
|
||||
("market.entity_link", len(links) + len(sic_rows)),
|
||||
):
|
||||
log_ingest(
|
||||
con, module="market.edgar", table_name=table, rows=n, source_file=src
|
||||
)
|
||||
_publish()
|
||||
typer.echo(f"stored: fts links {len(links)} (retired {retired})")
|
||||
|
||||
|
||||
@app.command()
|
||||
def gaps() -> None:
|
||||
"""Sources that could not be read (market.source_gap)."""
|
||||
|
||||
@@ -22,7 +22,14 @@ import httpx
|
||||
|
||||
from market import http
|
||||
from market.seed import Cite, SeedCompany, SeedOrg, norm_org
|
||||
from market.tables import EntityRow, FilingRow, LinkRow, SecurityRow
|
||||
from market.tables import (
|
||||
EntityRow,
|
||||
FilingRow,
|
||||
HitRow,
|
||||
LinkRow,
|
||||
PassageRow,
|
||||
SecurityRow,
|
||||
)
|
||||
|
||||
TICKERS = "https://www.sec.gov/files/company_tickers.json"
|
||||
SUBMISSIONS = "https://data.sec.gov/submissions/{name}"
|
||||
@@ -325,21 +332,23 @@ def _fetch(
|
||||
refresh: bool,
|
||||
offline: bool,
|
||||
ok_404: bool = False,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> bytes | None:
|
||||
"""The newest hoarded copy of *path*, downloading when there is none (or
|
||||
on *refresh*) and not *offline*. A changed body lands beside the old; a
|
||||
tolerated 404 is ``None`` and is not stored."""
|
||||
have = http.newest(path)
|
||||
shown = str(httpx.URL(url, params=params)) if params else url
|
||||
if have.is_file() and (offline or not refresh):
|
||||
http.manifest(have, url)
|
||||
http.manifest(have, shown)
|
||||
return have.read_bytes()
|
||||
if offline or cl is None:
|
||||
return None
|
||||
r = http.get(cl, url, pacer=pacer, source=SOURCE, ok_404=ok_404)
|
||||
r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=ok_404)
|
||||
if r.status_code == 404:
|
||||
return None
|
||||
saved = http.save(path, r.content)
|
||||
http.manifest(saved, url)
|
||||
http.manifest(saved, shown)
|
||||
return saved.read_bytes()
|
||||
|
||||
|
||||
@@ -535,3 +544,358 @@ def shares_ama_run(
|
||||
else AmaIndex.of(descriptors, n)
|
||||
)
|
||||
return not grams.isdisjoint(index)
|
||||
|
||||
|
||||
# ── full-text search ─────────────────────────────────────────────────
|
||||
|
||||
EFTS = "https://efts.sec.gov/LATEST/search-index"
|
||||
FTS_PAGE = 100 # hits per page; ``from`` is the offset
|
||||
FTS_FIRST_YEAR = 2001 # EDGAR full-text search starts in 2001
|
||||
|
||||
PROGRAM_TERMS: dict[str, tuple[str, ...]] = {
|
||||
"PFS": (
|
||||
"Medicare Physician Fee Schedule",
|
||||
"physician fee schedule",
|
||||
"RBRVS",
|
||||
"Relative Value Scale Update Committee",
|
||||
"CPT Editorial Panel",
|
||||
),
|
||||
"QPP-MIPS": ("Merit-based Incentive Payment System", "Quality Payment Program"),
|
||||
"MSSP": ("Medicare Shared Savings Program",),
|
||||
"APM": ("Advanced Alternative Payment Model",),
|
||||
"TELEHEALTH": ("Medicare telehealth",),
|
||||
"HOME-BASED-CARE": ("home-based primary care", "home-based care"),
|
||||
"CARE-MANAGEMENT": ("chronic care management", "transitional care management"),
|
||||
}
|
||||
|
||||
|
||||
def fts_params(term: str, form: str, year: int, offset: int) -> dict[str, Any]:
|
||||
"""One query per (term, form, year) keeps each under the 10,000-hit cap."""
|
||||
return {
|
||||
"q": f'"{term}"',
|
||||
"forms": form,
|
||||
"dateRange": "custom",
|
||||
"startdt": f"{year}-01-01",
|
||||
"enddt": f"{year}-12-31",
|
||||
"from": offset,
|
||||
}
|
||||
|
||||
|
||||
def family_terms(fam: Any) -> tuple[str, ...]:
|
||||
"""The family name plus every multi-word synonym (single words are too
|
||||
common to search for)."""
|
||||
out: dict[str, str] = {}
|
||||
for t in (fam.name, *(s for s in fam.synonyms if len(s.split()) > 1)):
|
||||
out.setdefault(t.lower(), t)
|
||||
return tuple(out.values())
|
||||
|
||||
|
||||
def parse_hits(
|
||||
payload: dict[str, Any], *, term: str, scope: str, target: str
|
||||
) -> list[HitRow]:
|
||||
"""The hits of one search page; ``_id`` is ``"{adsh}:{filename}"``."""
|
||||
rows = []
|
||||
for hit in payload.get("hits", {}).get("hits", []):
|
||||
src = hit["_source"]
|
||||
adsh, _, filename = hit["_id"].partition(":")
|
||||
names = src.get("display_names") or [""]
|
||||
rows.append(
|
||||
HitRow(
|
||||
term=term,
|
||||
scope=scope,
|
||||
target=target,
|
||||
cik=cik10(src["ciks"][0]),
|
||||
adsh=adsh,
|
||||
filename=filename,
|
||||
form=src.get("form") or "",
|
||||
file_date=dt.date.fromisoformat(src["file_date"]),
|
||||
period_ending=_date(src.get("period_ending") or ""),
|
||||
sics=",".join(src.get("sics") or []),
|
||||
display_name=names[0],
|
||||
)
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def doc_url(cik: str, adsh: str, filename: str) -> str:
|
||||
return (
|
||||
f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/"
|
||||
f"{adsh.replace('-', '')}/{filename}"
|
||||
)
|
||||
|
||||
|
||||
def fts_cache(term: str, form: str, year: int, offset: int) -> Path:
|
||||
return http.hoard(
|
||||
"sec", "fts", http.slug(term), form.replace("/", "_"), f"{year}.{offset}.json"
|
||||
)
|
||||
|
||||
|
||||
def fts_cached(term: str, form: str, year: int) -> bool:
|
||||
return http.newest(fts_cache(term, form, year, 0)).is_file()
|
||||
|
||||
|
||||
def fetch_fts(
|
||||
cl: httpx.Client | None,
|
||||
term: str,
|
||||
form: str,
|
||||
year: int,
|
||||
*,
|
||||
pacer: http.Pacer,
|
||||
refresh: bool = False,
|
||||
offline: bool = False,
|
||||
) -> Iterator[tuple[int, dict[str, Any]]]:
|
||||
"""Every page ``(offset, payload)`` of one (term, form, year) query,
|
||||
hoarded at ``data/market/sec/fts/<slug(term)>/<form>/<year>.<offset>.json``
|
||||
and read from the hoard on a rerun."""
|
||||
offset = 0
|
||||
while True:
|
||||
body = _fetch(
|
||||
cl,
|
||||
EFTS,
|
||||
fts_cache(term, form, year, offset),
|
||||
pacer=pacer,
|
||||
refresh=refresh,
|
||||
offline=offline,
|
||||
params=fts_params(term, form, year, offset),
|
||||
)
|
||||
if body is None:
|
||||
return
|
||||
payload = json.loads(body)
|
||||
yield offset, payload
|
||||
offset += FTS_PAGE
|
||||
if offset >= payload.get("hits", {}).get("total", {}).get("value", 0):
|
||||
return
|
||||
|
||||
|
||||
#: documents we can read as text
|
||||
READABLE = (".htm", ".html", ".txt")
|
||||
|
||||
|
||||
def fetch_doc(
|
||||
cl: httpx.Client | None,
|
||||
hit: HitRow,
|
||||
*,
|
||||
pacer: http.Pacer,
|
||||
refresh: bool = False,
|
||||
offline: bool = False,
|
||||
) -> Path | None:
|
||||
"""The hoarded copy of the hit's document (``sec/docs/<cik>/<adsh>/<file>``),
|
||||
downloaded when absent and not *offline*; ``None`` when unavailable."""
|
||||
path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename)
|
||||
body = _fetch(
|
||||
cl,
|
||||
doc_url(hit.cik, hit.adsh, hit.filename),
|
||||
path,
|
||||
pacer=pacer,
|
||||
refresh=refresh,
|
||||
offline=offline,
|
||||
ok_404=True,
|
||||
)
|
||||
return http.newest(path) if body is not None else None
|
||||
|
||||
|
||||
def read_doc(path: Path) -> str:
|
||||
return normalize_text(path.read_bytes().decode("utf-8", errors="replace"))
|
||||
|
||||
|
||||
# ── passages ─────────────────────────────────────────────────────────
|
||||
|
||||
_SENTENCE_END = (". ", "? ", "! ")
|
||||
|
||||
|
||||
def _window(text: str, a: int, b: int, max_chars: int) -> tuple[int, int]:
|
||||
"""The sentence around ``text[a:b]`` when it fits *max_chars*, else a
|
||||
window centred on the match, cut at word boundaries."""
|
||||
start = max(text.rfind(m, 0, a) for m in _SENTENCE_END) + 2
|
||||
start = start if start >= 2 else 0
|
||||
ends = [e for m in _SENTENCE_END if (e := text.find(m, b)) != -1]
|
||||
end = min(ends) + 1 if ends else len(text)
|
||||
if end - start <= max_chars:
|
||||
return start, end
|
||||
start = max(0, a - (max_chars - (b - a)) // 2)
|
||||
end = min(len(text), start + max_chars)
|
||||
start = max(0, end - max_chars)
|
||||
if start > 0 and text[start - 1] != " ":
|
||||
cut = text.find(" ", start, a)
|
||||
start = cut + 1 if cut != -1 else start
|
||||
if end < len(text) and text[end] != " ":
|
||||
cut = text.rfind(" ", b, end)
|
||||
end = cut if cut != -1 else end
|
||||
return start, end
|
||||
|
||||
|
||||
def _scan(
|
||||
text: str,
|
||||
term: str,
|
||||
sizes: Sequence[int],
|
||||
max_n: int,
|
||||
grams: AmaIndex | None,
|
||||
) -> tuple[list[tuple[int, str]], int]:
|
||||
"""Up to *max_n* kept ``(char_start, passage)`` and how many occurrences
|
||||
were withheld. An occurrence is tried at each of *sizes* in turn (widest
|
||||
first) until its passage shares no run with *grams*."""
|
||||
low, needle = text.lower(), term.lower()
|
||||
out: list[tuple[int, str]] = []
|
||||
seen: set[int] = set()
|
||||
withheld = 0
|
||||
pos = 0
|
||||
while len(out) < max_n and (i := low.find(needle, pos)) != -1:
|
||||
pos = i + len(needle)
|
||||
for size in sizes:
|
||||
start, end = _window(text, i, pos, size)
|
||||
chunk = text[start:end]
|
||||
start += len(chunk) - len(chunk.lstrip())
|
||||
chunk = chunk.strip()
|
||||
if grams is None or ama_safe(chunk, grams):
|
||||
if start not in seen:
|
||||
seen.add(start)
|
||||
out.append((start, chunk))
|
||||
break
|
||||
else:
|
||||
withheld += 1
|
||||
return out, withheld
|
||||
|
||||
|
||||
def passages(
|
||||
text: str, term: str, *, max_chars: int = 400, max_n: int = 3
|
||||
) -> list[tuple[int, str]]:
|
||||
"""Up to *max_n* ``(char_start, passage)`` around case-insensitive matches
|
||||
of *term* in *text*, each widened to its sentence within *max_chars*."""
|
||||
return _scan(text, term, (max_chars,), max_n, None)[0]
|
||||
|
||||
|
||||
def safe_passages(
|
||||
text: str, term: str, grams: AmaIndex, *, max_chars: int = 400, max_n: int = 3
|
||||
) -> tuple[list[tuple[int, str]], int]:
|
||||
""":func:`passages` with the D3 guard: a passage that shares an 8-word run
|
||||
with a CPT descriptor is retried at 60% and 30% of *max_chars*, then
|
||||
withheld. Returns the kept passages and the number withheld."""
|
||||
sizes = (max_chars, max_chars * 3 // 5, max_chars * 3 // 10)
|
||||
return _scan(text, term, sizes, max_n, grams)
|
||||
|
||||
|
||||
def cpt_grams(descriptors: Iterable[str], n: int = 8) -> AmaIndex:
|
||||
"""The 8-word runs of every descriptor (no run crosses descriptors),
|
||||
tokenized like ``pfs.origin.shares_run``."""
|
||||
return AmaIndex.of(descriptors, n)
|
||||
|
||||
|
||||
def ama_safe(passage: str, grams: AmaIndex) -> bool:
|
||||
"""D3: *passage* shares no 8-word run with any CPT descriptor."""
|
||||
return not shares_ama_run(passage, grams)
|
||||
|
||||
|
||||
# ── local classification ─────────────────────────────────────────────
|
||||
|
||||
EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental")
|
||||
EXPOSURE_PROMPT_VERSION = "exposure-v1"
|
||||
EXPOSURE_SCHEMA: dict[str, Any] = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"label": {"type": "string", "enum": list(EXPOSURE_LABELS)},
|
||||
"reason": {"type": "string", "maxLength": 200},
|
||||
},
|
||||
"required": ["label", "reason"],
|
||||
}
|
||||
EXPOSURE_SYSTEM = (
|
||||
"You read one passage of a company's annual report that mentions a Medicare "
|
||||
"program or payment term. Label how the passage ties the company to it: "
|
||||
"revenue_exposure (the company says it is paid under, or earns revenue "
|
||||
"through, the program or schedule), risk_factor (the company says changes "
|
||||
"to the program could hurt or affect it), or incidental (the passage "
|
||||
"defines, lists or mentions the term without tying the company to it). "
|
||||
"Answer with JSON only: a label and a reason of at most 200 characters."
|
||||
)
|
||||
|
||||
|
||||
def classify_passage(passage: str, term: str, ask: Callable[..., Any]) -> str | None:
|
||||
"""A closed label for one passage from a local model, or ``None`` when
|
||||
the reply is missing or outside :data:`EXPOSURE_LABELS`."""
|
||||
out = ask(
|
||||
EXPOSURE_SYSTEM,
|
||||
f"Term: {term}\nPassage: {passage}",
|
||||
EXPOSURE_SCHEMA,
|
||||
)
|
||||
label = out.get("label") if isinstance(out, dict) else None
|
||||
return label if label in EXPOSURE_LABELS else None
|
||||
|
||||
|
||||
# ── links ────────────────────────────────────────────────────────────
|
||||
|
||||
_RANK = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
|
||||
|
||||
|
||||
def fts_links(
|
||||
hits: Iterable[HitRow],
|
||||
passage_rows: Iterable[PassageRow],
|
||||
entities: Iterable[EntityRow],
|
||||
seed_ciks: Iterable[str],
|
||||
) -> list[LinkRow]:
|
||||
"""``basis="fts"`` links, one per (cik, scope, target): the strongest
|
||||
evidence, then the newest filing. A hit counts when its SIC is a health
|
||||
SIC or its CIK is in the seed. The strength is the passage label, or
|
||||
``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit
|
||||
without a kept passage links with ``passage_id=""``."""
|
||||
seed = set(seed_ciks)
|
||||
sic_of = {e.cik: e.sic for e in entities}
|
||||
by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {}
|
||||
for p in passage_rows:
|
||||
by_doc.setdefault((p.cik, p.adsh, p.filename, p.term), []).append(p)
|
||||
best: dict[tuple[str, str, str], tuple[tuple, LinkRow]] = {}
|
||||
for h in hits:
|
||||
sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")}
|
||||
if h.cik not in seed and not (sics & set(HEALTH_SICS)):
|
||||
continue
|
||||
kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
|
||||
cands: list[tuple[str, str]] = [
|
||||
(p.label or "mention", p.passage_id) for p in kept
|
||||
] or [("mention", "")]
|
||||
for strength, pid in cands:
|
||||
if strength == "incidental":
|
||||
continue
|
||||
rank = (_RANK[strength], -h.file_date.toordinal(), pid == "", pid)
|
||||
link = LinkRow(
|
||||
h.cik,
|
||||
h.scope,
|
||||
h.target,
|
||||
"fts",
|
||||
strength,
|
||||
"adsh",
|
||||
f"{h.adsh}:{h.filename}",
|
||||
pid,
|
||||
)
|
||||
key = (h.cik, h.scope, h.target)
|
||||
if key not in best or rank < best[key][0]:
|
||||
best[key] = (rank, link)
|
||||
return [link for _, link in sorted(best.values(), key=lambda b: b[1].entity_key)]
|
||||
|
||||
|
||||
def cite_filing(store: Any, hit: HitRow, path: Path) -> str:
|
||||
"""The bib Download for the filing behind a hit (keyed by its document
|
||||
URL, so a rerun returns the same key), with the document attached."""
|
||||
from bib.item import Download
|
||||
from bib.tag import Tag
|
||||
|
||||
url = doc_url(hit.cik, hit.adsh, hit.filename)
|
||||
item = Download(
|
||||
title=(
|
||||
f"{hit.display_name} Form {hit.form} filed {hit.file_date} "
|
||||
f"(accession {hit.adsh})"
|
||||
),
|
||||
url=url,
|
||||
file_urls=[url],
|
||||
year=hit.file_date.year,
|
||||
page_type="filing",
|
||||
website_title="U.S. Securities and Exchange Commission",
|
||||
)
|
||||
key = store.upsert(
|
||||
item,
|
||||
tags=[
|
||||
Tag.module("market"),
|
||||
Tag.table("market.filing_passage"),
|
||||
Tag.source("sec-edgar"),
|
||||
Tag.year(hit.file_date.year),
|
||||
],
|
||||
)
|
||||
store.attach_file(key, path, title=path.name)
|
||||
return key
|
||||
|
||||
@@ -123,15 +123,24 @@ def get(
|
||||
max_retry_after: float = 600.0,
|
||||
ok_404: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""GET with pacing. 5xx is retried with a ``2 * 2**n`` second backoff,
|
||||
a 429 with a short ``Retry-After`` waits once, 401/403 stop the source,
|
||||
"""GET with pacing. A transport error (reset, timeout) and a 5xx are
|
||||
retried with a ``2 * 2**n`` second backoff (a transport error that
|
||||
outlasts the retries is a :class:`Refused` with status 0), a 429 with a short ``Retry-After`` waits once, 401/403 stop the source,
|
||||
and a 404 is returned only when *ok_404*. This is the market tier's one
|
||||
retry loop."""
|
||||
attempt = 0
|
||||
rate_waited = False
|
||||
while True:
|
||||
pacer.wait()
|
||||
r = cl.get(url, params=params)
|
||||
try:
|
||||
r = cl.get(url, params=params)
|
||||
except httpx.TransportError as exc:
|
||||
# a dropped connection or a read timeout: back off and retry
|
||||
if attempt < retries:
|
||||
sleep(2.0 * 2**attempt)
|
||||
attempt += 1
|
||||
continue
|
||||
raise Refused(source, 0, type(exc).__name__) from exc
|
||||
status = r.status_code
|
||||
if status < 400:
|
||||
return r
|
||||
|
||||
@@ -33,6 +33,16 @@ CREATE TABLE IF NOT EXISTS market.filing (
|
||||
cik VARCHAR, adsh VARCHAR, form VARCHAR, filed DATE, accepted VARCHAR,
|
||||
report_date DATE, items VARCHAR, primary_doc VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.filing_hit (
|
||||
term VARCHAR, scope VARCHAR, target VARCHAR, cik VARCHAR, adsh VARCHAR,
|
||||
filename VARCHAR, form VARCHAR, file_date DATE, period_ending DATE,
|
||||
sics VARCHAR, display_name VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.filing_passage (
|
||||
passage_id VARCHAR, cik VARCHAR, adsh VARCHAR, filename VARCHAR,
|
||||
term VARCHAR, char_start INTEGER, passage VARCHAR, label VARCHAR,
|
||||
label_version VARCHAR, item_key VARCHAR
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS market.source_gap (
|
||||
source VARCHAR, note VARCHAR, recorded_on DATE
|
||||
);
|
||||
@@ -128,6 +138,41 @@ class FilingRow:
|
||||
primary_doc: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HitRow:
|
||||
"""One EDGAR full-text-search hit of *term*, owned by one *target* (a
|
||||
program or family key)."""
|
||||
|
||||
term: str
|
||||
scope: str # program | family
|
||||
target: str
|
||||
cik: str
|
||||
adsh: str
|
||||
filename: str
|
||||
form: str
|
||||
file_date: date
|
||||
period_ending: date | None
|
||||
sics: str # comma-joined
|
||||
display_name: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PassageRow:
|
||||
"""At most 400 characters of public filing text around *term*; the
|
||||
*label* is a model reading when ``label_version`` is set."""
|
||||
|
||||
passage_id: str # "{adsh}:{filename}:{char_start}"
|
||||
cik: str
|
||||
adsh: str
|
||||
filename: str
|
||||
term: str
|
||||
char_start: int
|
||||
passage: str
|
||||
label: str = ""
|
||||
label_version: str = ""
|
||||
item_key: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GapRow:
|
||||
source: str
|
||||
@@ -359,6 +404,50 @@ def read_filings(
|
||||
return sorted((FilingRow(*r) for r in rows), key=lambda f: (f.filed, f.adsh))
|
||||
|
||||
|
||||
# ── filing_hit / filing_passage ──────────────────────────────────────
|
||||
|
||||
|
||||
def write_hits(con: Any, rows: Sequence[HitRow]) -> int:
|
||||
"""Upsert by ``(term, scope, target, adsh, filename)``."""
|
||||
ensure_tables(con)
|
||||
keys = sorted({(r.term, r.scope, r.target, r.adsh, r.filename) for r in rows})
|
||||
for part in _chunks(keys, 400):
|
||||
con.execute(
|
||||
"DELETE FROM market.filing_hit WHERE (term, scope, target, adsh, filename) "
|
||||
"IN (SELECT * FROM (VALUES "
|
||||
+ ", ".join(["(?, ?, ?, ?, ?)"] * len(part))
|
||||
+ "))",
|
||||
[v for k in part for v in k],
|
||||
)
|
||||
return _insert(con, "market.filing_hit", rows)
|
||||
|
||||
|
||||
def read_hits(con: Any) -> list[HitRow]:
|
||||
rows = _select(con, HitRow, "market.filing_hit")
|
||||
return sorted(
|
||||
(HitRow(*r) for r in rows),
|
||||
key=lambda h: (h.scope, h.target, h.term, h.cik, h.adsh, h.filename),
|
||||
)
|
||||
|
||||
|
||||
def write_passages(con: Any, rows: Sequence[PassageRow]) -> int:
|
||||
"""Replace the passages of each ``(adsh, filename, term)`` in *rows*."""
|
||||
ensure_tables(con)
|
||||
keys = sorted({(r.adsh, r.filename, r.term) for r in rows})
|
||||
for part in _chunks(keys, 400):
|
||||
con.execute(
|
||||
"DELETE FROM market.filing_passage WHERE (adsh, filename, term) "
|
||||
"IN (SELECT * FROM (VALUES " + ", ".join(["(?, ?, ?)"] * len(part)) + "))",
|
||||
[v for k in part for v in k],
|
||||
)
|
||||
return _insert(con, "market.filing_passage", rows)
|
||||
|
||||
|
||||
def read_passages(con: Any) -> list[PassageRow]:
|
||||
rows = _select(con, PassageRow, "market.filing_passage")
|
||||
return sorted((PassageRow(*r) for r in rows), key=lambda p: p.passage_id)
|
||||
|
||||
|
||||
# ── source_gap ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -1004,3 +1004,509 @@ def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
|
||||
assert "re-pulling from 2023-12-25" in res.output
|
||||
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
|
||||
assert row.ret == pytest.approx(60.5 / 55 - 1)
|
||||
|
||||
|
||||
# ── fts ──────────────────────────────────────────────────────────────
|
||||
MPFS = '"Medicare Physician Fee Schedule"'
|
||||
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
|
||||
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
|
||||
|
||||
|
||||
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
|
||||
return {
|
||||
"_id": f"{adsh}:{filename}",
|
||||
"_source": {
|
||||
"ciks": [cik],
|
||||
"adsh": adsh,
|
||||
"form": "10-K",
|
||||
"file_date": date_,
|
||||
"period_ending": "2022-12-31",
|
||||
"sics": sics,
|
||||
"display_names": [f"Filer {cik} (CIK {cik})"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
HITS = [
|
||||
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
|
||||
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
|
||||
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
|
||||
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
|
||||
]
|
||||
|
||||
|
||||
def _fts_net(
|
||||
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
|
||||
):
|
||||
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
|
||||
2023; every other query is empty."""
|
||||
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
|
||||
requested: list[str] = []
|
||||
|
||||
def efts(m):
|
||||
p = dict(httpx.URL(m.group(0)).params)
|
||||
mine = p["q"] == MPFS and p["from"] == "0"
|
||||
return httpx.Response(
|
||||
status,
|
||||
json={
|
||||
"hits": {
|
||||
"total": {"value": len(hits) if mine else 0},
|
||||
"hits": hits if mine else [],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
def doc(m):
|
||||
requested.append(m.group(1))
|
||||
if broken and m.group(1) in broken:
|
||||
if broken[m.group(1)] == "timeout":
|
||||
raise httpx.ReadTimeout("slow")
|
||||
return httpx.Response(broken[m.group(1)])
|
||||
body = docs.get(m.group(1))
|
||||
if forbid_docs and body:
|
||||
raise AssertionError("no document download expected")
|
||||
return httpx.Response(200, content=body) if body else httpx.Response(404)
|
||||
|
||||
def sub(m):
|
||||
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
|
||||
|
||||
_online(
|
||||
monkeypatch,
|
||||
[
|
||||
(r"efts\.sec\.gov/.*", efts),
|
||||
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
|
||||
(
|
||||
r"company_tickers\.json",
|
||||
lambda m: httpx.Response(
|
||||
200, content=(FX / "company_tickers.json").read_bytes()
|
||||
),
|
||||
),
|
||||
(r"CIK(\d{10})\.json", sub),
|
||||
],
|
||||
)
|
||||
return requested
|
||||
|
||||
|
||||
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
|
||||
|
||||
|
||||
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
|
||||
c, _, published, hoard = env
|
||||
monkeypatch.setattr(
|
||||
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
|
||||
)
|
||||
res = runner.invoke(app, [*FTS, "--dry-run"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "program PFS: 5 terms, 5 queries" in res.output
|
||||
assert "5 queries (0 already hoarded)" in res.output
|
||||
assert published == []
|
||||
full = runner.invoke(app, ["market", "fts", "--dry-run"])
|
||||
assert "program MSSP: 1 terms" in full.output
|
||||
assert "program CARE-MANAGEMENT: 2 terms" in full.output
|
||||
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
|
||||
|
||||
|
||||
def dt_year():
|
||||
import datetime
|
||||
|
||||
return datetime.date.today().year
|
||||
|
||||
|
||||
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
|
||||
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
|
||||
res = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"market",
|
||||
"fts",
|
||||
"--family",
|
||||
"HOME-VISITS",
|
||||
"--dry-run",
|
||||
"--from",
|
||||
"2023",
|
||||
"--to",
|
||||
"2023",
|
||||
],
|
||||
)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
|
||||
assert (
|
||||
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
|
||||
== 2
|
||||
)
|
||||
assert (
|
||||
runner.invoke(
|
||||
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
|
||||
).exit_code
|
||||
== 2
|
||||
)
|
||||
|
||||
|
||||
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
|
||||
c, _, published, hoard = env
|
||||
_fts_net(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
|
||||
)
|
||||
res = runner.invoke(app, FTS)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
|
||||
assert "passages kept 1," in res.output and "new entities 1" in res.output
|
||||
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
|
||||
assert (
|
||||
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
|
||||
in res.output
|
||||
)
|
||||
assert (
|
||||
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
|
||||
in res.output
|
||||
)
|
||||
assert published == []
|
||||
# hoarded: the pages and the one downloaded readable document
|
||||
assert (
|
||||
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
|
||||
).is_file()
|
||||
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
|
||||
|
||||
|
||||
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
|
||||
c, store, published, hoard = env
|
||||
curated = T.LinkRow(
|
||||
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
|
||||
)
|
||||
T.write_links(c, [curated])
|
||||
_fts_net(monkeypatch)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
(ent,) = T.read_entities(c, "company")
|
||||
assert ent.key == "0000012345" and ent.sic == "8082"
|
||||
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
|
||||
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
|
||||
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
|
||||
assert T.read_links(c, basis="curated") == [curated] # survives
|
||||
(fts_link,) = T.read_links(c, basis="fts")
|
||||
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
|
||||
"program",
|
||||
"PFS",
|
||||
"mention",
|
||||
)
|
||||
assert fts_link.item_key and fts_link.passage_id.startswith(
|
||||
"0000012345-23-000001:a10k.htm:"
|
||||
)
|
||||
# the link's passage is verbatim in the hoarded filing, short, and cited
|
||||
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
|
||||
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
|
||||
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
|
||||
assert p.passage in text and p.label == "" and p.label_version == ""
|
||||
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
|
||||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||||
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
|
||||
logs = {
|
||||
t
|
||||
for (t,) in c.execute(
|
||||
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
|
||||
).fetchall()
|
||||
}
|
||||
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
|
||||
assert published == [True] and store.count() == 1
|
||||
# a rerun is idempotent and downloads nothing new
|
||||
n_docs = _fts_net(monkeypatch, forbid_docs=True)
|
||||
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
||||
assert (
|
||||
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
|
||||
)
|
||||
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
|
||||
|
||||
|
||||
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
_fts_net(monkeypatch, status=403)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
||||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||||
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
|
||||
assert published == [True]
|
||||
|
||||
|
||||
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
|
||||
c, _, _, _ = env
|
||||
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
|
||||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "passages kept 0, withheld by the AMA guard 1" in res.output
|
||||
assert "fts links 0 (none)" in res.output
|
||||
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
|
||||
assert len(T.read_hits(c)) == 1 # the hit stays
|
||||
|
||||
|
||||
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
|
||||
c, _, _, hoard = env
|
||||
_fts_net(monkeypatch, hits=HITS[:1])
|
||||
real = mk._verified
|
||||
|
||||
def tamper(links, passages):
|
||||
for p in passages.values():
|
||||
p.__dict__["passage"] = "not in the filing"
|
||||
return real(links, passages)
|
||||
|
||||
monkeypatch.setattr(mk, "_verified", tamper)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
|
||||
assert T.read_links(c, basis="fts") == []
|
||||
|
||||
|
||||
def test_verified_handles_missing_filing_and_unpassaged_links(env):
|
||||
_, _, _, hoard = env
|
||||
link = T.LinkRow(
|
||||
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
|
||||
)
|
||||
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
|
||||
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
|
||||
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
|
||||
assert mk._verified([bare], {}) == ([], 1)
|
||||
|
||||
|
||||
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
hits = [
|
||||
HITS[0],
|
||||
_hit_json(
|
||||
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
||||
),
|
||||
]
|
||||
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
|
||||
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "1 filing(s) not downloaded yet" in res.output
|
||||
assert "incomplete (limit or time): nothing written" in res.output
|
||||
assert published == []
|
||||
# the newest filing was fetched first
|
||||
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
|
||||
|
||||
|
||||
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
_fts_net(monkeypatch)
|
||||
ticks = iter(range(0, 10_000, 100))
|
||||
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
|
||||
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "incomplete" in res.output and published == []
|
||||
|
||||
|
||||
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
|
||||
c, _, _, _ = env
|
||||
_fts_net(monkeypatch)
|
||||
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
||||
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
|
||||
c.execute(
|
||||
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
|
||||
"WHERE passage_id = ?",
|
||||
[p.passage_id],
|
||||
)
|
||||
_fts_net(monkeypatch, forbid_docs=True)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0 and "new entities 0" in res.output
|
||||
got = {q.passage_id: q for q in T.read_passages(c)}
|
||||
assert (
|
||||
got[p.passage_id].label == "risk_factor"
|
||||
and got[p.passage_id].label_version == "v"
|
||||
)
|
||||
(link,) = T.read_links(c, basis="fts")
|
||||
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
|
||||
|
||||
|
||||
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
|
||||
c, _, _, _ = env
|
||||
_fts_net(monkeypatch, hits=HITS[:1])
|
||||
_online(
|
||||
monkeypatch,
|
||||
[
|
||||
(
|
||||
r"efts\.sec\.gov/.*",
|
||||
lambda m: (
|
||||
httpx.Response(
|
||||
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
||||
)
|
||||
if "Medicare+Physician" in m.group(0)
|
||||
or "Medicare%20Physician" in m.group(0)
|
||||
else httpx.Response(
|
||||
200, json={"hits": {"total": {"value": 0}, "hits": []}}
|
||||
)
|
||||
),
|
||||
),
|
||||
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
||||
(
|
||||
r"company_tickers\.json",
|
||||
lambda m: httpx.Response(
|
||||
200, content=(FX / "company_tickers.json").read_bytes()
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "CIK0000012345: no submissions, no entity" in res.output
|
||||
assert T.read_entities(c) == []
|
||||
|
||||
|
||||
# classification: the local model, resumable through the ledger
|
||||
class _Ask:
|
||||
model = "qwen-test"
|
||||
|
||||
def __init__(self, answers):
|
||||
self.answers = answers
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, system, user, schema):
|
||||
self.calls += 1
|
||||
return self.answers.pop(0)
|
||||
|
||||
|
||||
TWO = [
|
||||
HITS[0],
|
||||
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
|
||||
]
|
||||
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
|
||||
|
||||
|
||||
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
|
||||
c, _, _, _ = env
|
||||
monkeypatch.setattr(
|
||||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
|
||||
)
|
||||
# a passage of a term outside this run's scope is never sent to the model
|
||||
T.write_passages(
|
||||
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
|
||||
)
|
||||
ask = _Ask(
|
||||
[
|
||||
{"label": "revenue_exposure", "reason": "r"},
|
||||
None,
|
||||
{"label": "incidental", "reason": "i"},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||||
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
||||
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
|
||||
labelled = [p for p in T.read_passages(c) if p.label]
|
||||
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
|
||||
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
|
||||
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0 and ask.calls == 2
|
||||
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
|
||||
# a rerun without --classify keeps what is stored
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert len([p for p in T.read_passages(c) if p.label]) == 1
|
||||
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
|
||||
assert got == ["mention", "revenue_exposure"]
|
||||
|
||||
|
||||
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
|
||||
c, _, _, _ = env
|
||||
monkeypatch.setattr(
|
||||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||||
)
|
||||
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
|
||||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||||
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
|
||||
|
||||
|
||||
def test_asker_builds_json_extractor(monkeypatch):
|
||||
sentinel = object()
|
||||
seen = {}
|
||||
monkeypatch.setattr("llm.config.load", lambda: "cfg")
|
||||
monkeypatch.setattr(
|
||||
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
|
||||
)
|
||||
|
||||
def make(cfg, pool):
|
||||
seen["args"] = (cfg, pool)
|
||||
return sentinel
|
||||
|
||||
monkeypatch.setattr("llm.structured.json_extractor", make)
|
||||
assert mk._asker() is sentinel
|
||||
assert seen["args"] == ("cfg", ("pool", "cfg"))
|
||||
|
||||
|
||||
def test_gap_note_truncates():
|
||||
note = mk._gap_note([f"i{n}" for n in range(20)])
|
||||
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
|
||||
|
||||
|
||||
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
_fts_net(monkeypatch, hits=HITS[:1])
|
||||
_online(
|
||||
monkeypatch,
|
||||
[
|
||||
(
|
||||
r"efts\.sec\.gov/.*",
|
||||
lambda m: httpx.Response(
|
||||
200,
|
||||
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
|
||||
if "Medicare%20Physician" in m.group(0)
|
||||
or "Medicare+Physician" in m.group(0)
|
||||
else {"hits": {"total": {"value": 0}, "hits": []}},
|
||||
),
|
||||
),
|
||||
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
|
||||
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
|
||||
(r"company_tickers\.json", lambda m: httpx.Response(403)),
|
||||
],
|
||||
)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
|
||||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
|
||||
assert _count(c, "market.filing_hit") == 0 and published == [True]
|
||||
|
||||
|
||||
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
|
||||
c, _, _, _ = env
|
||||
hits = [
|
||||
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
|
||||
for n in range(22)
|
||||
]
|
||||
_fts_net(monkeypatch, hits=hits)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: sec-edgar:fts: ... 2 more" in res.output
|
||||
assert "... 7 more" in T.read_gaps(c)[0].note
|
||||
|
||||
|
||||
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
|
||||
hits = [
|
||||
HITS[0],
|
||||
_hit_json(
|
||||
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
|
||||
),
|
||||
]
|
||||
_fts_net(
|
||||
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
|
||||
)
|
||||
monkeypatch.setattr("time.sleep", lambda s: None)
|
||||
res = runner.invoke(app, FTS)
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
|
||||
assert "passages kept 1," in res.output
|
||||
|
||||
|
||||
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
|
||||
c, _, published, _ = env
|
||||
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
|
||||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
|
||||
True
|
||||
]
|
||||
|
||||
140
tests/market/fixtures/efts_mpfs_2023.json
Normal file
140
tests/market/fixtures/efts_mpfs_2023.json
Normal file
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"hits": {
|
||||
"total": {
|
||||
"value": 57,
|
||||
"relation": "eq"
|
||||
},
|
||||
"max_score": 8.786024,
|
||||
"hits": [
|
||||
{
|
||||
"_index": "edgar_file",
|
||||
"_id": "0000790526-23-000003:rdnt-20221231.htm",
|
||||
"_score": 8.786024,
|
||||
"_source": {
|
||||
"ciks": [
|
||||
"0000790526"
|
||||
],
|
||||
"period_ending": "2022-12-31",
|
||||
"file_num": [
|
||||
"001-33307"
|
||||
],
|
||||
"display_names": [
|
||||
"RadNet, Inc. (RDNT) (CIK 0000790526)"
|
||||
],
|
||||
"xsl": null,
|
||||
"sequence": 1,
|
||||
"root_forms": [
|
||||
"10-K"
|
||||
],
|
||||
"file_date": "2023-03-01",
|
||||
"biz_states": [
|
||||
"CA"
|
||||
],
|
||||
"sics": [
|
||||
"8071"
|
||||
],
|
||||
"form": "10-K",
|
||||
"adsh": "0000790526-23-000003",
|
||||
"film_num": [
|
||||
"23695229"
|
||||
],
|
||||
"biz_locations": [
|
||||
"Los Angeles, CA"
|
||||
],
|
||||
"file_type": "10-K",
|
||||
"file_description": "10-K",
|
||||
"inc_states": [
|
||||
"DE"
|
||||
],
|
||||
"items": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "edgar_file",
|
||||
"_id": "0001798270-23-000007:ionm-20221231x10k.htm",
|
||||
"_score": 8.094215,
|
||||
"_source": {
|
||||
"ciks": [
|
||||
"0001798270"
|
||||
],
|
||||
"period_ending": "2022-12-31",
|
||||
"file_num": [
|
||||
"001-40785"
|
||||
],
|
||||
"display_names": [
|
||||
"Assure Holdings Corp. (IONM) (CIK 0001798270)"
|
||||
],
|
||||
"xsl": null,
|
||||
"sequence": 1,
|
||||
"root_forms": [
|
||||
"10-K"
|
||||
],
|
||||
"file_date": "2023-03-31",
|
||||
"biz_states": [
|
||||
"CO"
|
||||
],
|
||||
"sics": [
|
||||
"3841"
|
||||
],
|
||||
"form": "10-K",
|
||||
"adsh": "0001798270-23-000007",
|
||||
"film_num": [
|
||||
"23788435"
|
||||
],
|
||||
"biz_locations": [
|
||||
"Greenwood Village, CO"
|
||||
],
|
||||
"file_type": "10-K",
|
||||
"file_description": "10-K",
|
||||
"inc_states": [
|
||||
"NV"
|
||||
],
|
||||
"items": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"_index": "edgar_file",
|
||||
"_id": "0000355019-23-000044:fonar_10-k.htm",
|
||||
"_score": 7.894161,
|
||||
"_source": {
|
||||
"ciks": [
|
||||
"0000355019"
|
||||
],
|
||||
"period_ending": "2023-06-30",
|
||||
"file_num": [
|
||||
"000-10248"
|
||||
],
|
||||
"display_names": [
|
||||
"FONAR CORP (FONR) (CIK 0000355019)"
|
||||
],
|
||||
"xsl": null,
|
||||
"sequence": 1,
|
||||
"root_forms": [
|
||||
"10-K"
|
||||
],
|
||||
"file_date": "2023-09-28",
|
||||
"biz_states": [
|
||||
"NY"
|
||||
],
|
||||
"sics": [
|
||||
"3845"
|
||||
],
|
||||
"form": "10-K",
|
||||
"adsh": "0000355019-23-000044",
|
||||
"film_num": [
|
||||
"231289120"
|
||||
],
|
||||
"biz_locations": [
|
||||
"Melville, NY"
|
||||
],
|
||||
"file_type": "10-K",
|
||||
"file_description": "FONAR CORPORATION FORM 10-K AT JUNE 30, 2023",
|
||||
"inc_states": [
|
||||
"DE"
|
||||
],
|
||||
"items": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
7
tests/market/fixtures/filing_snippet.htm
Normal file
7
tests/market/fixtures/filing_snippet.htm
Normal file
@@ -0,0 +1,7 @@
|
||||
<html><head><style>p { color: red; }</style></head><body>
|
||||
<ix:header><ix:hidden>HIDDEN XBRL TEXT physician fee schedule</ix:hidden></ix:header>
|
||||
<p>Item 1. Business. We operate imaging centers across several states.</p>
|
||||
<p>Substantially all of our Medicare revenue is paid under the Medicare Physician Fee Schedule, and the Centers for Medicare & Medicaid Services’ annual updates to the conversion factor affect our results. We cannot predict future rates.</p>
|
||||
<p>Our contracts with commercial payors are described in Item 7.</p>
|
||||
<p>Changes to the physician fee schedule could reduce our reimbursement.</p>
|
||||
</body></html>
|
||||
387
tests/market/test_fts.py
Normal file
387
tests/market/test_fts.py
Normal file
@@ -0,0 +1,387 @@
|
||||
"""market.edgar full-text search: queries, hits, passages, the AMA guard,
|
||||
the local classifier and the links they produce."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from bib.store import Store
|
||||
from market import edgar, http
|
||||
from market import tables as T
|
||||
from market.seed import PROGRAMS
|
||||
|
||||
FX = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
class _Pacer:
|
||||
def wait(self):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hoard(monkeypatch, tmp_path):
|
||||
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
|
||||
return tmp_path / "market"
|
||||
|
||||
|
||||
def test_fts_params_one_year():
|
||||
p = edgar.fts_params("Medicare Physician Fee Schedule", "10-K", 2023, 0)
|
||||
assert p["q"] == '"Medicare Physician Fee Schedule"'
|
||||
assert p["startdt"] == "2023-01-01" and p["enddt"] == "2023-12-31"
|
||||
assert p["from"] == 0 and p["forms"] == "10-K" and p["dateRange"] == "custom"
|
||||
|
||||
|
||||
def test_parse_hits():
|
||||
payload = json.loads((FX / "efts_mpfs_2023.json").read_text())
|
||||
rows = edgar.parse_hits(
|
||||
payload, term="Medicare Physician Fee Schedule", scope="program", target="PFS"
|
||||
)
|
||||
assert len(rows) == 3
|
||||
assert rows[0] == T.HitRow(
|
||||
term="Medicare Physician Fee Schedule",
|
||||
scope="program",
|
||||
target="PFS",
|
||||
cik="0000790526",
|
||||
adsh="0000790526-23-000003",
|
||||
filename="rdnt-20221231.htm",
|
||||
form="10-K",
|
||||
file_date=date(2023, 3, 1),
|
||||
period_ending=date(2022, 12, 31),
|
||||
sics="8071",
|
||||
display_name="RadNet, Inc. (RDNT) (CIK 0000790526)",
|
||||
)
|
||||
|
||||
|
||||
def test_parse_hits_without_optional_fields():
|
||||
payload = {
|
||||
"hits": {
|
||||
"hits": [
|
||||
{
|
||||
"_id": "0000000001-23-000001:a.htm",
|
||||
"_source": {"ciks": ["1"], "file_date": "2023-01-02"},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
(h,) = edgar.parse_hits(payload, term="t", scope="family", target="CCM")
|
||||
assert (h.cik, h.period_ending, h.sics, h.display_name, h.form) == (
|
||||
"0000000001",
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
)
|
||||
assert edgar.parse_hits({}, term="t", scope="program", target="PFS") == []
|
||||
|
||||
|
||||
def _pages(total):
|
||||
def handler(request: httpx.Request):
|
||||
q = dict(request.url.params)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"hits": {
|
||||
"total": {"value": total},
|
||||
"hits": [
|
||||
{
|
||||
"_id": f"0000000001-23-00000{q['from']}:a.htm",
|
||||
"_source": {"ciks": ["1"], "file_date": "2023-01-02"},
|
||||
}
|
||||
],
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
def test_fts_pages_until_total(hoard):
|
||||
cl = httpx.Client(transport=httpx.MockTransport(_pages(150)))
|
||||
got = list(edgar.fetch_fts(cl, "RBRVS", "10-K", 2023, pacer=_Pacer()))
|
||||
assert [off for off, _ in got] == [0, 100]
|
||||
cached = hoard / "sec" / "fts" / "rbrvs" / "10-K" / "2023.100.json"
|
||||
assert cached.is_file()
|
||||
assert edgar.fts_cached("RBRVS", "10-K", 2023)
|
||||
assert not edgar.fts_cached("RBRVS", "10-K", 2022)
|
||||
|
||||
|
||||
def test_fts_pages_come_from_the_hoard_and_offline_stops(hoard):
|
||||
cl = httpx.Client(transport=httpx.MockTransport(_pages(50)))
|
||||
first = list(edgar.fetch_fts(cl, "RBRVS", "10-K/A", 2023, pacer=_Pacer()))
|
||||
assert (hoard / "sec/fts/rbrvs/10-K_A/2023.0.json").is_file()
|
||||
again = list(
|
||||
edgar.fetch_fts(None, "RBRVS", "10-K/A", 2023, pacer=_Pacer(), offline=True)
|
||||
)
|
||||
assert again == first
|
||||
assert (
|
||||
list(edgar.fetch_fts(None, "RBRVS", "10-K", 1999, pacer=_Pacer(), offline=True))
|
||||
== []
|
||||
)
|
||||
|
||||
|
||||
def test_doc_url():
|
||||
assert (
|
||||
edgar.doc_url("0000790526", "0000790526-23-000003", "rdnt-20221231.htm")
|
||||
== "https://www.sec.gov/Archives/edgar/data/790526/000079052623000003/"
|
||||
"rdnt-20221231.htm"
|
||||
)
|
||||
|
||||
|
||||
def test_family_terms():
|
||||
from pfs.families import FAMILIES
|
||||
|
||||
terms = edgar.family_terms(FAMILIES["HOME-VISITS"])
|
||||
low = [t.lower() for t in terms]
|
||||
assert "house calls" in low and "home or residence" in low
|
||||
assert "domiciliary" not in low
|
||||
assert len(low) == len(set(low)) and terms[0] == FAMILIES["HOME-VISITS"].name
|
||||
|
||||
|
||||
def test_program_terms_keys_are_programs():
|
||||
assert set(edgar.PROGRAM_TERMS) == set(PROGRAMS)
|
||||
assert "RBRVS" in edgar.PROGRAM_TERMS["PFS"]
|
||||
|
||||
|
||||
def test_passages_window_and_cap():
|
||||
text = (
|
||||
"x " * 500
|
||||
+ "We bill under the Medicare Physician Fee Schedule today. "
|
||||
+ "y " * 500
|
||||
)
|
||||
at = text.index("Medicare Physician")
|
||||
(one, *_) = edgar.passages(text, "medicare physician fee schedule")
|
||||
start, p = one
|
||||
assert start <= at < start + len(p) and len(p) <= 400
|
||||
assert "Medicare Physician Fee Schedule" in p
|
||||
assert text[start : start + len(p)] == p
|
||||
# a short sentence is returned whole
|
||||
short = "Intro. Our pay follows RBRVS rules. Next."
|
||||
assert edgar.passages(short, "rbrvs") == [(7, "Our pay follows RBRVS rules.")]
|
||||
many = " ".join(f"{'filler ' * 200} RBRVS applies." for _ in range(5))
|
||||
got = edgar.passages(many, "rbrvs")
|
||||
assert len(got) == 3 and all(len(p) <= 400 for _, p in got)
|
||||
assert edgar.passages("nothing here", "rbrvs") == []
|
||||
|
||||
|
||||
def test_passages_dedupe_same_sentence():
|
||||
got = edgar.passages("Both RBRVS and RBRVS appear here.", "rbrvs")
|
||||
assert got == [(0, "Both RBRVS and RBRVS appear here.")]
|
||||
|
||||
|
||||
def test_normalize_and_snippet_passages():
|
||||
text = edgar.normalize_text((FX / "filing_snippet.htm").read_text())
|
||||
assert "HIDDEN" not in text and "color: red" not in text
|
||||
assert "Medicaid Services' annual" in text # entity + curly quote decoded
|
||||
(a, b) = edgar.passages(text, "physician fee schedule")
|
||||
assert a[1].startswith("Substantially all of our Medicare revenue")
|
||||
assert (
|
||||
b[1] == "Changes to the physician fee schedule could reduce our reimbursement."
|
||||
)
|
||||
assert edgar.quote_in(text, a[1])
|
||||
|
||||
|
||||
DESC = "Office or other outpatient visit for the evaluation and management of an established patient"
|
||||
|
||||
|
||||
def test_ama_guard_matches_shares_run():
|
||||
from pfs.origin import shares_run
|
||||
|
||||
pairs = [
|
||||
("We see patients: " + DESC + " each day.", DESC),
|
||||
("Office visits are our business and management of patients too", DESC),
|
||||
("short", DESC),
|
||||
]
|
||||
for passage, d in pairs:
|
||||
assert edgar.ama_safe(passage, edgar.cpt_grams([d])) == (
|
||||
not shares_run(passage, d)
|
||||
)
|
||||
assert edgar.cpt_grams(["a b c"], 8) == frozenset()
|
||||
|
||||
|
||||
def test_safe_passages_retry_narrower_then_withhold():
|
||||
grams = edgar.cpt_grams([DESC])
|
||||
long = f"{'pad ' * 60}Our term RBRVS is used. {DESC} is one service. {'pad ' * 60}"
|
||||
kept, held = edgar.safe_passages(long, "rbrvs", grams)
|
||||
assert held == 0 and kept and "RBRVS" in kept[0][1]
|
||||
assert all(edgar.ama_safe(p, grams) for _, p in kept)
|
||||
# a term inside the descriptor cannot be separated from it
|
||||
inside = f"{DESC} for RBRVS"
|
||||
short = f"The text {DESC}."
|
||||
assert edgar.safe_passages(short, "evaluation", grams) == ([], 1)
|
||||
assert edgar.safe_passages(inside, "rbrvs", grams)[1] in (0, 1)
|
||||
|
||||
|
||||
def test_passage_quoting_descriptor_without_code_withheld():
|
||||
grams = edgar.cpt_grams([DESC])
|
||||
text = f"We provide {DESC}. This is billed under RBRVS."
|
||||
assert edgar.safe_passages(text, "established patient", grams) == ([], 1)
|
||||
h = T.HitRow(
|
||||
"established patient",
|
||||
"program",
|
||||
"PFS",
|
||||
"0000790526",
|
||||
"0000790526-23-000003",
|
||||
"a.htm",
|
||||
"10-K",
|
||||
date(2023, 3, 1),
|
||||
None,
|
||||
"8071",
|
||||
"RadNet",
|
||||
)
|
||||
(link,) = edgar.fts_links([h], [], [], [])
|
||||
assert link.passage_id == "" and link.strength == "mention"
|
||||
assert link.cite_ref == "0000790526-23-000003:a.htm"
|
||||
|
||||
|
||||
def _hit(cik="0000790526", sics="8071", adsh="0000790526-23-000003", **kw):
|
||||
base = dict(
|
||||
term="rbrvs",
|
||||
scope="program",
|
||||
target="PFS",
|
||||
cik=cik,
|
||||
adsh=adsh,
|
||||
filename="a.htm",
|
||||
form="10-K",
|
||||
file_date=date(2023, 3, 1),
|
||||
period_ending=None,
|
||||
sics=sics,
|
||||
display_name="X",
|
||||
)
|
||||
base.update(kw)
|
||||
return T.HitRow(**base)
|
||||
|
||||
|
||||
def _passage(h, label="", start=10):
|
||||
return T.PassageRow(
|
||||
f"{h.adsh}:{h.filename}:{start}",
|
||||
h.cik,
|
||||
h.adsh,
|
||||
h.filename,
|
||||
h.term,
|
||||
start,
|
||||
"text",
|
||||
label,
|
||||
)
|
||||
|
||||
|
||||
def test_fts_links_require_health_sic_or_seed():
|
||||
ok, other = _hit(), _hit(cik="0000000002", sics="6798")
|
||||
links = edgar.fts_links([ok, other], [], [], [])
|
||||
assert [l.entity_key for l in links] == ["0000790526"]
|
||||
assert links[0].basis == "fts" and links[0].cite_kind == "adsh"
|
||||
# a seed CIK links whatever its SIC; an entity's own SIC counts too
|
||||
assert (
|
||||
edgar.fts_links([other], [], [], ["0000000002"])[0].entity_key == "0000000002"
|
||||
)
|
||||
ent = T.EntityRow("0000000002", "E", "company", cik="0000000002", sic="8082")
|
||||
assert len(edgar.fts_links([_hit(cik="0000000002", sics="")], [], [ent], [])) == 1
|
||||
assert edgar.fts_links([_hit(cik="0000000002", sics="")], [], [], []) == []
|
||||
|
||||
|
||||
def test_link_strength_from_label():
|
||||
h = _hit()
|
||||
for label, want in (("", "mention"), ("revenue_exposure", "revenue_exposure")):
|
||||
(l,) = edgar.fts_links([h], [_passage(h, label)], [], [])
|
||||
assert l.strength == want and l.passage_id == f"{h.adsh}:a.htm:10"
|
||||
assert edgar.fts_links([h], [_passage(h, "incidental")], [], []) == []
|
||||
|
||||
|
||||
def test_best_evidence_wins_per_target():
|
||||
old = _hit(adsh="0000790526-20-000001", file_date=date(2020, 3, 1))
|
||||
new = _hit(adsh="0000790526-23-000003")
|
||||
got = edgar.fts_links(
|
||||
[old, new],
|
||||
[_passage(old, "risk_factor"), _passage(new, "revenue_exposure")],
|
||||
[],
|
||||
[],
|
||||
)
|
||||
assert len(got) == 1 and got[0].strength == "revenue_exposure"
|
||||
# equal strength: the newest filing
|
||||
got = edgar.fts_links([old, new], [_passage(old), _passage(new)], [], [])
|
||||
assert got[0].cite_ref.startswith("0000790526-23")
|
||||
# a labelled passage beats an unlabelled sibling
|
||||
got = edgar.fts_links(
|
||||
[new], [_passage(new, "", 5), _passage(new, "risk_factor", 9)], [], []
|
||||
)
|
||||
assert got[0].strength == "risk_factor"
|
||||
|
||||
|
||||
def test_classify_passage_closed_labels():
|
||||
seen = []
|
||||
|
||||
def ask(system, user, schema):
|
||||
seen.append((system, user, schema))
|
||||
return {"label": "revenue_exposure", "reason": "paid under it"}
|
||||
|
||||
assert edgar.classify_passage("p", "rbrvs", ask) == "revenue_exposure"
|
||||
assert "rbrvs" in seen[0][1] and seen[0][2]["properties"]["label"]["enum"] == list(
|
||||
edgar.EXPOSURE_LABELS
|
||||
)
|
||||
assert edgar.classify_passage("p", "t", lambda *a: {"label": "bogus"}) is None
|
||||
assert edgar.classify_passage("p", "t", lambda *a: None) is None
|
||||
assert edgar.EXPOSURE_PROMPT_VERSION == "exposure-v1"
|
||||
|
||||
|
||||
def test_fetch_doc_hoards_and_reads(hoard):
|
||||
h = _hit()
|
||||
calls = []
|
||||
|
||||
def handler(request):
|
||||
calls.append(str(request.url))
|
||||
if request.url.path.endswith("gone.htm"):
|
||||
return httpx.Response(404)
|
||||
return httpx.Response(200, content=(FX / "filing_snippet.htm").read_bytes())
|
||||
|
||||
cl = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
path = edgar.fetch_doc(cl, h, pacer=_Pacer())
|
||||
assert path == hoard / "sec/docs/0000790526/0000790526-23-000003/a.htm"
|
||||
assert "Substantially all" in edgar.read_doc(path)
|
||||
again = edgar.fetch_doc(cl, h, pacer=_Pacer())
|
||||
assert again == path and len(calls) == 1 # never refetched
|
||||
gone = edgar.fetch_doc(cl, _hit(filename="gone.htm"), pacer=_Pacer())
|
||||
assert gone is None
|
||||
assert (
|
||||
edgar.fetch_doc(None, _hit(filename="x.htm"), pacer=_Pacer(), offline=True)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
def test_cite_filing_attaches_document(tmp_path):
|
||||
store = Store(":memory:", storage_dir=tmp_path / "st")
|
||||
doc = tmp_path / "a.htm"
|
||||
doc.write_text("<html>doc</html>")
|
||||
h = _hit(display_name="RadNet, Inc.")
|
||||
key = edgar.cite_filing(store, h, doc)
|
||||
assert edgar.cite_filing(store, h, doc) == key
|
||||
item = store.get(key)
|
||||
assert item.title == (
|
||||
"RadNet, Inc. Form 10-K filed 2023-03-01 (accession 0000790526-23-000003)"
|
||||
)
|
||||
assert item.website_title == "U.S. Securities and Exchange Commission"
|
||||
names = {t["name"] if isinstance(t, dict) else t[0] for t in store.list_tags()}
|
||||
assert {"module:market", "source:sec-edgar", "year:2023"} <= names
|
||||
assert "table:market.filing_passage" in names
|
||||
n = store._con().execute("SELECT count(*) FROM attachments").fetchone()[0]
|
||||
assert n == 1
|
||||
store.close()
|
||||
|
||||
|
||||
def test_window_snaps_to_word_boundaries_in_a_long_sentence():
|
||||
words = " ".join(f"word{i}" for i in range(400)) # one 2,800-char sentence
|
||||
mid = words.index("word200")
|
||||
(start, p), *_ = edgar.passages(words + ".", "word200")
|
||||
assert len(p) <= 400 and p.startswith("word") and p.split()[-1].startswith("word")
|
||||
assert start <= mid and "word200" in p
|
||||
# a run with no spaces cannot be snapped: still capped, term kept
|
||||
blob = "z" * 900 + "RBRVS" + "z" * 900
|
||||
(start, p), *_ = edgar.passages(blob, "rbrvs")
|
||||
assert len(p) == 400 and "RBRVS" in p and blob[start : start + 400] == p
|
||||
# near the end of the text the window is pulled back to full size
|
||||
tail = ("q " * 600) + "RBRVS"
|
||||
(start, p), *_ = edgar.passages(tail, "rbrvs")
|
||||
assert p.endswith("RBRVS") and len(p) <= 400
|
||||
@@ -109,6 +109,42 @@ def test_get_5xx_exhausted_raises_refused():
|
||||
assert ei.value.status == 502 and slept == [2.0, 4.0]
|
||||
|
||||
|
||||
def test_get_retries_transport_errors_then_refuses():
|
||||
slept = []
|
||||
calls = []
|
||||
|
||||
def boom(request):
|
||||
calls.append(1)
|
||||
raise httpx.ReadTimeout("slow", request=request)
|
||||
|
||||
with pytest.raises(Refused) as ei:
|
||||
get(
|
||||
_cl(boom),
|
||||
"https://x.test/a",
|
||||
pacer=_Pacer(),
|
||||
source="s",
|
||||
sleep=slept.append,
|
||||
retries=2,
|
||||
)
|
||||
assert ei.value.status == 0 and "ReadTimeout" in str(ei.value)
|
||||
assert slept == [2.0, 4.0] and len(calls) == 3
|
||||
|
||||
|
||||
def test_get_transport_error_then_ok():
|
||||
state = []
|
||||
|
||||
def flaky(request):
|
||||
if not state:
|
||||
state.append(1)
|
||||
raise httpx.ConnectError("reset", request=request)
|
||||
return httpx.Response(200, text="ok")
|
||||
|
||||
r = get(
|
||||
_cl(flaky), "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None
|
||||
)
|
||||
assert r.text == "ok"
|
||||
|
||||
|
||||
def test_get_429_retry_after_waits_once():
|
||||
slept = []
|
||||
cl = _cl(
|
||||
|
||||
@@ -36,6 +36,8 @@ def test_ensure_tables_idempotent():
|
||||
"security",
|
||||
"entity_link",
|
||||
"filing",
|
||||
"filing_hit",
|
||||
"filing_passage",
|
||||
"source_gap",
|
||||
"price_daily",
|
||||
"series",
|
||||
@@ -75,6 +77,31 @@ FILING = T.FilingRow(
|
||||
"d.htm",
|
||||
)
|
||||
GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today())
|
||||
HIT = T.HitRow(
|
||||
"RBRVS",
|
||||
"program",
|
||||
"PFS",
|
||||
"0000790526",
|
||||
"0000790526-23-000003",
|
||||
"rdnt-20221231.htm",
|
||||
"10-K",
|
||||
date(2023, 3, 1),
|
||||
date(2022, 12, 31),
|
||||
"8071",
|
||||
"RadNet, Inc. (RDNT)",
|
||||
)
|
||||
PASSAGE = T.PassageRow(
|
||||
"0000790526-23-000003:rdnt-20221231.htm:42",
|
||||
"0000790526",
|
||||
"0000790526-23-000003",
|
||||
"rdnt-20221231.htm",
|
||||
"RBRVS",
|
||||
42,
|
||||
"Our pay follows RBRVS rules.",
|
||||
"revenue_exposure",
|
||||
"model-reading exposure-v1 m",
|
||||
"K",
|
||||
)
|
||||
|
||||
PRICE = TI.PriceRow(
|
||||
"OSH",
|
||||
@@ -121,6 +148,16 @@ ROUNDTRIP = {
|
||||
lambda c, r: TI.write_series(c, r.series_id, [r]),
|
||||
lambda c: TI.read_series(c, ["OSH"]),
|
||||
),
|
||||
"filing_hit": (
|
||||
HIT,
|
||||
lambda c, r: T.write_hits(c, [r]),
|
||||
lambda c: T.read_hits(c),
|
||||
),
|
||||
"filing_passage": (
|
||||
PASSAGE,
|
||||
lambda c, r: T.write_passages(c, [r]),
|
||||
lambda c: T.read_passages(c),
|
||||
),
|
||||
"gap": (
|
||||
GAP,
|
||||
lambda c, r: T.write_gap(c, r.source, r.note),
|
||||
@@ -350,3 +387,24 @@ def test_ensure_tables_adds_priced_to_an_existing_security_table():
|
||||
(old,) = T.read_securities(c)
|
||||
assert old.priced is True
|
||||
c.close()
|
||||
|
||||
|
||||
def test_hits_and_passages_upsert_by_their_keys(con):
|
||||
T.write_hits(con, [HIT, dataclasses_replace(HIT, term="x")])
|
||||
T.write_hits(con, [dataclasses_replace(HIT, sics="8082")])
|
||||
assert sorted((h.term, h.sics) for h in T.read_hits(con)) == [
|
||||
("RBRVS", "8082"),
|
||||
("x", "8071"),
|
||||
]
|
||||
sibling = dataclasses_replace(PASSAGE, term="x", label="")
|
||||
T.write_passages(con, [PASSAGE, sibling])
|
||||
T.write_passages(con, [dataclasses_replace(PASSAGE, passage="new text", label="")])
|
||||
got = {p.term: p.passage for p in T.read_passages(con)}
|
||||
assert got == {"RBRVS": "new text", "x": PASSAGE.passage}
|
||||
assert T.write_hits(con, []) == 0 and T.write_passages(con, []) == 0
|
||||
|
||||
|
||||
def dataclasses_replace(row, **kw):
|
||||
import dataclasses
|
||||
|
||||
return dataclasses.replace(row, **kw)
|
||||
|
||||
Reference in New Issue
Block a user