From 6a45806759a86c29b253dc42a27d264366f21feb Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 02:19:01 -0500 Subject: [PATCH] feat(market): EDGAR full-text exposure passages per program and family, cited and AMA-guarded --- src/cli/market.py | 450 +++++++++++++++++++ src/market/edgar.py | 372 +++++++++++++++- src/market/http.py | 15 +- src/market/tables.py | 89 ++++ tests/cli/test_market_cli.py | 506 ++++++++++++++++++++++ tests/market/fixtures/efts_mpfs_2023.json | 140 ++++++ tests/market/fixtures/filing_snippet.htm | 7 + tests/market/test_fts.py | 387 +++++++++++++++++ tests/market/test_http.py | 36 ++ tests/market/test_tables.py | 58 +++ 10 files changed, 2053 insertions(+), 7 deletions(-) create mode 100644 tests/market/fixtures/efts_mpfs_2023.json create mode 100644 tests/market/fixtures/filing_snippet.htm create mode 100644 tests/market/test_fts.py diff --git a/src/cli/market.py b/src/cli/market.py index 42a9b58..e672bd2 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -13,9 +13,11 @@ from __future__ import annotations import dataclasses import datetime as dt +import time from collections import Counter from typing import Any +import httpx import typer from market import edgar, http, tiingo @@ -30,18 +32,26 @@ from market.seed import ( from market.tables import ( EntityRow, FilingRow, + HitRow, LinkRow, + PassageRow, SecurityRow, clear_gap, ensure_tables, + read_entities, read_gaps, + read_hits, + read_links, + read_passages, read_prices, retire_links, retire_organizations, write_entities, write_filings, write_gap, + write_hits, write_links, + write_passages, write_prices, write_securities, write_series, @@ -52,6 +62,7 @@ app = typer.Typer(no_args_is_help=True) ENTITIES_GAP = "sec-edgar:entities" PRICES_GAP = "tiingo:prices" +FTS_GAP = "sec-edgar:fts" # ── indirections the tests monkeypatch ─────────────────────────────── @@ -79,6 +90,17 @@ def _publish() -> None: typer.echo(f"replica → {publish_replica('aco')}") +def _asker() -> Any: + """The local JSON extractor on ``cfg.instruct_model`` (Ollama through the + host pool; never a cloud model).""" + from llm import config as llm_config + from llm.pool import HostPool + from llm.structured import json_extractor + + cfg = llm_config.load() + return json_extractor(cfg, HostPool.from_config(cfg)) + + # ── gaps ───────────────────────────────────────────────────────────── def _gap(source: str, note: str, *, write: bool) -> None: """Print ``gap: : ``; with *write* record it (and nothing @@ -570,6 +592,434 @@ def prices( typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows") +# ── full-text search ───────────────────────────────────────────────── +def _fts_scope( + families: list[str], programs: list[str] +) -> list[tuple[str, str, tuple[str, ...]]]: + """``(scope, target, terms)`` for the requested programs and families (all + programs when none is named).""" + out: list[tuple[str, str, tuple[str, ...]]] = [] + if not families and not programs: + programs = list(edgar.PROGRAM_TERMS) + for key in programs: + if key not in edgar.PROGRAM_TERMS: + raise typer.BadParameter(f"unknown program {key!r}") + out.append(("program", key, edgar.PROGRAM_TERMS[key])) + if families: + from pfs.families import FAMILIES, refresh_from + + con = _read() + try: + refresh_from(con) + finally: + con.close() + for key in families: + if key not in FAMILIES: + raise typer.BadParameter(f"unknown family {key!r}") + out.append(("family", key, edgar.family_terms(FAMILIES[key]))) + return out + + +def _ama_index() -> edgar.AmaIndex: + """The 8-word runs of every licensed CPT descriptor, all editions.""" + con = _read() + try: + return edgar.cpt_grams( + d + for (d,) in con.execute("SELECT descriptor FROM pfs.cpt_code").fetchall() + if d + ) + finally: + con.close() + + +def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRow]]: + return ( + _or_empty(read_hits, con_read, []), + _or_empty(read_passages, con_read, []), + _or_empty(read_entities, con_read, []), + ) + + +def _classify( + kept: list[PassageRow], limit: int +) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]: + """Label *kept* with the local model, resumably (``pfs.runlog``). Returns + the ledger outputs by unit, the run counts and the model name.""" + from pfs.runlog import RunLog, input_hash, ledger_path, run_units + + ask = _asker() + model = getattr(ask, "model", "local") + version = edgar.EXPOSURE_PROMPT_VERSION + log = RunLog(ledger_path("market-exposure", "all")) + + def unit(p: PassageRow) -> Any: + def run() -> dict | None: + label = edgar.classify_passage(p.passage, p.term, ask) + return {"label": label} if label else None + + return run + + units = [ + (f"{p.passage_id}|{p.term}", input_hash(p.passage, p.term), unit(p)) + for p in kept + ] + counts = run_units(units, log, model=model, version=version, limit=limit) + return log.outputs(model, version), counts, model + + +def _verified( + links: list[LinkRow], passage_rows: dict[Any, PassageRow] +) -> tuple[list[LinkRow], int]: + """M12: keep a link only when its passage occurs verbatim in the hoarded + filing (re-read from disk) and is at most 400 characters.""" + by_id = {p.passage_id: p for p in passage_rows.values()} + texts: dict[str, str | None] = {} + kept: list[LinkRow] = [] + for lk in links: + p = by_id.get(lk.passage_id) + if p is None: + continue + if (p.adsh, p.filename) not in texts: + path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename)) + texts[(p.adsh, p.filename)] = ( + edgar.read_doc(path) if path.is_file() else None + ) + text = texts[(p.adsh, p.filename)] + if text is not None and len(p.passage) <= 400 and p.passage in text: + kept.append(lk) + return kept, len(links) - len(kept) + + +def _new_entities( + ciks: list[str], seeds: dict[str, Any], issues: list[str] +) -> list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]]: + """An entity, its securities and its sweep-form filings for each CIK the + registry does not know yet (submissions are hoarded).""" + if not ciks: + return [] + out: list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]] = [] + pacer = edgar.new_pacer() + with http.client(edgar.SOURCE) as cl: + tickers = edgar.fetch_tickers(cl, pacer=pacer) + for cik in ciks: + sub = edgar.fetch_submissions(cl, f"CIK{cik}.json", pacer=pacer) + if sub is None: + issues.append(f"CIK{cik}: no submissions, no entity") + continue + seed = seeds.get(cik) + filings = [ + f + for f in edgar.filings_from_submissions(sub) + if f.form in edgar.SWEEP_FORMS + ] + out.append( + ( + edgar.entity_from_submissions(sub, seed), + edgar.securities_for(sub, seed, tickers.get(cik, [])), + filings, + ) + ) + return out + + +def _gap_note(issues: list[str]) -> str: + shown = "; ".join(issues[:15]) + more = f"; ... {len(issues) - 15} more" if len(issues) > 15 else "" + return f"{len(issues)} piece(s) missing: {shown}{more}" + + +@app.command() +def fts( + family: list[str] = typer.Option([], "--family", help="A family key (repeatable)."), + program: list[str] = typer.Option( + [], "--program", help="A program key (repeatable; all when none given)." + ), + forms: list[str] = typer.Option( + ["10-K"], "--forms", help="Filing form to search (repeatable)." + ), + from_year: int = typer.Option(edgar.FTS_FIRST_YEAR, "--from"), + to_year: int = typer.Option(0, "--to", help="Default: this year."), + limit: int = typer.Option( + 0, "--limit", help="At most N new filing downloads and model calls." + ), + max_minutes: float = typer.Option( + 0.0, "--max-minutes", help="Stop fetching after this long (rerun resumes)." + ), + refresh: bool = typer.Option(False, "--refresh", help="Download again."), + dry_run: bool = typer.Option(False, "--dry-run", help="List the queries only."), + classify: bool = typer.Option( + False, "--classify", help="Label passages with the local model." + ), + write: bool = typer.Option(False, "--write", help="Store hits, passages, links."), +) -> None: + """EDGAR full-text search for program and family terms in 10-K filings: + hits (market.filing_hit), verbatim passages of at most 400 characters + that pass the AMA guard (market.filing_passage) and ``fts`` links + (market.entity_link) for health-SIC and seed companies. A link needs a + kept passage. Everything fetched is hoarded; a rerun resumes.""" + to_year = to_year or dt.date.today().year + scope = _fts_scope(family, program) + owners: dict[str, list[tuple[str, str]]] = {} + for sc, target, terms in scope: + for t in terms: + owners.setdefault(t, []).append((sc, target)) + queries = [ + (t, f, y) for t in owners for f in forms for y in range(from_year, to_year + 1) + ] + if dry_run: + for sc, target, terms in scope: + n = len(terms) * len(forms) * (to_year - from_year + 1) + typer.echo(f"{sc} {target}: {len(terms)} terms, {n} queries") + cached = sum(edgar.fts_cached(*q) for q in queries) + typer.echo( + f"{len(queries)} queries ({cached} already hoarded), " + f"{edgar.FTS_PAGE} hits a page, at most 10 requests a second" + ) + return + + started = time.monotonic() + + def out_of_time() -> bool: + return bool(max_minutes) and time.monotonic() - started > max_minutes * 60 + + pacer = edgar.new_pacer() + prior_hits: list[HitRow] = [] + prior_passages: list[PassageRow] = [] + prior_entities: list[EntityRow] = [] + con = _read() + try: + prior_hits, prior_passages, prior_entities = _prior(con) + finally: + con.close() + companies, _orgs = load_seed() + seeds = {c.cik: c for c in companies} + seed_ciks = set(seeds) + health = set(edgar.HEALTH_SICS) + known = {e.key for e in prior_entities} + + new_hits: dict[tuple[str, str, str, str, str], HitRow] = {} + new_passages: list[PassageRow] = [] + issues: list[str] = [] + withheld = 0 + pending = 0 + incomplete = False + cl = http.client(edgar.SOURCE) + # a stalled connection should fail in seconds, not minutes; the retry + # opens a new one + cl.timeout = httpx.Timeout(30.0) + try: + for term, form, year in queries: + if out_of_time(): + incomplete = True + break + n = 0 + for _offset, payload in edgar.fetch_fts( + cl, term, form, year, pacer=pacer, refresh=refresh + ): + for sc, target in owners[term]: + for h in edgar.parse_hits( + payload, term=term, scope=sc, target=target + ): + new_hits[(h.term, h.scope, h.target, h.adsh, h.filename)] = h + n += 1 + typer.echo(f"{term} {form} {year}: {n} hits") + + def eligible(h: HitRow) -> bool: + return h.cik in seed_ciks or bool(set(h.sics.split(",")) & health) + + docs: dict[tuple[str, str, str], list[HitRow]] = {} + for h in new_hits.values(): + if eligible(h): + docs.setdefault((h.cik, h.adsh, h.filename), []).append(h) + ama = _ama_index() if docs else edgar.cpt_grams([]) + fetched = 0 + order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k)) + for cik, adsh, filename in order: + hs = docs[(cik, adsh, filename)] + if not filename.lower().endswith(edgar.READABLE): + issues.append(f"{adsh}:{filename}: format not readable as text") + continue + path = http.hoard("sec", "docs", cik, adsh, filename) + have = http.newest(path).is_file() + if not have and ((limit and fetched >= limit) or out_of_time()): + pending += 1 + continue + try: + got = edgar.fetch_doc(cl, hs[0], pacer=pacer, refresh=refresh) + except http.Refused as exc: + if exc.status in (0, 500, 502, 503, 504): + # one flaky document: skip it, a rerun retries + issues.append(f"{adsh}:{filename}: {exc}") + continue + raise + if got is None: + issues.append(f"{adsh}:{filename}: document not found") + continue + fetched += 0 if have else 1 + text = edgar.read_doc(got) + for h in {x.term: x for x in hs}.values(): + kept, held = edgar.safe_passages(text, h.term, ama) + withheld += held + for start, quote in kept: + new_passages.append( + PassageRow( + f"{adsh}:{filename}:{start}", + cik, + adsh, + filename, + h.term, + start, + quote, + ) + ) + if pending: + incomplete = True + issues.append(f"{pending} filing(s) not downloaded yet (limit or time)") + except http.Refused as exc: + _gap(FTS_GAP, str(exc), write=write) + return + finally: + cl.close() + + # the union of this run and what earlier runs stored + hits = {(h.term, h.scope, h.target, h.adsh, h.filename): h for h in prior_hits} + hits.update(new_hits) + redone = {(p.adsh, p.filename, p.term) for p in new_passages} + prior_by_id = {(p.passage_id, p.term): p for p in prior_passages} + merged: dict[tuple[str, str], PassageRow] = { + k: p + for k, p in prior_by_id.items() + if (p.adsh, p.filename, p.term) not in redone + } + for p in new_passages: + old = prior_by_id.get((p.passage_id, p.term)) + if old and old.passage == p.passage: + p = dataclasses.replace( + p, + label=old.label, + label_version=old.label_version, + item_key=old.item_key, + ) + merged[(p.passage_id, p.term)] = p + + if classify: + filed = {(h.adsh, h.filename): h.file_date for h in hits.values()} + todo = sorted( + (p for p in merged.values() if p.term in owners), + key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id), + ) + outputs, counts, model = _classify(todo, limit) + typer.echo(f"classified with {model}: {counts}") + for k, p in list(merged.items()): + got = outputs.get(f"{p.passage_id}|{p.term}") + if got: + merged[k] = dataclasses.replace( + p, + label=got["label"], + label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}", + ) + entities = list(prior_entities) + all_hits = list(hits.values()) + with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()} + linkable = [ + h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage + ] + links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks) + mine = {(sc, t) for sc, t, _ in scope} + links = [lk for lk in links if (lk.scope, lk.target) in mine] + links, unverified = _verified(links, merged) + if unverified: + issues.append(f"{unverified} link(s) dropped: quote not found in the hoard") + try: + new_entities = _new_entities( + sorted({lk.entity_key for lk in links} - known), seeds, issues + ) + except http.Refused as exc: + _gap(FTS_GAP, str(exc), write=write) + return + have = known | {e.key for e, _, _ in new_entities} + links = [lk for lk in links if lk.entity_key in have] + by_target = Counter(lk.target for lk in links) + typer.echo( + f"hits {len(new_hits)} (stored {len(all_hits)}); passages kept " + f"{len(new_passages)}, withheld by the AMA guard {withheld}; new entities " + f"{len(new_entities)}; fts links {len(links)} " + f"({', '.join(f'{k} {n}' for k, n in sorted(by_target.items())) or 'none'})" + ) + for i in issues[:20]: + typer.echo(f"gap: {FTS_GAP}: {i}") + if len(issues) > 20: + typer.echo(f"gap: {FTS_GAP}: ... {len(issues) - 20} more") + if not write: + typer.echo("dry run: pass --write to store") + return + if incomplete: + typer.echo("incomplete (limit or time): nothing written; rerun to resume") + return + + store = _store() + docs_of = {(h.adsh, h.filename): h for h in all_hits} + item_of: dict[tuple[str, str], str] = {} + for lk in links: + adsh, _, filename = lk.cite_ref.partition(":") + if (adsh, filename) not in item_of: + h = docs_of[(adsh, filename)] + item_of[(adsh, filename)] = edgar.cite_filing( + store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename)) + ) + links = [ + dataclasses.replace( + lk, + item_key=item_of[tuple(lk.cite_ref.split(":", 1))], # type: ignore[arg-type] + ) + for lk in links + ] + stamped = [ + dataclasses.replace(p, item_key=item_of[(p.adsh, p.filename)]) + if (p.adsh, p.filename) in item_of and not p.item_key + else p + for p in merged.values() + ] + + from cms.ingest_log import log_ingest + + sic_rows = [r for e, _, _ in new_entities for r in edgar.sic_links(e)] + with _batch() as con: + ensure_tables(con) + write_entities(con, [e for e, _, _ in new_entities]) + for e, secs, filings in new_entities: + write_securities(con, e.key, secs) + write_filings(con, e.cik, filings) + write_links(con, sic_rows) + write_hits(con, list(new_hits.values())) + write_passages(con, stamped) + write_links(con, links) + stored = {(lk.entity_key, lk.scope, lk.target) for lk in links} + old_fts = [lk for lk in read_links(con, basis="fts")] + keep = stored | { + (lk.entity_key, lk.scope, lk.target) + for lk in old_fts + if (lk.scope, lk.target) not in mine + } + retired = retire_links(con, "fts", {lk.entity_key for lk in old_fts}, keep) + if issues: + write_gap(con, FTS_GAP, _gap_note(issues)) + else: + clear_gap(con, FTS_GAP) + src = str(http.hoard(http.MANIFEST)) + for table, n in ( + ("market.filing_hit", len(new_hits)), + ("market.filing_passage", len(stamped)), + ("market.entity_link", len(links) + len(sic_rows)), + ): + log_ingest( + con, module="market.edgar", table_name=table, rows=n, source_file=src + ) + _publish() + typer.echo(f"stored: fts links {len(links)} (retired {retired})") + + @app.command() def gaps() -> None: """Sources that could not be read (market.source_gap).""" diff --git a/src/market/edgar.py b/src/market/edgar.py index 5db4f3c..ce22f69 100644 --- a/src/market/edgar.py +++ b/src/market/edgar.py @@ -22,7 +22,14 @@ import httpx from market import http from market.seed import Cite, SeedCompany, SeedOrg, norm_org -from market.tables import EntityRow, FilingRow, LinkRow, SecurityRow +from market.tables import ( + EntityRow, + FilingRow, + HitRow, + LinkRow, + PassageRow, + SecurityRow, +) TICKERS = "https://www.sec.gov/files/company_tickers.json" SUBMISSIONS = "https://data.sec.gov/submissions/{name}" @@ -325,21 +332,23 @@ def _fetch( refresh: bool, offline: bool, ok_404: bool = False, + params: dict[str, Any] | None = None, ) -> bytes | None: """The newest hoarded copy of *path*, downloading when there is none (or on *refresh*) and not *offline*. A changed body lands beside the old; a tolerated 404 is ``None`` and is not stored.""" have = http.newest(path) + shown = str(httpx.URL(url, params=params)) if params else url if have.is_file() and (offline or not refresh): - http.manifest(have, url) + http.manifest(have, shown) return have.read_bytes() if offline or cl is None: return None - r = http.get(cl, url, pacer=pacer, source=SOURCE, ok_404=ok_404) + r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=ok_404) if r.status_code == 404: return None saved = http.save(path, r.content) - http.manifest(saved, url) + http.manifest(saved, shown) return saved.read_bytes() @@ -535,3 +544,358 @@ def shares_ama_run( else AmaIndex.of(descriptors, n) ) return not grams.isdisjoint(index) + + +# ── full-text search ───────────────────────────────────────────────── + +EFTS = "https://efts.sec.gov/LATEST/search-index" +FTS_PAGE = 100 # hits per page; ``from`` is the offset +FTS_FIRST_YEAR = 2001 # EDGAR full-text search starts in 2001 + +PROGRAM_TERMS: dict[str, tuple[str, ...]] = { + "PFS": ( + "Medicare Physician Fee Schedule", + "physician fee schedule", + "RBRVS", + "Relative Value Scale Update Committee", + "CPT Editorial Panel", + ), + "QPP-MIPS": ("Merit-based Incentive Payment System", "Quality Payment Program"), + "MSSP": ("Medicare Shared Savings Program",), + "APM": ("Advanced Alternative Payment Model",), + "TELEHEALTH": ("Medicare telehealth",), + "HOME-BASED-CARE": ("home-based primary care", "home-based care"), + "CARE-MANAGEMENT": ("chronic care management", "transitional care management"), +} + + +def fts_params(term: str, form: str, year: int, offset: int) -> dict[str, Any]: + """One query per (term, form, year) keeps each under the 10,000-hit cap.""" + return { + "q": f'"{term}"', + "forms": form, + "dateRange": "custom", + "startdt": f"{year}-01-01", + "enddt": f"{year}-12-31", + "from": offset, + } + + +def family_terms(fam: Any) -> tuple[str, ...]: + """The family name plus every multi-word synonym (single words are too + common to search for).""" + out: dict[str, str] = {} + for t in (fam.name, *(s for s in fam.synonyms if len(s.split()) > 1)): + out.setdefault(t.lower(), t) + return tuple(out.values()) + + +def parse_hits( + payload: dict[str, Any], *, term: str, scope: str, target: str +) -> list[HitRow]: + """The hits of one search page; ``_id`` is ``"{adsh}:{filename}"``.""" + rows = [] + for hit in payload.get("hits", {}).get("hits", []): + src = hit["_source"] + adsh, _, filename = hit["_id"].partition(":") + names = src.get("display_names") or [""] + rows.append( + HitRow( + term=term, + scope=scope, + target=target, + cik=cik10(src["ciks"][0]), + adsh=adsh, + filename=filename, + form=src.get("form") or "", + file_date=dt.date.fromisoformat(src["file_date"]), + period_ending=_date(src.get("period_ending") or ""), + sics=",".join(src.get("sics") or []), + display_name=names[0], + ) + ) + return rows + + +def doc_url(cik: str, adsh: str, filename: str) -> str: + return ( + f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/" + f"{adsh.replace('-', '')}/{filename}" + ) + + +def fts_cache(term: str, form: str, year: int, offset: int) -> Path: + return http.hoard( + "sec", "fts", http.slug(term), form.replace("/", "_"), f"{year}.{offset}.json" + ) + + +def fts_cached(term: str, form: str, year: int) -> bool: + return http.newest(fts_cache(term, form, year, 0)).is_file() + + +def fetch_fts( + cl: httpx.Client | None, + term: str, + form: str, + year: int, + *, + pacer: http.Pacer, + refresh: bool = False, + offline: bool = False, +) -> Iterator[tuple[int, dict[str, Any]]]: + """Every page ``(offset, payload)`` of one (term, form, year) query, + hoarded at ``data/market/sec/fts//
/..json`` + and read from the hoard on a rerun.""" + offset = 0 + while True: + body = _fetch( + cl, + EFTS, + fts_cache(term, form, year, offset), + pacer=pacer, + refresh=refresh, + offline=offline, + params=fts_params(term, form, year, offset), + ) + if body is None: + return + payload = json.loads(body) + yield offset, payload + offset += FTS_PAGE + if offset >= payload.get("hits", {}).get("total", {}).get("value", 0): + return + + +#: documents we can read as text +READABLE = (".htm", ".html", ".txt") + + +def fetch_doc( + cl: httpx.Client | None, + hit: HitRow, + *, + pacer: http.Pacer, + refresh: bool = False, + offline: bool = False, +) -> Path | None: + """The hoarded copy of the hit's document (``sec/docs///``), + downloaded when absent and not *offline*; ``None`` when unavailable.""" + path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename) + body = _fetch( + cl, + doc_url(hit.cik, hit.adsh, hit.filename), + path, + pacer=pacer, + refresh=refresh, + offline=offline, + ok_404=True, + ) + return http.newest(path) if body is not None else None + + +def read_doc(path: Path) -> str: + return normalize_text(path.read_bytes().decode("utf-8", errors="replace")) + + +# ── passages ───────────────────────────────────────────────────────── + +_SENTENCE_END = (". ", "? ", "! ") + + +def _window(text: str, a: int, b: int, max_chars: int) -> tuple[int, int]: + """The sentence around ``text[a:b]`` when it fits *max_chars*, else a + window centred on the match, cut at word boundaries.""" + start = max(text.rfind(m, 0, a) for m in _SENTENCE_END) + 2 + start = start if start >= 2 else 0 + ends = [e for m in _SENTENCE_END if (e := text.find(m, b)) != -1] + end = min(ends) + 1 if ends else len(text) + if end - start <= max_chars: + return start, end + start = max(0, a - (max_chars - (b - a)) // 2) + end = min(len(text), start + max_chars) + start = max(0, end - max_chars) + if start > 0 and text[start - 1] != " ": + cut = text.find(" ", start, a) + start = cut + 1 if cut != -1 else start + if end < len(text) and text[end] != " ": + cut = text.rfind(" ", b, end) + end = cut if cut != -1 else end + return start, end + + +def _scan( + text: str, + term: str, + sizes: Sequence[int], + max_n: int, + grams: AmaIndex | None, +) -> tuple[list[tuple[int, str]], int]: + """Up to *max_n* kept ``(char_start, passage)`` and how many occurrences + were withheld. An occurrence is tried at each of *sizes* in turn (widest + first) until its passage shares no run with *grams*.""" + low, needle = text.lower(), term.lower() + out: list[tuple[int, str]] = [] + seen: set[int] = set() + withheld = 0 + pos = 0 + while len(out) < max_n and (i := low.find(needle, pos)) != -1: + pos = i + len(needle) + for size in sizes: + start, end = _window(text, i, pos, size) + chunk = text[start:end] + start += len(chunk) - len(chunk.lstrip()) + chunk = chunk.strip() + if grams is None or ama_safe(chunk, grams): + if start not in seen: + seen.add(start) + out.append((start, chunk)) + break + else: + withheld += 1 + return out, withheld + + +def passages( + text: str, term: str, *, max_chars: int = 400, max_n: int = 3 +) -> list[tuple[int, str]]: + """Up to *max_n* ``(char_start, passage)`` around case-insensitive matches + of *term* in *text*, each widened to its sentence within *max_chars*.""" + return _scan(text, term, (max_chars,), max_n, None)[0] + + +def safe_passages( + text: str, term: str, grams: AmaIndex, *, max_chars: int = 400, max_n: int = 3 +) -> tuple[list[tuple[int, str]], int]: + """:func:`passages` with the D3 guard: a passage that shares an 8-word run + with a CPT descriptor is retried at 60% and 30% of *max_chars*, then + withheld. Returns the kept passages and the number withheld.""" + sizes = (max_chars, max_chars * 3 // 5, max_chars * 3 // 10) + return _scan(text, term, sizes, max_n, grams) + + +def cpt_grams(descriptors: Iterable[str], n: int = 8) -> AmaIndex: + """The 8-word runs of every descriptor (no run crosses descriptors), + tokenized like ``pfs.origin.shares_run``.""" + return AmaIndex.of(descriptors, n) + + +def ama_safe(passage: str, grams: AmaIndex) -> bool: + """D3: *passage* shares no 8-word run with any CPT descriptor.""" + return not shares_ama_run(passage, grams) + + +# ── local classification ───────────────────────────────────────────── + +EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental") +EXPOSURE_PROMPT_VERSION = "exposure-v1" +EXPOSURE_SCHEMA: dict[str, Any] = { + "type": "object", + "properties": { + "label": {"type": "string", "enum": list(EXPOSURE_LABELS)}, + "reason": {"type": "string", "maxLength": 200}, + }, + "required": ["label", "reason"], +} +EXPOSURE_SYSTEM = ( + "You read one passage of a company's annual report that mentions a Medicare " + "program or payment term. Label how the passage ties the company to it: " + "revenue_exposure (the company says it is paid under, or earns revenue " + "through, the program or schedule), risk_factor (the company says changes " + "to the program could hurt or affect it), or incidental (the passage " + "defines, lists or mentions the term without tying the company to it). " + "Answer with JSON only: a label and a reason of at most 200 characters." +) + + +def classify_passage(passage: str, term: str, ask: Callable[..., Any]) -> str | None: + """A closed label for one passage from a local model, or ``None`` when + the reply is missing or outside :data:`EXPOSURE_LABELS`.""" + out = ask( + EXPOSURE_SYSTEM, + f"Term: {term}\nPassage: {passage}", + EXPOSURE_SCHEMA, + ) + label = out.get("label") if isinstance(out, dict) else None + return label if label in EXPOSURE_LABELS else None + + +# ── links ──────────────────────────────────────────────────────────── + +_RANK = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2} + + +def fts_links( + hits: Iterable[HitRow], + passage_rows: Iterable[PassageRow], + entities: Iterable[EntityRow], + seed_ciks: Iterable[str], +) -> list[LinkRow]: + """``basis="fts"`` links, one per (cik, scope, target): the strongest + evidence, then the newest filing. A hit counts when its SIC is a health + SIC or its CIK is in the seed. The strength is the passage label, or + ``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit + without a kept passage links with ``passage_id=""``.""" + seed = set(seed_ciks) + sic_of = {e.cik: e.sic for e in entities} + by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {} + for p in passage_rows: + by_doc.setdefault((p.cik, p.adsh, p.filename, p.term), []).append(p) + best: dict[tuple[str, str, str], tuple[tuple, LinkRow]] = {} + for h in hits: + sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")} + if h.cik not in seed and not (sics & set(HEALTH_SICS)): + continue + kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), []) + cands: list[tuple[str, str]] = [ + (p.label or "mention", p.passage_id) for p in kept + ] or [("mention", "")] + for strength, pid in cands: + if strength == "incidental": + continue + rank = (_RANK[strength], -h.file_date.toordinal(), pid == "", pid) + link = LinkRow( + h.cik, + h.scope, + h.target, + "fts", + strength, + "adsh", + f"{h.adsh}:{h.filename}", + pid, + ) + key = (h.cik, h.scope, h.target) + if key not in best or rank < best[key][0]: + best[key] = (rank, link) + return [link for _, link in sorted(best.values(), key=lambda b: b[1].entity_key)] + + +def cite_filing(store: Any, hit: HitRow, path: Path) -> str: + """The bib Download for the filing behind a hit (keyed by its document + URL, so a rerun returns the same key), with the document attached.""" + from bib.item import Download + from bib.tag import Tag + + url = doc_url(hit.cik, hit.adsh, hit.filename) + item = Download( + title=( + f"{hit.display_name} Form {hit.form} filed {hit.file_date} " + f"(accession {hit.adsh})" + ), + url=url, + file_urls=[url], + year=hit.file_date.year, + page_type="filing", + website_title="U.S. Securities and Exchange Commission", + ) + key = store.upsert( + item, + tags=[ + Tag.module("market"), + Tag.table("market.filing_passage"), + Tag.source("sec-edgar"), + Tag.year(hit.file_date.year), + ], + ) + store.attach_file(key, path, title=path.name) + return key diff --git a/src/market/http.py b/src/market/http.py index 904a847..70edfcd 100644 --- a/src/market/http.py +++ b/src/market/http.py @@ -123,15 +123,24 @@ def get( max_retry_after: float = 600.0, ok_404: bool = False, ) -> httpx.Response: - """GET with pacing. 5xx is retried with a ``2 * 2**n`` second backoff, - a 429 with a short ``Retry-After`` waits once, 401/403 stop the source, + """GET with pacing. A transport error (reset, timeout) and a 5xx are + retried with a ``2 * 2**n`` second backoff (a transport error that + outlasts the retries is a :class:`Refused` with status 0), a 429 with a short ``Retry-After`` waits once, 401/403 stop the source, and a 404 is returned only when *ok_404*. This is the market tier's one retry loop.""" attempt = 0 rate_waited = False while True: pacer.wait() - r = cl.get(url, params=params) + try: + r = cl.get(url, params=params) + except httpx.TransportError as exc: + # a dropped connection or a read timeout: back off and retry + if attempt < retries: + sleep(2.0 * 2**attempt) + attempt += 1 + continue + raise Refused(source, 0, type(exc).__name__) from exc status = r.status_code if status < 400: return r diff --git a/src/market/tables.py b/src/market/tables.py index 9077a14..5525abd 100644 --- a/src/market/tables.py +++ b/src/market/tables.py @@ -33,6 +33,16 @@ CREATE TABLE IF NOT EXISTS market.filing ( cik VARCHAR, adsh VARCHAR, form VARCHAR, filed DATE, accepted VARCHAR, report_date DATE, items VARCHAR, primary_doc VARCHAR ); +CREATE TABLE IF NOT EXISTS market.filing_hit ( + term VARCHAR, scope VARCHAR, target VARCHAR, cik VARCHAR, adsh VARCHAR, + filename VARCHAR, form VARCHAR, file_date DATE, period_ending DATE, + sics VARCHAR, display_name VARCHAR +); +CREATE TABLE IF NOT EXISTS market.filing_passage ( + passage_id VARCHAR, cik VARCHAR, adsh VARCHAR, filename VARCHAR, + term VARCHAR, char_start INTEGER, passage VARCHAR, label VARCHAR, + label_version VARCHAR, item_key VARCHAR +); CREATE TABLE IF NOT EXISTS market.source_gap ( source VARCHAR, note VARCHAR, recorded_on DATE ); @@ -128,6 +138,41 @@ class FilingRow: primary_doc: str +@dataclass(frozen=True) +class HitRow: + """One EDGAR full-text-search hit of *term*, owned by one *target* (a + program or family key).""" + + term: str + scope: str # program | family + target: str + cik: str + adsh: str + filename: str + form: str + file_date: date + period_ending: date | None + sics: str # comma-joined + display_name: str + + +@dataclass(frozen=True) +class PassageRow: + """At most 400 characters of public filing text around *term*; the + *label* is a model reading when ``label_version`` is set.""" + + passage_id: str # "{adsh}:{filename}:{char_start}" + cik: str + adsh: str + filename: str + term: str + char_start: int + passage: str + label: str = "" + label_version: str = "" + item_key: str = "" + + @dataclass(frozen=True) class GapRow: source: str @@ -359,6 +404,50 @@ def read_filings( return sorted((FilingRow(*r) for r in rows), key=lambda f: (f.filed, f.adsh)) +# ── filing_hit / filing_passage ────────────────────────────────────── + + +def write_hits(con: Any, rows: Sequence[HitRow]) -> int: + """Upsert by ``(term, scope, target, adsh, filename)``.""" + ensure_tables(con) + keys = sorted({(r.term, r.scope, r.target, r.adsh, r.filename) for r in rows}) + for part in _chunks(keys, 400): + con.execute( + "DELETE FROM market.filing_hit WHERE (term, scope, target, adsh, filename) " + "IN (SELECT * FROM (VALUES " + + ", ".join(["(?, ?, ?, ?, ?)"] * len(part)) + + "))", + [v for k in part for v in k], + ) + return _insert(con, "market.filing_hit", rows) + + +def read_hits(con: Any) -> list[HitRow]: + rows = _select(con, HitRow, "market.filing_hit") + return sorted( + (HitRow(*r) for r in rows), + key=lambda h: (h.scope, h.target, h.term, h.cik, h.adsh, h.filename), + ) + + +def write_passages(con: Any, rows: Sequence[PassageRow]) -> int: + """Replace the passages of each ``(adsh, filename, term)`` in *rows*.""" + ensure_tables(con) + keys = sorted({(r.adsh, r.filename, r.term) for r in rows}) + for part in _chunks(keys, 400): + con.execute( + "DELETE FROM market.filing_passage WHERE (adsh, filename, term) " + "IN (SELECT * FROM (VALUES " + ", ".join(["(?, ?, ?)"] * len(part)) + "))", + [v for k in part for v in k], + ) + return _insert(con, "market.filing_passage", rows) + + +def read_passages(con: Any) -> list[PassageRow]: + rows = _select(con, PassageRow, "market.filing_passage") + return sorted((PassageRow(*r) for r in rows), key=lambda p: p.passage_id) + + # ── source_gap ─────────────────────────────────────────────────────── diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index af92601..f89588f 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -1004,3 +1004,509 @@ def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch): assert "re-pulling from 2023-12-25" in res.output row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)] assert row.ret == pytest.approx(60.5 / 55 - 1) + + +# ── fts ────────────────────────────────────────────────────────────── +MPFS = '"Medicare Physician Fee Schedule"' +SNIPPET = (FX / "filing_snippet.htm").read_bytes() +LICENSED = "a licensed descriptor that no filing quote should ever copy word for word" + + +def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"): + return { + "_id": f"{adsh}:{filename}", + "_source": { + "ciks": [cik], + "adsh": adsh, + "form": "10-K", + "file_date": date_, + "period_ending": "2022-12-31", + "sics": sics, + "display_names": [f"Filer {cik} (CIK {cik})"], + }, + } + + +HITS = [ + _hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]), + _hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]), + _hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]), + _hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]), +] + + +def _fts_net( + monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None +): + """A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in + 2023; every other query is empty.""" + docs = docs if docs is not None else {"a10k.htm": SNIPPET} + requested: list[str] = [] + + def efts(m): + p = dict(httpx.URL(m.group(0)).params) + mine = p["q"] == MPFS and p["from"] == "0" + return httpx.Response( + status, + json={ + "hits": { + "total": {"value": len(hits) if mine else 0}, + "hits": hits if mine else [], + } + }, + ) + + def doc(m): + requested.append(m.group(1)) + if broken and m.group(1) in broken: + if broken[m.group(1)] == "timeout": + raise httpx.ReadTimeout("slow") + return httpx.Response(broken[m.group(1)]) + body = docs.get(m.group(1)) + if forbid_docs and body: + raise AssertionError("no document download expected") + return httpx.Response(200, content=body) if body else httpx.Response(404) + + def sub(m): + return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]}) + + _online( + monkeypatch, + [ + (r"efts\.sec\.gov/.*", efts), + (r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc), + ( + r"company_tickers\.json", + lambda m: httpx.Response( + 200, content=(FX / "company_tickers.json").read_bytes() + ), + ), + (r"CIK(\d{10})\.json", sub), + ], + ) + return requested + + +FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"] + + +def test_fts_cli_dry_run_lists_queries(env, monkeypatch): + c, _, published, hoard = env + monkeypatch.setattr( + http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net")) + ) + res = runner.invoke(app, [*FTS, "--dry-run"]) + assert res.exit_code == 0, res.output + assert "program PFS: 5 terms, 5 queries" in res.output + assert "5 queries (0 already hoarded)" in res.output + assert published == [] + full = runner.invoke(app, ["market", "fts", "--dry-run"]) + assert "program MSSP: 1 terms" in full.output + assert "program CARE-MANAGEMENT: 2 terms" in full.output + assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output + + +def dt_year(): + import datetime + + return datetime.date.today().year + + +def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch): + monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0) + res = runner.invoke( + app, + [ + "market", + "fts", + "--family", + "HOME-VISITS", + "--dry-run", + "--from", + "2023", + "--to", + "2023", + ], + ) + assert res.exit_code == 0, res.output + assert "family HOME-VISITS: 8 terms, 8 queries" in res.output + assert ( + runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code + == 2 + ) + assert ( + runner.invoke( + app, ["market", "fts", "--program", "NOPE", "--dry-run"] + ).exit_code + == 2 + ) + + +def test_fts_cli_without_write_stores_nothing(env, monkeypatch): + c, _, published, hoard = env + _fts_net(monkeypatch) + monkeypatch.setattr( + mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch")) + ) + res = runner.invoke(app, FTS) + assert res.exit_code == 0, res.output + assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output + assert "passages kept 1," in res.output and "new entities 1" in res.output + assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output + assert ( + "gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable" + in res.output + ) + assert ( + "gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found" + in res.output + ) + assert published == [] + # hoarded: the pages and the one downloaded readable document + assert ( + hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json" + ).is_file() + assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file() + + +def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch): + c, store, published, hoard = env + curated = T.LinkRow( + "0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K" + ) + T.write_links(c, [curated]) + _fts_net(monkeypatch) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + (ent,) = T.read_entities(c, "company") + assert ent.key == "0000012345" and ent.sic == "8082" + assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"] + assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"] + assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"] + assert T.read_links(c, basis="curated") == [curated] # survives + (fts_link,) = T.read_links(c, basis="fts") + assert (fts_link.scope, fts_link.target, fts_link.strength) == ( + "program", + "PFS", + "mention", + ) + assert fts_link.item_key and fts_link.passage_id.startswith( + "0000012345-23-000001:a10k.htm:" + ) + # the link's passage is verbatim in the hoarded filing, short, and cited + (p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id] + assert len(p.passage) <= 400 and p.item_key == fts_link.item_key + text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm") + assert p.passage in text and p.label == "" and p.label_version == "" + assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1 + assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] + assert "2 piece(s) missing" in T.read_gaps(c)[0].note + logs = { + t + for (t,) in c.execute( + "SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'" + ).fetchall() + } + assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs + assert published == [True] and store.count() == 1 + # a rerun is idempotent and downloads nothing new + n_docs = _fts_net(monkeypatch, forbid_docs=True) + assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0 + assert ( + _count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1 + ) + assert n_docs == ["gone.htm"] # a 404 is retried, never stored + + +def test_fts_cli_refusal_prints_gap(env, monkeypatch): + c, _, published, _ = env + _fts_net(monkeypatch, status=403) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "gap: sec-edgar:fts:" in res.output and "403" in res.output + assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] + assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0 + assert published == [True] + + +def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch): + c, _, _, _ = env + doc = f"

The Medicare Physician Fee Schedule: {LICENSED}.

".encode() + _fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc}) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "passages kept 0, withheld by the AMA guard 1" in res.output + assert "fts links 0 (none)" in res.output + assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == [] + assert len(T.read_hits(c)) == 1 # the hit stays + + +def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch): + c, _, _, hoard = env + _fts_net(monkeypatch, hits=HITS[:1]) + real = mk._verified + + def tamper(links, passages): + for p in passages.values(): + p.__dict__["passage"] = "not in the filing" + return real(links, passages) + + monkeypatch.setattr(mk, "_verified", tamper) + res = runner.invoke(app, [*FTS, "--write"]) + assert "fts links 0" in res.output and "quote not found in the hoard" in res.output + assert T.read_links(c, basis="fts") == [] + + +def test_verified_handles_missing_filing_and_unpassaged_links(env): + _, _, _, hoard = env + link = T.LinkRow( + "1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0" + ) + p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x") + assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1) + bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "") + assert mk._verified([bare], {}) == ([], 1) + + +def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch): + c, _, published, _ = env + hits = [ + HITS[0], + _hit_json( + "0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01" + ), + ] + _fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET}) + res = runner.invoke(app, [*FTS, "--limit", "1", "--write"]) + assert res.exit_code == 0, res.output + assert "1 filing(s) not downloaded yet" in res.output + assert "incomplete (limit or time): nothing written" in res.output + assert published == [] + # the newest filing was fetched first + assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm"))) + + +def test_fts_cli_deadline_stops_fetching(env, monkeypatch): + c, _, published, _ = env + _fts_net(monkeypatch) + ticks = iter(range(0, 10_000, 100)) + monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks)) + res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"]) + assert res.exit_code == 0, res.output + assert "incomplete" in res.output and published == [] + + +def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch): + c, _, _, _ = env + _fts_net(monkeypatch) + assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0 + (p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")] + c.execute( + "UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' " + "WHERE passage_id = ?", + [p.passage_id], + ) + _fts_net(monkeypatch, forbid_docs=True) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0 and "new entities 0" in res.output + got = {q.passage_id: q for q in T.read_passages(c)} + assert ( + got[p.passage_id].label == "risk_factor" + and got[p.passage_id].label_version == "v" + ) + (link,) = T.read_links(c, basis="fts") + assert link.strength == "risk_factor" and link.passage_id == p.passage_id + + +def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch): + c, _, _, _ = env + _fts_net(monkeypatch, hits=HITS[:1]) + _online( + monkeypatch, + [ + ( + r"efts\.sec\.gov/.*", + lambda m: ( + httpx.Response( + 200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}} + ) + if "Medicare+Physician" in m.group(0) + or "Medicare%20Physician" in m.group(0) + else httpx.Response( + 200, json={"hits": {"total": {"value": 0}, "hits": []}} + ) + ), + ), + (r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)), + ( + r"company_tickers\.json", + lambda m: httpx.Response( + 200, content=(FX / "company_tickers.json").read_bytes() + ), + ), + ], + ) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "CIK0000012345: no submissions, no entity" in res.output + assert T.read_entities(c) == [] + + +# classification: the local model, resumable through the ledger +class _Ask: + model = "qwen-test" + + def __init__(self, answers): + self.answers = answers + self.calls = 0 + + def __call__(self, system, user, schema): + self.calls += 1 + return self.answers.pop(0) + + +TWO = [ + HITS[0], + _hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"), +] +TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET} + + +def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path): + c, _, _, _ = env + monkeypatch.setattr( + "pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl" + ) + # a passage of a term outside this run's scope is never sent to the model + T.write_passages( + c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")] + ) + ask = _Ask( + [ + {"label": "revenue_exposure", "reason": "r"}, + None, + {"label": "incidental", "reason": "i"}, + ] + ) + monkeypatch.setattr(mk, "_asker", lambda: ask) + _fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS) + assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first + res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"]) + assert res.exit_code == 0, res.output + assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1 + labelled = [p for p in T.read_passages(c) if p.label] + assert len(labelled) == 1 and labelled[0].label == "revenue_exposure" + assert labelled[0].label_version == "model-reading exposure-v1 qwen-test" + # resume: the done unit is skipped; the next call fails (None) and stays unlabelled + res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"]) + assert res.exit_code == 0 and ask.calls == 2 + assert "'skipped': 1" in res.output and "'failed': 1" in res.output + # a rerun without --classify keeps what is stored + res = runner.invoke(app, [*FTS, "--write"]) + assert len([p for p in T.read_passages(c) if p.label]) == 1 + got = sorted(l.strength for l in T.read_links(c, basis="fts")) + assert got == ["mention", "revenue_exposure"] + + +def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path): + c, _, _, _ = env + monkeypatch.setattr( + "pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl" + ) + ask = _Ask([{"label": "incidental", "reason": "i"}] * 2) + monkeypatch.setattr(mk, "_asker", lambda: ask) + _fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS) + res = runner.invoke(app, [*FTS, "--classify", "--write"]) + assert res.exit_code == 0, res.output + assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2 + + +def test_asker_builds_json_extractor(monkeypatch): + sentinel = object() + seen = {} + monkeypatch.setattr("llm.config.load", lambda: "cfg") + monkeypatch.setattr( + "llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg)) + ) + + def make(cfg, pool): + seen["args"] = (cfg, pool) + return sentinel + + monkeypatch.setattr("llm.structured.json_extractor", make) + assert mk._asker() is sentinel + assert seen["args"] == ("cfg", ("pool", "cfg")) + + +def test_gap_note_truncates(): + note = mk._gap_note([f"i{n}" for n in range(20)]) + assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note + + +def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch): + c, _, published, _ = env + _fts_net(monkeypatch, hits=HITS[:1]) + _online( + monkeypatch, + [ + ( + r"efts\.sec\.gov/.*", + lambda m: httpx.Response( + 200, + json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}} + if "Medicare%20Physician" in m.group(0) + or "Medicare+Physician" in m.group(0) + else {"hits": {"total": {"value": 0}, "hits": []}}, + ), + ), + (r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)), + (r"CIK\d{10}\.json", lambda m: httpx.Response(403)), + (r"company_tickers\.json", lambda m: httpx.Response(403)), + ], + ) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "gap: sec-edgar:fts:" in res.output and "403" in res.output + assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] + assert _count(c, "market.filing_hit") == 0 and published == [True] + + +def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch): + c, _, _, _ = env + hits = [ + _hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"]) + for n in range(22) + ] + _fts_net(monkeypatch, hits=hits) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "gap: sec-edgar:fts: ... 2 more" in res.output + assert "... 7 more" in T.read_gaps(c)[0].note + + +def test_fts_cli_flaky_document_is_skipped(env, monkeypatch): + hits = [ + HITS[0], + _hit_json( + "0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01" + ), + ] + _fts_net( + monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"} + ) + monkeypatch.setattr("time.sleep", lambda s: None) + res = runner.invoke(app, FTS) + assert res.exit_code == 0, res.output + assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output + assert "passages kept 1," in res.output + + +def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch): + c, _, published, _ = env + _fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403}) + res = runner.invoke(app, [*FTS, "--write"]) + assert res.exit_code == 0, res.output + assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output + assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [ + True + ] diff --git a/tests/market/fixtures/efts_mpfs_2023.json b/tests/market/fixtures/efts_mpfs_2023.json new file mode 100644 index 0000000..dadd267 --- /dev/null +++ b/tests/market/fixtures/efts_mpfs_2023.json @@ -0,0 +1,140 @@ +{ + "hits": { + "total": { + "value": 57, + "relation": "eq" + }, + "max_score": 8.786024, + "hits": [ + { + "_index": "edgar_file", + "_id": "0000790526-23-000003:rdnt-20221231.htm", + "_score": 8.786024, + "_source": { + "ciks": [ + "0000790526" + ], + "period_ending": "2022-12-31", + "file_num": [ + "001-33307" + ], + "display_names": [ + "RadNet, Inc. (RDNT) (CIK 0000790526)" + ], + "xsl": null, + "sequence": 1, + "root_forms": [ + "10-K" + ], + "file_date": "2023-03-01", + "biz_states": [ + "CA" + ], + "sics": [ + "8071" + ], + "form": "10-K", + "adsh": "0000790526-23-000003", + "film_num": [ + "23695229" + ], + "biz_locations": [ + "Los Angeles, CA" + ], + "file_type": "10-K", + "file_description": "10-K", + "inc_states": [ + "DE" + ], + "items": [] + } + }, + { + "_index": "edgar_file", + "_id": "0001798270-23-000007:ionm-20221231x10k.htm", + "_score": 8.094215, + "_source": { + "ciks": [ + "0001798270" + ], + "period_ending": "2022-12-31", + "file_num": [ + "001-40785" + ], + "display_names": [ + "Assure Holdings Corp. (IONM) (CIK 0001798270)" + ], + "xsl": null, + "sequence": 1, + "root_forms": [ + "10-K" + ], + "file_date": "2023-03-31", + "biz_states": [ + "CO" + ], + "sics": [ + "3841" + ], + "form": "10-K", + "adsh": "0001798270-23-000007", + "film_num": [ + "23788435" + ], + "biz_locations": [ + "Greenwood Village, CO" + ], + "file_type": "10-K", + "file_description": "10-K", + "inc_states": [ + "NV" + ], + "items": [] + } + }, + { + "_index": "edgar_file", + "_id": "0000355019-23-000044:fonar_10-k.htm", + "_score": 7.894161, + "_source": { + "ciks": [ + "0000355019" + ], + "period_ending": "2023-06-30", + "file_num": [ + "000-10248" + ], + "display_names": [ + "FONAR CORP (FONR) (CIK 0000355019)" + ], + "xsl": null, + "sequence": 1, + "root_forms": [ + "10-K" + ], + "file_date": "2023-09-28", + "biz_states": [ + "NY" + ], + "sics": [ + "3845" + ], + "form": "10-K", + "adsh": "0000355019-23-000044", + "film_num": [ + "231289120" + ], + "biz_locations": [ + "Melville, NY" + ], + "file_type": "10-K", + "file_description": "FONAR CORPORATION FORM 10-K AT JUNE 30, 2023", + "inc_states": [ + "DE" + ], + "items": [] + } + } + ] + } +} \ No newline at end of file diff --git a/tests/market/fixtures/filing_snippet.htm b/tests/market/fixtures/filing_snippet.htm new file mode 100644 index 0000000..d64ea44 --- /dev/null +++ b/tests/market/fixtures/filing_snippet.htm @@ -0,0 +1,7 @@ + +HIDDEN XBRL TEXT physician fee schedule +

Item 1. Business. We operate imaging centers across several states.

+

Substantially all of our Medicare revenue is paid under the Medicare Physician Fee Schedule, and the Centers for Medicare & Medicaid Services’ annual updates to the conversion factor affect our results. We cannot predict future rates.

+

Our contracts with commercial payors are described in Item 7.

+

Changes to the physician fee schedule could reduce our reimbursement.

+ diff --git a/tests/market/test_fts.py b/tests/market/test_fts.py new file mode 100644 index 0000000..ef27d3b --- /dev/null +++ b/tests/market/test_fts.py @@ -0,0 +1,387 @@ +"""market.edgar full-text search: queries, hits, passages, the AMA guard, +the local classifier and the links they produce.""" + +from __future__ import annotations + +import json +from datetime import date +from pathlib import Path + +import httpx +import pytest + +from bib.store import Store +from market import edgar, http +from market import tables as T +from market.seed import PROGRAMS + +FX = Path(__file__).parent / "fixtures" + + +class _Pacer: + def wait(self): + pass + + +@pytest.fixture +def hoard(monkeypatch, tmp_path): + monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market") + return tmp_path / "market" + + +def test_fts_params_one_year(): + p = edgar.fts_params("Medicare Physician Fee Schedule", "10-K", 2023, 0) + assert p["q"] == '"Medicare Physician Fee Schedule"' + assert p["startdt"] == "2023-01-01" and p["enddt"] == "2023-12-31" + assert p["from"] == 0 and p["forms"] == "10-K" and p["dateRange"] == "custom" + + +def test_parse_hits(): + payload = json.loads((FX / "efts_mpfs_2023.json").read_text()) + rows = edgar.parse_hits( + payload, term="Medicare Physician Fee Schedule", scope="program", target="PFS" + ) + assert len(rows) == 3 + assert rows[0] == T.HitRow( + term="Medicare Physician Fee Schedule", + scope="program", + target="PFS", + cik="0000790526", + adsh="0000790526-23-000003", + filename="rdnt-20221231.htm", + form="10-K", + file_date=date(2023, 3, 1), + period_ending=date(2022, 12, 31), + sics="8071", + display_name="RadNet, Inc. (RDNT) (CIK 0000790526)", + ) + + +def test_parse_hits_without_optional_fields(): + payload = { + "hits": { + "hits": [ + { + "_id": "0000000001-23-000001:a.htm", + "_source": {"ciks": ["1"], "file_date": "2023-01-02"}, + } + ] + } + } + (h,) = edgar.parse_hits(payload, term="t", scope="family", target="CCM") + assert (h.cik, h.period_ending, h.sics, h.display_name, h.form) == ( + "0000000001", + None, + "", + "", + "", + ) + assert edgar.parse_hits({}, term="t", scope="program", target="PFS") == [] + + +def _pages(total): + def handler(request: httpx.Request): + q = dict(request.url.params) + return httpx.Response( + 200, + json={ + "hits": { + "total": {"value": total}, + "hits": [ + { + "_id": f"0000000001-23-00000{q['from']}:a.htm", + "_source": {"ciks": ["1"], "file_date": "2023-01-02"}, + } + ], + } + }, + ) + + return handler + + +def test_fts_pages_until_total(hoard): + cl = httpx.Client(transport=httpx.MockTransport(_pages(150))) + got = list(edgar.fetch_fts(cl, "RBRVS", "10-K", 2023, pacer=_Pacer())) + assert [off for off, _ in got] == [0, 100] + cached = hoard / "sec" / "fts" / "rbrvs" / "10-K" / "2023.100.json" + assert cached.is_file() + assert edgar.fts_cached("RBRVS", "10-K", 2023) + assert not edgar.fts_cached("RBRVS", "10-K", 2022) + + +def test_fts_pages_come_from_the_hoard_and_offline_stops(hoard): + cl = httpx.Client(transport=httpx.MockTransport(_pages(50))) + first = list(edgar.fetch_fts(cl, "RBRVS", "10-K/A", 2023, pacer=_Pacer())) + assert (hoard / "sec/fts/rbrvs/10-K_A/2023.0.json").is_file() + again = list( + edgar.fetch_fts(None, "RBRVS", "10-K/A", 2023, pacer=_Pacer(), offline=True) + ) + assert again == first + assert ( + list(edgar.fetch_fts(None, "RBRVS", "10-K", 1999, pacer=_Pacer(), offline=True)) + == [] + ) + + +def test_doc_url(): + assert ( + edgar.doc_url("0000790526", "0000790526-23-000003", "rdnt-20221231.htm") + == "https://www.sec.gov/Archives/edgar/data/790526/000079052623000003/" + "rdnt-20221231.htm" + ) + + +def test_family_terms(): + from pfs.families import FAMILIES + + terms = edgar.family_terms(FAMILIES["HOME-VISITS"]) + low = [t.lower() for t in terms] + assert "house calls" in low and "home or residence" in low + assert "domiciliary" not in low + assert len(low) == len(set(low)) and terms[0] == FAMILIES["HOME-VISITS"].name + + +def test_program_terms_keys_are_programs(): + assert set(edgar.PROGRAM_TERMS) == set(PROGRAMS) + assert "RBRVS" in edgar.PROGRAM_TERMS["PFS"] + + +def test_passages_window_and_cap(): + text = ( + "x " * 500 + + "We bill under the Medicare Physician Fee Schedule today. " + + "y " * 500 + ) + at = text.index("Medicare Physician") + (one, *_) = edgar.passages(text, "medicare physician fee schedule") + start, p = one + assert start <= at < start + len(p) and len(p) <= 400 + assert "Medicare Physician Fee Schedule" in p + assert text[start : start + len(p)] == p + # a short sentence is returned whole + short = "Intro. Our pay follows RBRVS rules. Next." + assert edgar.passages(short, "rbrvs") == [(7, "Our pay follows RBRVS rules.")] + many = " ".join(f"{'filler ' * 200} RBRVS applies." for _ in range(5)) + got = edgar.passages(many, "rbrvs") + assert len(got) == 3 and all(len(p) <= 400 for _, p in got) + assert edgar.passages("nothing here", "rbrvs") == [] + + +def test_passages_dedupe_same_sentence(): + got = edgar.passages("Both RBRVS and RBRVS appear here.", "rbrvs") + assert got == [(0, "Both RBRVS and RBRVS appear here.")] + + +def test_normalize_and_snippet_passages(): + text = edgar.normalize_text((FX / "filing_snippet.htm").read_text()) + assert "HIDDEN" not in text and "color: red" not in text + assert "Medicaid Services' annual" in text # entity + curly quote decoded + (a, b) = edgar.passages(text, "physician fee schedule") + assert a[1].startswith("Substantially all of our Medicare revenue") + assert ( + b[1] == "Changes to the physician fee schedule could reduce our reimbursement." + ) + assert edgar.quote_in(text, a[1]) + + +DESC = "Office or other outpatient visit for the evaluation and management of an established patient" + + +def test_ama_guard_matches_shares_run(): + from pfs.origin import shares_run + + pairs = [ + ("We see patients: " + DESC + " each day.", DESC), + ("Office visits are our business and management of patients too", DESC), + ("short", DESC), + ] + for passage, d in pairs: + assert edgar.ama_safe(passage, edgar.cpt_grams([d])) == ( + not shares_run(passage, d) + ) + assert edgar.cpt_grams(["a b c"], 8) == frozenset() + + +def test_safe_passages_retry_narrower_then_withhold(): + grams = edgar.cpt_grams([DESC]) + long = f"{'pad ' * 60}Our term RBRVS is used. {DESC} is one service. {'pad ' * 60}" + kept, held = edgar.safe_passages(long, "rbrvs", grams) + assert held == 0 and kept and "RBRVS" in kept[0][1] + assert all(edgar.ama_safe(p, grams) for _, p in kept) + # a term inside the descriptor cannot be separated from it + inside = f"{DESC} for RBRVS" + short = f"The text {DESC}." + assert edgar.safe_passages(short, "evaluation", grams) == ([], 1) + assert edgar.safe_passages(inside, "rbrvs", grams)[1] in (0, 1) + + +def test_passage_quoting_descriptor_without_code_withheld(): + grams = edgar.cpt_grams([DESC]) + text = f"We provide {DESC}. This is billed under RBRVS." + assert edgar.safe_passages(text, "established patient", grams) == ([], 1) + h = T.HitRow( + "established patient", + "program", + "PFS", + "0000790526", + "0000790526-23-000003", + "a.htm", + "10-K", + date(2023, 3, 1), + None, + "8071", + "RadNet", + ) + (link,) = edgar.fts_links([h], [], [], []) + assert link.passage_id == "" and link.strength == "mention" + assert link.cite_ref == "0000790526-23-000003:a.htm" + + +def _hit(cik="0000790526", sics="8071", adsh="0000790526-23-000003", **kw): + base = dict( + term="rbrvs", + scope="program", + target="PFS", + cik=cik, + adsh=adsh, + filename="a.htm", + form="10-K", + file_date=date(2023, 3, 1), + period_ending=None, + sics=sics, + display_name="X", + ) + base.update(kw) + return T.HitRow(**base) + + +def _passage(h, label="", start=10): + return T.PassageRow( + f"{h.adsh}:{h.filename}:{start}", + h.cik, + h.adsh, + h.filename, + h.term, + start, + "text", + label, + ) + + +def test_fts_links_require_health_sic_or_seed(): + ok, other = _hit(), _hit(cik="0000000002", sics="6798") + links = edgar.fts_links([ok, other], [], [], []) + assert [l.entity_key for l in links] == ["0000790526"] + assert links[0].basis == "fts" and links[0].cite_kind == "adsh" + # a seed CIK links whatever its SIC; an entity's own SIC counts too + assert ( + edgar.fts_links([other], [], [], ["0000000002"])[0].entity_key == "0000000002" + ) + ent = T.EntityRow("0000000002", "E", "company", cik="0000000002", sic="8082") + assert len(edgar.fts_links([_hit(cik="0000000002", sics="")], [], [ent], [])) == 1 + assert edgar.fts_links([_hit(cik="0000000002", sics="")], [], [], []) == [] + + +def test_link_strength_from_label(): + h = _hit() + for label, want in (("", "mention"), ("revenue_exposure", "revenue_exposure")): + (l,) = edgar.fts_links([h], [_passage(h, label)], [], []) + assert l.strength == want and l.passage_id == f"{h.adsh}:a.htm:10" + assert edgar.fts_links([h], [_passage(h, "incidental")], [], []) == [] + + +def test_best_evidence_wins_per_target(): + old = _hit(adsh="0000790526-20-000001", file_date=date(2020, 3, 1)) + new = _hit(adsh="0000790526-23-000003") + got = edgar.fts_links( + [old, new], + [_passage(old, "risk_factor"), _passage(new, "revenue_exposure")], + [], + [], + ) + assert len(got) == 1 and got[0].strength == "revenue_exposure" + # equal strength: the newest filing + got = edgar.fts_links([old, new], [_passage(old), _passage(new)], [], []) + assert got[0].cite_ref.startswith("0000790526-23") + # a labelled passage beats an unlabelled sibling + got = edgar.fts_links( + [new], [_passage(new, "", 5), _passage(new, "risk_factor", 9)], [], [] + ) + assert got[0].strength == "risk_factor" + + +def test_classify_passage_closed_labels(): + seen = [] + + def ask(system, user, schema): + seen.append((system, user, schema)) + return {"label": "revenue_exposure", "reason": "paid under it"} + + assert edgar.classify_passage("p", "rbrvs", ask) == "revenue_exposure" + assert "rbrvs" in seen[0][1] and seen[0][2]["properties"]["label"]["enum"] == list( + edgar.EXPOSURE_LABELS + ) + assert edgar.classify_passage("p", "t", lambda *a: {"label": "bogus"}) is None + assert edgar.classify_passage("p", "t", lambda *a: None) is None + assert edgar.EXPOSURE_PROMPT_VERSION == "exposure-v1" + + +def test_fetch_doc_hoards_and_reads(hoard): + h = _hit() + calls = [] + + def handler(request): + calls.append(str(request.url)) + if request.url.path.endswith("gone.htm"): + return httpx.Response(404) + return httpx.Response(200, content=(FX / "filing_snippet.htm").read_bytes()) + + cl = httpx.Client(transport=httpx.MockTransport(handler)) + path = edgar.fetch_doc(cl, h, pacer=_Pacer()) + assert path == hoard / "sec/docs/0000790526/0000790526-23-000003/a.htm" + assert "Substantially all" in edgar.read_doc(path) + again = edgar.fetch_doc(cl, h, pacer=_Pacer()) + assert again == path and len(calls) == 1 # never refetched + gone = edgar.fetch_doc(cl, _hit(filename="gone.htm"), pacer=_Pacer()) + assert gone is None + assert ( + edgar.fetch_doc(None, _hit(filename="x.htm"), pacer=_Pacer(), offline=True) + is None + ) + + +def test_cite_filing_attaches_document(tmp_path): + store = Store(":memory:", storage_dir=tmp_path / "st") + doc = tmp_path / "a.htm" + doc.write_text("doc") + h = _hit(display_name="RadNet, Inc.") + key = edgar.cite_filing(store, h, doc) + assert edgar.cite_filing(store, h, doc) == key + item = store.get(key) + assert item.title == ( + "RadNet, Inc. Form 10-K filed 2023-03-01 (accession 0000790526-23-000003)" + ) + assert item.website_title == "U.S. Securities and Exchange Commission" + names = {t["name"] if isinstance(t, dict) else t[0] for t in store.list_tags()} + assert {"module:market", "source:sec-edgar", "year:2023"} <= names + assert "table:market.filing_passage" in names + n = store._con().execute("SELECT count(*) FROM attachments").fetchone()[0] + assert n == 1 + store.close() + + +def test_window_snaps_to_word_boundaries_in_a_long_sentence(): + words = " ".join(f"word{i}" for i in range(400)) # one 2,800-char sentence + mid = words.index("word200") + (start, p), *_ = edgar.passages(words + ".", "word200") + assert len(p) <= 400 and p.startswith("word") and p.split()[-1].startswith("word") + assert start <= mid and "word200" in p + # a run with no spaces cannot be snapped: still capped, term kept + blob = "z" * 900 + "RBRVS" + "z" * 900 + (start, p), *_ = edgar.passages(blob, "rbrvs") + assert len(p) == 400 and "RBRVS" in p and blob[start : start + 400] == p + # near the end of the text the window is pulled back to full size + tail = ("q " * 600) + "RBRVS" + (start, p), *_ = edgar.passages(tail, "rbrvs") + assert p.endswith("RBRVS") and len(p) <= 400 diff --git a/tests/market/test_http.py b/tests/market/test_http.py index 7747dea..525c0e5 100644 --- a/tests/market/test_http.py +++ b/tests/market/test_http.py @@ -109,6 +109,42 @@ def test_get_5xx_exhausted_raises_refused(): assert ei.value.status == 502 and slept == [2.0, 4.0] +def test_get_retries_transport_errors_then_refuses(): + slept = [] + calls = [] + + def boom(request): + calls.append(1) + raise httpx.ReadTimeout("slow", request=request) + + with pytest.raises(Refused) as ei: + get( + _cl(boom), + "https://x.test/a", + pacer=_Pacer(), + source="s", + sleep=slept.append, + retries=2, + ) + assert ei.value.status == 0 and "ReadTimeout" in str(ei.value) + assert slept == [2.0, 4.0] and len(calls) == 3 + + +def test_get_transport_error_then_ok(): + state = [] + + def flaky(request): + if not state: + state.append(1) + raise httpx.ConnectError("reset", request=request) + return httpx.Response(200, text="ok") + + r = get( + _cl(flaky), "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None + ) + assert r.text == "ok" + + def test_get_429_retry_after_waits_once(): slept = [] cl = _cl( diff --git a/tests/market/test_tables.py b/tests/market/test_tables.py index 948924a..f5e17d5 100644 --- a/tests/market/test_tables.py +++ b/tests/market/test_tables.py @@ -36,6 +36,8 @@ def test_ensure_tables_idempotent(): "security", "entity_link", "filing", + "filing_hit", + "filing_passage", "source_gap", "price_daily", "series", @@ -75,6 +77,31 @@ FILING = T.FilingRow( "d.htm", ) GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today()) +HIT = T.HitRow( + "RBRVS", + "program", + "PFS", + "0000790526", + "0000790526-23-000003", + "rdnt-20221231.htm", + "10-K", + date(2023, 3, 1), + date(2022, 12, 31), + "8071", + "RadNet, Inc. (RDNT)", +) +PASSAGE = T.PassageRow( + "0000790526-23-000003:rdnt-20221231.htm:42", + "0000790526", + "0000790526-23-000003", + "rdnt-20221231.htm", + "RBRVS", + 42, + "Our pay follows RBRVS rules.", + "revenue_exposure", + "model-reading exposure-v1 m", + "K", +) PRICE = TI.PriceRow( "OSH", @@ -121,6 +148,16 @@ ROUNDTRIP = { lambda c, r: TI.write_series(c, r.series_id, [r]), lambda c: TI.read_series(c, ["OSH"]), ), + "filing_hit": ( + HIT, + lambda c, r: T.write_hits(c, [r]), + lambda c: T.read_hits(c), + ), + "filing_passage": ( + PASSAGE, + lambda c, r: T.write_passages(c, [r]), + lambda c: T.read_passages(c), + ), "gap": ( GAP, lambda c, r: T.write_gap(c, r.source, r.note), @@ -350,3 +387,24 @@ def test_ensure_tables_adds_priced_to_an_existing_security_table(): (old,) = T.read_securities(c) assert old.priced is True c.close() + + +def test_hits_and_passages_upsert_by_their_keys(con): + T.write_hits(con, [HIT, dataclasses_replace(HIT, term="x")]) + T.write_hits(con, [dataclasses_replace(HIT, sics="8082")]) + assert sorted((h.term, h.sics) for h in T.read_hits(con)) == [ + ("RBRVS", "8082"), + ("x", "8071"), + ] + sibling = dataclasses_replace(PASSAGE, term="x", label="") + T.write_passages(con, [PASSAGE, sibling]) + T.write_passages(con, [dataclasses_replace(PASSAGE, passage="new text", label="")]) + got = {p.term: p.passage for p in T.read_passages(con)} + assert got == {"RBRVS": "new text", "x": PASSAGE.passage} + assert T.write_hits(con, []) == 0 and T.write_passages(con, []) == 0 + + +def dataclasses_replace(row, **kw): + import dataclasses + + return dataclasses.replace(row, **kw)