feat(market): EDGAR full-text exposure passages per program and family, cited and AMA-guarded
Some checks failed
CI / lint (push) Successful in 57s
CI / notebooks-smoke (push) Successful in 2m45s
Deploy / changes (push) Successful in 18s
Infra CI / notebooks (push) Successful in 1m14s
CI / test (push) Failing after 5m26s
Infra CI / zotero (push) Successful in 21s
Infra CI / docs (push) Successful in 1m35s
Infra CI / api (push) Successful in 1m22s
Infra CI / llm (push) Successful in 1m14s
Infra CI / mc (push) Successful in 36s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / report (push) Has been cancelled
Deploy / bcda (push) Has been cancelled
Deploy / llm (push) Has been cancelled
Deploy / mc (push) Has been cancelled
Deploy / api (push) Has been cancelled
Infra CI / bcda (push) Has been cancelled

This commit is contained in:
kert
2026-11-13 02:19:01 -05:00
parent 58a3495e34
commit 6a45806759
10 changed files with 2053 additions and 7 deletions

View File

@@ -13,9 +13,11 @@ from __future__ import annotations
import dataclasses
import datetime as dt
import time
from collections import Counter
from typing import Any
import httpx
import typer
from market import edgar, http, tiingo
@@ -30,18 +32,26 @@ from market.seed import (
from market.tables import (
EntityRow,
FilingRow,
HitRow,
LinkRow,
PassageRow,
SecurityRow,
clear_gap,
ensure_tables,
read_entities,
read_gaps,
read_hits,
read_links,
read_passages,
read_prices,
retire_links,
retire_organizations,
write_entities,
write_filings,
write_gap,
write_hits,
write_links,
write_passages,
write_prices,
write_securities,
write_series,
@@ -52,6 +62,7 @@ app = typer.Typer(no_args_is_help=True)
ENTITIES_GAP = "sec-edgar:entities"
PRICES_GAP = "tiingo:prices"
FTS_GAP = "sec-edgar:fts"
# ── indirections the tests monkeypatch ───────────────────────────────
@@ -79,6 +90,17 @@ def _publish() -> None:
typer.echo(f"replica → {publish_replica('aco')}")
def _asker() -> Any:
"""The local JSON extractor on ``cfg.instruct_model`` (Ollama through the
host pool; never a cloud model)."""
from llm import config as llm_config
from llm.pool import HostPool
from llm.structured import json_extractor
cfg = llm_config.load()
return json_extractor(cfg, HostPool.from_config(cfg))
# ── gaps ─────────────────────────────────────────────────────────────
def _gap(source: str, note: str, *, write: bool) -> None:
"""Print ``gap: <source>: <note>``; with *write* record it (and nothing
@@ -570,6 +592,434 @@ def prices(
typer.echo(f"stored: {n_rows} price rows, {n_series} monthly rows")
# ── full-text search ─────────────────────────────────────────────────
def _fts_scope(
families: list[str], programs: list[str]
) -> list[tuple[str, str, tuple[str, ...]]]:
"""``(scope, target, terms)`` for the requested programs and families (all
programs when none is named)."""
out: list[tuple[str, str, tuple[str, ...]]] = []
if not families and not programs:
programs = list(edgar.PROGRAM_TERMS)
for key in programs:
if key not in edgar.PROGRAM_TERMS:
raise typer.BadParameter(f"unknown program {key!r}")
out.append(("program", key, edgar.PROGRAM_TERMS[key]))
if families:
from pfs.families import FAMILIES, refresh_from
con = _read()
try:
refresh_from(con)
finally:
con.close()
for key in families:
if key not in FAMILIES:
raise typer.BadParameter(f"unknown family {key!r}")
out.append(("family", key, edgar.family_terms(FAMILIES[key])))
return out
def _ama_index() -> edgar.AmaIndex:
"""The 8-word runs of every licensed CPT descriptor, all editions."""
con = _read()
try:
return edgar.cpt_grams(
d
for (d,) in con.execute("SELECT descriptor FROM pfs.cpt_code").fetchall()
if d
)
finally:
con.close()
def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRow]]:
return (
_or_empty(read_hits, con_read, []),
_or_empty(read_passages, con_read, []),
_or_empty(read_entities, con_read, []),
)
def _classify(
kept: list[PassageRow], limit: int
) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]:
"""Label *kept* with the local model, resumably (``pfs.runlog``). Returns
the ledger outputs by unit, the run counts and the model name."""
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
ask = _asker()
model = getattr(ask, "model", "local")
version = edgar.EXPOSURE_PROMPT_VERSION
log = RunLog(ledger_path("market-exposure", "all"))
def unit(p: PassageRow) -> Any:
def run() -> dict | None:
label = edgar.classify_passage(p.passage, p.term, ask)
return {"label": label} if label else None
return run
units = [
(f"{p.passage_id}|{p.term}", input_hash(p.passage, p.term), unit(p))
for p in kept
]
counts = run_units(units, log, model=model, version=version, limit=limit)
return log.outputs(model, version), counts, model
def _verified(
links: list[LinkRow], passage_rows: dict[Any, PassageRow]
) -> tuple[list[LinkRow], int]:
"""M12: keep a link only when its passage occurs verbatim in the hoarded
filing (re-read from disk) and is at most 400 characters."""
by_id = {p.passage_id: p for p in passage_rows.values()}
texts: dict[str, str | None] = {}
kept: list[LinkRow] = []
for lk in links:
p = by_id.get(lk.passage_id)
if p is None:
continue
if (p.adsh, p.filename) not in texts:
path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename))
texts[(p.adsh, p.filename)] = (
edgar.read_doc(path) if path.is_file() else None
)
text = texts[(p.adsh, p.filename)]
if text is not None and len(p.passage) <= 400 and p.passage in text:
kept.append(lk)
return kept, len(links) - len(kept)
def _new_entities(
ciks: list[str], seeds: dict[str, Any], issues: list[str]
) -> list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]]:
"""An entity, its securities and its sweep-form filings for each CIK the
registry does not know yet (submissions are hoarded)."""
if not ciks:
return []
out: list[tuple[EntityRow, list[SecurityRow], list[FilingRow]]] = []
pacer = edgar.new_pacer()
with http.client(edgar.SOURCE) as cl:
tickers = edgar.fetch_tickers(cl, pacer=pacer)
for cik in ciks:
sub = edgar.fetch_submissions(cl, f"CIK{cik}.json", pacer=pacer)
if sub is None:
issues.append(f"CIK{cik}: no submissions, no entity")
continue
seed = seeds.get(cik)
filings = [
f
for f in edgar.filings_from_submissions(sub)
if f.form in edgar.SWEEP_FORMS
]
out.append(
(
edgar.entity_from_submissions(sub, seed),
edgar.securities_for(sub, seed, tickers.get(cik, [])),
filings,
)
)
return out
def _gap_note(issues: list[str]) -> str:
shown = "; ".join(issues[:15])
more = f"; ... {len(issues) - 15} more" if len(issues) > 15 else ""
return f"{len(issues)} piece(s) missing: {shown}{more}"
@app.command()
def fts(
family: list[str] = typer.Option([], "--family", help="A family key (repeatable)."),
program: list[str] = typer.Option(
[], "--program", help="A program key (repeatable; all when none given)."
),
forms: list[str] = typer.Option(
["10-K"], "--forms", help="Filing form to search (repeatable)."
),
from_year: int = typer.Option(edgar.FTS_FIRST_YEAR, "--from"),
to_year: int = typer.Option(0, "--to", help="Default: this year."),
limit: int = typer.Option(
0, "--limit", help="At most N new filing downloads and model calls."
),
max_minutes: float = typer.Option(
0.0, "--max-minutes", help="Stop fetching after this long (rerun resumes)."
),
refresh: bool = typer.Option(False, "--refresh", help="Download again."),
dry_run: bool = typer.Option(False, "--dry-run", help="List the queries only."),
classify: bool = typer.Option(
False, "--classify", help="Label passages with the local model."
),
write: bool = typer.Option(False, "--write", help="Store hits, passages, links."),
) -> None:
"""EDGAR full-text search for program and family terms in 10-K filings:
hits (market.filing_hit), verbatim passages of at most 400 characters
that pass the AMA guard (market.filing_passage) and ``fts`` links
(market.entity_link) for health-SIC and seed companies. A link needs a
kept passage. Everything fetched is hoarded; a rerun resumes."""
to_year = to_year or dt.date.today().year
scope = _fts_scope(family, program)
owners: dict[str, list[tuple[str, str]]] = {}
for sc, target, terms in scope:
for t in terms:
owners.setdefault(t, []).append((sc, target))
queries = [
(t, f, y) for t in owners for f in forms for y in range(from_year, to_year + 1)
]
if dry_run:
for sc, target, terms in scope:
n = len(terms) * len(forms) * (to_year - from_year + 1)
typer.echo(f"{sc} {target}: {len(terms)} terms, {n} queries")
cached = sum(edgar.fts_cached(*q) for q in queries)
typer.echo(
f"{len(queries)} queries ({cached} already hoarded), "
f"{edgar.FTS_PAGE} hits a page, at most 10 requests a second"
)
return
started = time.monotonic()
def out_of_time() -> bool:
return bool(max_minutes) and time.monotonic() - started > max_minutes * 60
pacer = edgar.new_pacer()
prior_hits: list[HitRow] = []
prior_passages: list[PassageRow] = []
prior_entities: list[EntityRow] = []
con = _read()
try:
prior_hits, prior_passages, prior_entities = _prior(con)
finally:
con.close()
companies, _orgs = load_seed()
seeds = {c.cik: c for c in companies}
seed_ciks = set(seeds)
health = set(edgar.HEALTH_SICS)
known = {e.key for e in prior_entities}
new_hits: dict[tuple[str, str, str, str, str], HitRow] = {}
new_passages: list[PassageRow] = []
issues: list[str] = []
withheld = 0
pending = 0
incomplete = False
cl = http.client(edgar.SOURCE)
# a stalled connection should fail in seconds, not minutes; the retry
# opens a new one
cl.timeout = httpx.Timeout(30.0)
try:
for term, form, year in queries:
if out_of_time():
incomplete = True
break
n = 0
for _offset, payload in edgar.fetch_fts(
cl, term, form, year, pacer=pacer, refresh=refresh
):
for sc, target in owners[term]:
for h in edgar.parse_hits(
payload, term=term, scope=sc, target=target
):
new_hits[(h.term, h.scope, h.target, h.adsh, h.filename)] = h
n += 1
typer.echo(f"{term} {form} {year}: {n} hits")
def eligible(h: HitRow) -> bool:
return h.cik in seed_ciks or bool(set(h.sics.split(",")) & health)
docs: dict[tuple[str, str, str], list[HitRow]] = {}
for h in new_hits.values():
if eligible(h):
docs.setdefault((h.cik, h.adsh, h.filename), []).append(h)
ama = _ama_index() if docs else edgar.cpt_grams([])
fetched = 0
order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k))
for cik, adsh, filename in order:
hs = docs[(cik, adsh, filename)]
if not filename.lower().endswith(edgar.READABLE):
issues.append(f"{adsh}:{filename}: format not readable as text")
continue
path = http.hoard("sec", "docs", cik, adsh, filename)
have = http.newest(path).is_file()
if not have and ((limit and fetched >= limit) or out_of_time()):
pending += 1
continue
try:
got = edgar.fetch_doc(cl, hs[0], pacer=pacer, refresh=refresh)
except http.Refused as exc:
if exc.status in (0, 500, 502, 503, 504):
# one flaky document: skip it, a rerun retries
issues.append(f"{adsh}:{filename}: {exc}")
continue
raise
if got is None:
issues.append(f"{adsh}:{filename}: document not found")
continue
fetched += 0 if have else 1
text = edgar.read_doc(got)
for h in {x.term: x for x in hs}.values():
kept, held = edgar.safe_passages(text, h.term, ama)
withheld += held
for start, quote in kept:
new_passages.append(
PassageRow(
f"{adsh}:{filename}:{start}",
cik,
adsh,
filename,
h.term,
start,
quote,
)
)
if pending:
incomplete = True
issues.append(f"{pending} filing(s) not downloaded yet (limit or time)")
except http.Refused as exc:
_gap(FTS_GAP, str(exc), write=write)
return
finally:
cl.close()
# the union of this run and what earlier runs stored
hits = {(h.term, h.scope, h.target, h.adsh, h.filename): h for h in prior_hits}
hits.update(new_hits)
redone = {(p.adsh, p.filename, p.term) for p in new_passages}
prior_by_id = {(p.passage_id, p.term): p for p in prior_passages}
merged: dict[tuple[str, str], PassageRow] = {
k: p
for k, p in prior_by_id.items()
if (p.adsh, p.filename, p.term) not in redone
}
for p in new_passages:
old = prior_by_id.get((p.passage_id, p.term))
if old and old.passage == p.passage:
p = dataclasses.replace(
p,
label=old.label,
label_version=old.label_version,
item_key=old.item_key,
)
merged[(p.passage_id, p.term)] = p
if classify:
filed = {(h.adsh, h.filename): h.file_date for h in hits.values()}
todo = sorted(
(p for p in merged.values() if p.term in owners),
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
)
outputs, counts, model = _classify(todo, limit)
typer.echo(f"classified with {model}: {counts}")
for k, p in list(merged.items()):
got = outputs.get(f"{p.passage_id}|{p.term}")
if got:
merged[k] = dataclasses.replace(
p,
label=got["label"],
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
)
entities = list(prior_entities)
all_hits = list(hits.values())
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
linkable = [
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
]
links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
mine = {(sc, t) for sc, t, _ in scope}
links = [lk for lk in links if (lk.scope, lk.target) in mine]
links, unverified = _verified(links, merged)
if unverified:
issues.append(f"{unverified} link(s) dropped: quote not found in the hoard")
try:
new_entities = _new_entities(
sorted({lk.entity_key for lk in links} - known), seeds, issues
)
except http.Refused as exc:
_gap(FTS_GAP, str(exc), write=write)
return
have = known | {e.key for e, _, _ in new_entities}
links = [lk for lk in links if lk.entity_key in have]
by_target = Counter(lk.target for lk in links)
typer.echo(
f"hits {len(new_hits)} (stored {len(all_hits)}); passages kept "
f"{len(new_passages)}, withheld by the AMA guard {withheld}; new entities "
f"{len(new_entities)}; fts links {len(links)} "
f"({', '.join(f'{k} {n}' for k, n in sorted(by_target.items())) or 'none'})"
)
for i in issues[:20]:
typer.echo(f"gap: {FTS_GAP}: {i}")
if len(issues) > 20:
typer.echo(f"gap: {FTS_GAP}: ... {len(issues) - 20} more")
if not write:
typer.echo("dry run: pass --write to store")
return
if incomplete:
typer.echo("incomplete (limit or time): nothing written; rerun to resume")
return
store = _store()
docs_of = {(h.adsh, h.filename): h for h in all_hits}
item_of: dict[tuple[str, str], str] = {}
for lk in links:
adsh, _, filename = lk.cite_ref.partition(":")
if (adsh, filename) not in item_of:
h = docs_of[(adsh, filename)]
item_of[(adsh, filename)] = edgar.cite_filing(
store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename))
)
links = [
dataclasses.replace(
lk,
item_key=item_of[tuple(lk.cite_ref.split(":", 1))], # type: ignore[arg-type]
)
for lk in links
]
stamped = [
dataclasses.replace(p, item_key=item_of[(p.adsh, p.filename)])
if (p.adsh, p.filename) in item_of and not p.item_key
else p
for p in merged.values()
]
from cms.ingest_log import log_ingest
sic_rows = [r for e, _, _ in new_entities for r in edgar.sic_links(e)]
with _batch() as con:
ensure_tables(con)
write_entities(con, [e for e, _, _ in new_entities])
for e, secs, filings in new_entities:
write_securities(con, e.key, secs)
write_filings(con, e.cik, filings)
write_links(con, sic_rows)
write_hits(con, list(new_hits.values()))
write_passages(con, stamped)
write_links(con, links)
stored = {(lk.entity_key, lk.scope, lk.target) for lk in links}
old_fts = [lk for lk in read_links(con, basis="fts")]
keep = stored | {
(lk.entity_key, lk.scope, lk.target)
for lk in old_fts
if (lk.scope, lk.target) not in mine
}
retired = retire_links(con, "fts", {lk.entity_key for lk in old_fts}, keep)
if issues:
write_gap(con, FTS_GAP, _gap_note(issues))
else:
clear_gap(con, FTS_GAP)
src = str(http.hoard(http.MANIFEST))
for table, n in (
("market.filing_hit", len(new_hits)),
("market.filing_passage", len(stamped)),
("market.entity_link", len(links) + len(sic_rows)),
):
log_ingest(
con, module="market.edgar", table_name=table, rows=n, source_file=src
)
_publish()
typer.echo(f"stored: fts links {len(links)} (retired {retired})")
@app.command()
def gaps() -> None:
"""Sources that could not be read (market.source_gap)."""

View File

@@ -22,7 +22,14 @@ import httpx
from market import http
from market.seed import Cite, SeedCompany, SeedOrg, norm_org
from market.tables import EntityRow, FilingRow, LinkRow, SecurityRow
from market.tables import (
EntityRow,
FilingRow,
HitRow,
LinkRow,
PassageRow,
SecurityRow,
)
TICKERS = "https://www.sec.gov/files/company_tickers.json"
SUBMISSIONS = "https://data.sec.gov/submissions/{name}"
@@ -325,21 +332,23 @@ def _fetch(
refresh: bool,
offline: bool,
ok_404: bool = False,
params: dict[str, Any] | None = None,
) -> bytes | None:
"""The newest hoarded copy of *path*, downloading when there is none (or
on *refresh*) and not *offline*. A changed body lands beside the old; a
tolerated 404 is ``None`` and is not stored."""
have = http.newest(path)
shown = str(httpx.URL(url, params=params)) if params else url
if have.is_file() and (offline or not refresh):
http.manifest(have, url)
http.manifest(have, shown)
return have.read_bytes()
if offline or cl is None:
return None
r = http.get(cl, url, pacer=pacer, source=SOURCE, ok_404=ok_404)
r = http.get(cl, url, params=params, pacer=pacer, source=SOURCE, ok_404=ok_404)
if r.status_code == 404:
return None
saved = http.save(path, r.content)
http.manifest(saved, url)
http.manifest(saved, shown)
return saved.read_bytes()
@@ -535,3 +544,358 @@ def shares_ama_run(
else AmaIndex.of(descriptors, n)
)
return not grams.isdisjoint(index)
# ── full-text search ─────────────────────────────────────────────────
EFTS = "https://efts.sec.gov/LATEST/search-index"
FTS_PAGE = 100 # hits per page; ``from`` is the offset
FTS_FIRST_YEAR = 2001 # EDGAR full-text search starts in 2001
PROGRAM_TERMS: dict[str, tuple[str, ...]] = {
"PFS": (
"Medicare Physician Fee Schedule",
"physician fee schedule",
"RBRVS",
"Relative Value Scale Update Committee",
"CPT Editorial Panel",
),
"QPP-MIPS": ("Merit-based Incentive Payment System", "Quality Payment Program"),
"MSSP": ("Medicare Shared Savings Program",),
"APM": ("Advanced Alternative Payment Model",),
"TELEHEALTH": ("Medicare telehealth",),
"HOME-BASED-CARE": ("home-based primary care", "home-based care"),
"CARE-MANAGEMENT": ("chronic care management", "transitional care management"),
}
def fts_params(term: str, form: str, year: int, offset: int) -> dict[str, Any]:
"""One query per (term, form, year) keeps each under the 10,000-hit cap."""
return {
"q": f'"{term}"',
"forms": form,
"dateRange": "custom",
"startdt": f"{year}-01-01",
"enddt": f"{year}-12-31",
"from": offset,
}
def family_terms(fam: Any) -> tuple[str, ...]:
"""The family name plus every multi-word synonym (single words are too
common to search for)."""
out: dict[str, str] = {}
for t in (fam.name, *(s for s in fam.synonyms if len(s.split()) > 1)):
out.setdefault(t.lower(), t)
return tuple(out.values())
def parse_hits(
payload: dict[str, Any], *, term: str, scope: str, target: str
) -> list[HitRow]:
"""The hits of one search page; ``_id`` is ``"{adsh}:{filename}"``."""
rows = []
for hit in payload.get("hits", {}).get("hits", []):
src = hit["_source"]
adsh, _, filename = hit["_id"].partition(":")
names = src.get("display_names") or [""]
rows.append(
HitRow(
term=term,
scope=scope,
target=target,
cik=cik10(src["ciks"][0]),
adsh=adsh,
filename=filename,
form=src.get("form") or "",
file_date=dt.date.fromisoformat(src["file_date"]),
period_ending=_date(src.get("period_ending") or ""),
sics=",".join(src.get("sics") or []),
display_name=names[0],
)
)
return rows
def doc_url(cik: str, adsh: str, filename: str) -> str:
return (
f"https://www.sec.gov/Archives/edgar/data/{int(cik)}/"
f"{adsh.replace('-', '')}/{filename}"
)
def fts_cache(term: str, form: str, year: int, offset: int) -> Path:
return http.hoard(
"sec", "fts", http.slug(term), form.replace("/", "_"), f"{year}.{offset}.json"
)
def fts_cached(term: str, form: str, year: int) -> bool:
return http.newest(fts_cache(term, form, year, 0)).is_file()
def fetch_fts(
cl: httpx.Client | None,
term: str,
form: str,
year: int,
*,
pacer: http.Pacer,
refresh: bool = False,
offline: bool = False,
) -> Iterator[tuple[int, dict[str, Any]]]:
"""Every page ``(offset, payload)`` of one (term, form, year) query,
hoarded at ``data/market/sec/fts/<slug(term)>/<form>/<year>.<offset>.json``
and read from the hoard on a rerun."""
offset = 0
while True:
body = _fetch(
cl,
EFTS,
fts_cache(term, form, year, offset),
pacer=pacer,
refresh=refresh,
offline=offline,
params=fts_params(term, form, year, offset),
)
if body is None:
return
payload = json.loads(body)
yield offset, payload
offset += FTS_PAGE
if offset >= payload.get("hits", {}).get("total", {}).get("value", 0):
return
#: documents we can read as text
READABLE = (".htm", ".html", ".txt")
def fetch_doc(
cl: httpx.Client | None,
hit: HitRow,
*,
pacer: http.Pacer,
refresh: bool = False,
offline: bool = False,
) -> Path | None:
"""The hoarded copy of the hit's document (``sec/docs/<cik>/<adsh>/<file>``),
downloaded when absent and not *offline*; ``None`` when unavailable."""
path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename)
body = _fetch(
cl,
doc_url(hit.cik, hit.adsh, hit.filename),
path,
pacer=pacer,
refresh=refresh,
offline=offline,
ok_404=True,
)
return http.newest(path) if body is not None else None
def read_doc(path: Path) -> str:
return normalize_text(path.read_bytes().decode("utf-8", errors="replace"))
# ── passages ─────────────────────────────────────────────────────────
_SENTENCE_END = (". ", "? ", "! ")
def _window(text: str, a: int, b: int, max_chars: int) -> tuple[int, int]:
"""The sentence around ``text[a:b]`` when it fits *max_chars*, else a
window centred on the match, cut at word boundaries."""
start = max(text.rfind(m, 0, a) for m in _SENTENCE_END) + 2
start = start if start >= 2 else 0
ends = [e for m in _SENTENCE_END if (e := text.find(m, b)) != -1]
end = min(ends) + 1 if ends else len(text)
if end - start <= max_chars:
return start, end
start = max(0, a - (max_chars - (b - a)) // 2)
end = min(len(text), start + max_chars)
start = max(0, end - max_chars)
if start > 0 and text[start - 1] != " ":
cut = text.find(" ", start, a)
start = cut + 1 if cut != -1 else start
if end < len(text) and text[end] != " ":
cut = text.rfind(" ", b, end)
end = cut if cut != -1 else end
return start, end
def _scan(
text: str,
term: str,
sizes: Sequence[int],
max_n: int,
grams: AmaIndex | None,
) -> tuple[list[tuple[int, str]], int]:
"""Up to *max_n* kept ``(char_start, passage)`` and how many occurrences
were withheld. An occurrence is tried at each of *sizes* in turn (widest
first) until its passage shares no run with *grams*."""
low, needle = text.lower(), term.lower()
out: list[tuple[int, str]] = []
seen: set[int] = set()
withheld = 0
pos = 0
while len(out) < max_n and (i := low.find(needle, pos)) != -1:
pos = i + len(needle)
for size in sizes:
start, end = _window(text, i, pos, size)
chunk = text[start:end]
start += len(chunk) - len(chunk.lstrip())
chunk = chunk.strip()
if grams is None or ama_safe(chunk, grams):
if start not in seen:
seen.add(start)
out.append((start, chunk))
break
else:
withheld += 1
return out, withheld
def passages(
text: str, term: str, *, max_chars: int = 400, max_n: int = 3
) -> list[tuple[int, str]]:
"""Up to *max_n* ``(char_start, passage)`` around case-insensitive matches
of *term* in *text*, each widened to its sentence within *max_chars*."""
return _scan(text, term, (max_chars,), max_n, None)[0]
def safe_passages(
text: str, term: str, grams: AmaIndex, *, max_chars: int = 400, max_n: int = 3
) -> tuple[list[tuple[int, str]], int]:
""":func:`passages` with the D3 guard: a passage that shares an 8-word run
with a CPT descriptor is retried at 60% and 30% of *max_chars*, then
withheld. Returns the kept passages and the number withheld."""
sizes = (max_chars, max_chars * 3 // 5, max_chars * 3 // 10)
return _scan(text, term, sizes, max_n, grams)
def cpt_grams(descriptors: Iterable[str], n: int = 8) -> AmaIndex:
"""The 8-word runs of every descriptor (no run crosses descriptors),
tokenized like ``pfs.origin.shares_run``."""
return AmaIndex.of(descriptors, n)
def ama_safe(passage: str, grams: AmaIndex) -> bool:
"""D3: *passage* shares no 8-word run with any CPT descriptor."""
return not shares_ama_run(passage, grams)
# ── local classification ─────────────────────────────────────────────
EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental")
EXPOSURE_PROMPT_VERSION = "exposure-v1"
EXPOSURE_SCHEMA: dict[str, Any] = {
"type": "object",
"properties": {
"label": {"type": "string", "enum": list(EXPOSURE_LABELS)},
"reason": {"type": "string", "maxLength": 200},
},
"required": ["label", "reason"],
}
EXPOSURE_SYSTEM = (
"You read one passage of a company's annual report that mentions a Medicare "
"program or payment term. Label how the passage ties the company to it: "
"revenue_exposure (the company says it is paid under, or earns revenue "
"through, the program or schedule), risk_factor (the company says changes "
"to the program could hurt or affect it), or incidental (the passage "
"defines, lists or mentions the term without tying the company to it). "
"Answer with JSON only: a label and a reason of at most 200 characters."
)
def classify_passage(passage: str, term: str, ask: Callable[..., Any]) -> str | None:
"""A closed label for one passage from a local model, or ``None`` when
the reply is missing or outside :data:`EXPOSURE_LABELS`."""
out = ask(
EXPOSURE_SYSTEM,
f"Term: {term}\nPassage: {passage}",
EXPOSURE_SCHEMA,
)
label = out.get("label") if isinstance(out, dict) else None
return label if label in EXPOSURE_LABELS else None
# ── links ────────────────────────────────────────────────────────────
_RANK = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
def fts_links(
hits: Iterable[HitRow],
passage_rows: Iterable[PassageRow],
entities: Iterable[EntityRow],
seed_ciks: Iterable[str],
) -> list[LinkRow]:
"""``basis="fts"`` links, one per (cik, scope, target): the strongest
evidence, then the newest filing. A hit counts when its SIC is a health
SIC or its CIK is in the seed. The strength is the passage label, or
``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit
without a kept passage links with ``passage_id=""``."""
seed = set(seed_ciks)
sic_of = {e.cik: e.sic for e in entities}
by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {}
for p in passage_rows:
by_doc.setdefault((p.cik, p.adsh, p.filename, p.term), []).append(p)
best: dict[tuple[str, str, str], tuple[tuple, LinkRow]] = {}
for h in hits:
sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")}
if h.cik not in seed and not (sics & set(HEALTH_SICS)):
continue
kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
cands: list[tuple[str, str]] = [
(p.label or "mention", p.passage_id) for p in kept
] or [("mention", "")]
for strength, pid in cands:
if strength == "incidental":
continue
rank = (_RANK[strength], -h.file_date.toordinal(), pid == "", pid)
link = LinkRow(
h.cik,
h.scope,
h.target,
"fts",
strength,
"adsh",
f"{h.adsh}:{h.filename}",
pid,
)
key = (h.cik, h.scope, h.target)
if key not in best or rank < best[key][0]:
best[key] = (rank, link)
return [link for _, link in sorted(best.values(), key=lambda b: b[1].entity_key)]
def cite_filing(store: Any, hit: HitRow, path: Path) -> str:
"""The bib Download for the filing behind a hit (keyed by its document
URL, so a rerun returns the same key), with the document attached."""
from bib.item import Download
from bib.tag import Tag
url = doc_url(hit.cik, hit.adsh, hit.filename)
item = Download(
title=(
f"{hit.display_name} Form {hit.form} filed {hit.file_date} "
f"(accession {hit.adsh})"
),
url=url,
file_urls=[url],
year=hit.file_date.year,
page_type="filing",
website_title="U.S. Securities and Exchange Commission",
)
key = store.upsert(
item,
tags=[
Tag.module("market"),
Tag.table("market.filing_passage"),
Tag.source("sec-edgar"),
Tag.year(hit.file_date.year),
],
)
store.attach_file(key, path, title=path.name)
return key

View File

@@ -123,15 +123,24 @@ def get(
max_retry_after: float = 600.0,
ok_404: bool = False,
) -> httpx.Response:
"""GET with pacing. 5xx is retried with a ``2 * 2**n`` second backoff,
a 429 with a short ``Retry-After`` waits once, 401/403 stop the source,
"""GET with pacing. A transport error (reset, timeout) and a 5xx are
retried with a ``2 * 2**n`` second backoff (a transport error that
outlasts the retries is a :class:`Refused` with status 0), a 429 with a short ``Retry-After`` waits once, 401/403 stop the source,
and a 404 is returned only when *ok_404*. This is the market tier's one
retry loop."""
attempt = 0
rate_waited = False
while True:
pacer.wait()
r = cl.get(url, params=params)
try:
r = cl.get(url, params=params)
except httpx.TransportError as exc:
# a dropped connection or a read timeout: back off and retry
if attempt < retries:
sleep(2.0 * 2**attempt)
attempt += 1
continue
raise Refused(source, 0, type(exc).__name__) from exc
status = r.status_code
if status < 400:
return r

View File

@@ -33,6 +33,16 @@ CREATE TABLE IF NOT EXISTS market.filing (
cik VARCHAR, adsh VARCHAR, form VARCHAR, filed DATE, accepted VARCHAR,
report_date DATE, items VARCHAR, primary_doc VARCHAR
);
CREATE TABLE IF NOT EXISTS market.filing_hit (
term VARCHAR, scope VARCHAR, target VARCHAR, cik VARCHAR, adsh VARCHAR,
filename VARCHAR, form VARCHAR, file_date DATE, period_ending DATE,
sics VARCHAR, display_name VARCHAR
);
CREATE TABLE IF NOT EXISTS market.filing_passage (
passage_id VARCHAR, cik VARCHAR, adsh VARCHAR, filename VARCHAR,
term VARCHAR, char_start INTEGER, passage VARCHAR, label VARCHAR,
label_version VARCHAR, item_key VARCHAR
);
CREATE TABLE IF NOT EXISTS market.source_gap (
source VARCHAR, note VARCHAR, recorded_on DATE
);
@@ -128,6 +138,41 @@ class FilingRow:
primary_doc: str
@dataclass(frozen=True)
class HitRow:
"""One EDGAR full-text-search hit of *term*, owned by one *target* (a
program or family key)."""
term: str
scope: str # program | family
target: str
cik: str
adsh: str
filename: str
form: str
file_date: date
period_ending: date | None
sics: str # comma-joined
display_name: str
@dataclass(frozen=True)
class PassageRow:
"""At most 400 characters of public filing text around *term*; the
*label* is a model reading when ``label_version`` is set."""
passage_id: str # "{adsh}:{filename}:{char_start}"
cik: str
adsh: str
filename: str
term: str
char_start: int
passage: str
label: str = ""
label_version: str = ""
item_key: str = ""
@dataclass(frozen=True)
class GapRow:
source: str
@@ -359,6 +404,50 @@ def read_filings(
return sorted((FilingRow(*r) for r in rows), key=lambda f: (f.filed, f.adsh))
# ── filing_hit / filing_passage ──────────────────────────────────────
def write_hits(con: Any, rows: Sequence[HitRow]) -> int:
"""Upsert by ``(term, scope, target, adsh, filename)``."""
ensure_tables(con)
keys = sorted({(r.term, r.scope, r.target, r.adsh, r.filename) for r in rows})
for part in _chunks(keys, 400):
con.execute(
"DELETE FROM market.filing_hit WHERE (term, scope, target, adsh, filename) "
"IN (SELECT * FROM (VALUES "
+ ", ".join(["(?, ?, ?, ?, ?)"] * len(part))
+ "))",
[v for k in part for v in k],
)
return _insert(con, "market.filing_hit", rows)
def read_hits(con: Any) -> list[HitRow]:
rows = _select(con, HitRow, "market.filing_hit")
return sorted(
(HitRow(*r) for r in rows),
key=lambda h: (h.scope, h.target, h.term, h.cik, h.adsh, h.filename),
)
def write_passages(con: Any, rows: Sequence[PassageRow]) -> int:
"""Replace the passages of each ``(adsh, filename, term)`` in *rows*."""
ensure_tables(con)
keys = sorted({(r.adsh, r.filename, r.term) for r in rows})
for part in _chunks(keys, 400):
con.execute(
"DELETE FROM market.filing_passage WHERE (adsh, filename, term) "
"IN (SELECT * FROM (VALUES " + ", ".join(["(?, ?, ?)"] * len(part)) + "))",
[v for k in part for v in k],
)
return _insert(con, "market.filing_passage", rows)
def read_passages(con: Any) -> list[PassageRow]:
rows = _select(con, PassageRow, "market.filing_passage")
return sorted((PassageRow(*r) for r in rows), key=lambda p: p.passage_id)
# ── source_gap ───────────────────────────────────────────────────────

View File

@@ -1004,3 +1004,509 @@ def test_prices_missing_overlap_day_repulls_further_back(px, monkeypatch):
assert "re-pulling from 2023-12-25" in res.output
row = {r.day: r for r in T.read_prices(c, "AAA")}[date(2024, 1, 5)]
assert row.ret == pytest.approx(60.5 / 55 - 1)
# ── fts ──────────────────────────────────────────────────────────────
MPFS = '"Medicare Physician Fee Schedule"'
SNIPPET = (FX / "filing_snippet.htm").read_bytes()
LICENSED = "a licensed descriptor that no filing quote should ever copy word for word"
def _hit_json(cik, adsh, filename, sics, date_="2023-03-01"):
return {
"_id": f"{adsh}:{filename}",
"_source": {
"ciks": [cik],
"adsh": adsh,
"form": "10-K",
"file_date": date_,
"period_ending": "2022-12-31",
"sics": sics,
"display_names": [f"Filer {cik} (CIK {cik})"],
},
}
HITS = [
_hit_json("0000012345", "0000012345-23-000001", "a10k.htm", ["8071"]),
_hit_json("0000000002", "0000000002-23-000001", "reit.htm", ["6798"]),
_hit_json("0000012346", "0000012346-23-000001", "ex99.pdf", ["8082"]),
_hit_json("0000012347", "0000012347-23-000001", "gone.htm", ["8082"]),
]
def _fts_net(
monkeypatch, hits=HITS, docs=None, forbid_docs=False, status=200, broken=None
):
"""A mock EDGAR: the term "Medicare Physician Fee Schedule" has *hits* in
2023; every other query is empty."""
docs = docs if docs is not None else {"a10k.htm": SNIPPET}
requested: list[str] = []
def efts(m):
p = dict(httpx.URL(m.group(0)).params)
mine = p["q"] == MPFS and p["from"] == "0"
return httpx.Response(
status,
json={
"hits": {
"total": {"value": len(hits) if mine else 0},
"hits": hits if mine else [],
}
},
)
def doc(m):
requested.append(m.group(1))
if broken and m.group(1) in broken:
if broken[m.group(1)] == "timeout":
raise httpx.ReadTimeout("slow")
return httpx.Response(broken[m.group(1)])
body = docs.get(m.group(1))
if forbid_docs and body:
raise AssertionError("no document download expected")
return httpx.Response(200, content=body) if body else httpx.Response(404)
def sub(m):
return httpx.Response(200, json={**_sub_for(m.group(1)), "tickers": ["ZZZ"]})
_online(
monkeypatch,
[
(r"efts\.sec\.gov/.*", efts),
(r"/Archives/edgar/data/\d+/\d+/([^/?]+)$", doc),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
(r"CIK(\d{10})\.json", sub),
],
)
return requested
FTS = ["market", "fts", "--program", "PFS", "--from", "2023", "--to", "2023"]
def test_fts_cli_dry_run_lists_queries(env, monkeypatch):
c, _, published, hoard = env
monkeypatch.setattr(
http, "client", lambda *a, **k: (_ for _ in ()).throw(AssertionError("net"))
)
res = runner.invoke(app, [*FTS, "--dry-run"])
assert res.exit_code == 0, res.output
assert "program PFS: 5 terms, 5 queries" in res.output
assert "5 queries (0 already hoarded)" in res.output
assert published == []
full = runner.invoke(app, ["market", "fts", "--dry-run"])
assert "program MSSP: 1 terms" in full.output
assert "program CARE-MANAGEMENT: 2 terms" in full.output
assert f"{(5 + 2 + 1 + 1 + 1 + 2 + 2) * (dt_year() - 2000)} queries" in full.output
def dt_year():
import datetime
return datetime.date.today().year
def test_fts_cli_family_scope_and_bad_keys(env, monkeypatch):
monkeypatch.setattr("pfs.families.refresh_from", lambda con: 0)
res = runner.invoke(
app,
[
"market",
"fts",
"--family",
"HOME-VISITS",
"--dry-run",
"--from",
"2023",
"--to",
"2023",
],
)
assert res.exit_code == 0, res.output
assert "family HOME-VISITS: 8 terms, 8 queries" in res.output
assert (
runner.invoke(app, ["market", "fts", "--family", "NOPE", "--dry-run"]).exit_code
== 2
)
assert (
runner.invoke(
app, ["market", "fts", "--program", "NOPE", "--dry-run"]
).exit_code
== 2
)
def test_fts_cli_without_write_stores_nothing(env, monkeypatch):
c, _, published, hoard = env
_fts_net(monkeypatch)
monkeypatch.setattr(
mk, "_batch", lambda: (_ for _ in ()).throw(AssertionError("no batch"))
)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "Medicare Physician Fee Schedule 10-K 2023: 4 hits" in res.output
assert "passages kept 1," in res.output and "new entities 1" in res.output
assert "fts links 1 (PFS 1)" in res.output and "dry run" in res.output
assert (
"gap: sec-edgar:fts: 0000012346-23-000001:ex99.pdf: format not readable"
in res.output
)
assert (
"gap: sec-edgar:fts: 0000012347-23-000001:gone.htm: document not found"
in res.output
)
assert published == []
# hoarded: the pages and the one downloaded readable document
assert (
hoard / "sec/fts/medicare-physician-fee-schedule/10-K/2023.0.json"
).is_file()
assert (hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm").is_file()
def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
c, store, published, hoard = env
curated = T.LinkRow(
"0000012345", "program", "APM", "curated", "curated", "adsh", "x", "", "K"
)
T.write_links(c, [curated])
_fts_net(monkeypatch)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
(ent,) = T.read_entities(c, "company")
assert ent.key == "0000012345" and ent.sic == "8082"
assert [s.ticker for s in T.read_securities(c, ent.key)] == ["ZZZ"]
assert [f.form for f in T.read_filings(c, ent.key)] == ["10-K"]
assert [l.target for l in T.read_links(c, basis="sic")] == ["8082"]
assert T.read_links(c, basis="curated") == [curated] # survives
(fts_link,) = T.read_links(c, basis="fts")
assert (fts_link.scope, fts_link.target, fts_link.strength) == (
"program",
"PFS",
"mention",
)
assert fts_link.item_key and fts_link.passage_id.startswith(
"0000012345-23-000001:a10k.htm:"
)
# the link's passage is verbatim in the hoarded filing, short, and cited
(p,) = [p for p in T.read_passages(c) if p.passage_id == fts_link.passage_id]
assert len(p.passage) <= 400 and p.item_key == fts_link.item_key
text = edgar.read_doc(hoard / "sec/docs/0000012345/0000012345-23-000001/a10k.htm")
assert p.passage in text and p.label == "" and p.label_version == ""
assert len(T.read_hits(c)) == 4 and len(T.read_passages(c)) == 1
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert "2 piece(s) missing" in T.read_gaps(c)[0].note
logs = {
t
for (t,) in c.execute(
"SELECT table_name FROM cms.ingest_log WHERE module = 'market.edgar'"
).fetchall()
}
assert {"market.filing_hit", "market.filing_passage", "market.entity_link"} <= logs
assert published == [True] and store.count() == 1
# a rerun is idempotent and downloads nothing new
n_docs = _fts_net(monkeypatch, forbid_docs=True)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
assert (
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
)
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, status=403)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and _count(c, "market.entity") == 0
assert published == [True]
def test_fts_cli_withholds_ama_passages_and_drops_unsupported_links(env, monkeypatch):
c, _, _, _ = env
doc = f"<p>The Medicare Physician Fee Schedule: {LICENSED}.</p>".encode()
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": doc})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "passages kept 0, withheld by the AMA guard 1" in res.output
assert "fts links 0 (none)" in res.output
assert T.read_links(c, basis="fts") == [] and T.read_passages(c) == []
assert len(T.read_hits(c)) == 1 # the hit stays
def test_fts_cli_unverifiable_quote_drops_link(env, monkeypatch):
c, _, _, hoard = env
_fts_net(monkeypatch, hits=HITS[:1])
real = mk._verified
def tamper(links, passages):
for p in passages.values():
p.__dict__["passage"] = "not in the filing"
return real(links, passages)
monkeypatch.setattr(mk, "_verified", tamper)
res = runner.invoke(app, [*FTS, "--write"])
assert "fts links 0" in res.output and "quote not found in the hoard" in res.output
assert T.read_links(c, basis="fts") == []
def test_verified_handles_missing_filing_and_unpassaged_links(env):
_, _, _, hoard = env
link = T.LinkRow(
"1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "A:b.htm:0"
)
p = T.PassageRow("A:b.htm:0", "1", "A", "b.htm", "t", 0, "x")
assert mk._verified([link], {("A:b.htm:0", "t"): p}) == ([], 1)
bare = T.LinkRow("1", "program", "PFS", "fts", "mention", "adsh", "A:b.htm", "")
assert mk._verified([bare], {}) == ([], 1)
def test_fts_cli_limit_defers_downloads_and_writes_nothing(env, monkeypatch):
c, _, published, _ = env
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET, "b.htm": SNIPPET})
res = runner.invoke(app, [*FTS, "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "1 filing(s) not downloaded yet" in res.output
assert "incomplete (limit or time): nothing written" in res.output
assert published == []
# the newest filing was fetched first
assert "a10k.htm" in str(list(http.hoard("sec", "docs").rglob("*.htm")))
def test_fts_cli_deadline_stops_fetching(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch)
ticks = iter(range(0, 10_000, 100))
monkeypatch.setattr(mk.time, "monotonic", lambda: next(ticks))
res = runner.invoke(app, [*FTS, "--max-minutes", "1", "--write"])
assert res.exit_code == 0, res.output
assert "incomplete" in res.output and published == []
def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
(p,) = [p for p in T.read_passages(c) if p.passage.startswith("Substantially")]
c.execute(
"UPDATE market.filing_passage SET label = 'risk_factor', label_version = 'v' "
"WHERE passage_id = ?",
[p.passage_id],
)
_fts_net(monkeypatch, forbid_docs=True)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0 and "new entities 0" in res.output
got = {q.passage_id: q for q in T.read_passages(c)}
assert (
got[p.passage_id].label == "risk_factor"
and got[p.passage_id].label_version == "v"
)
(link,) = T.read_links(c, basis="fts")
assert link.strength == "risk_factor" and link.passage_id == p.passage_id
def test_fts_cli_missing_submissions_is_a_gap(env, monkeypatch):
c, _, _, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: (
httpx.Response(
200, json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
)
if "Medicare+Physician" in m.group(0)
or "Medicare%20Physician" in m.group(0)
else httpx.Response(
200, json={"hits": {"total": {"value": 0}, "hits": []}}
)
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(
r"company_tickers\.json",
lambda m: httpx.Response(
200, content=(FX / "company_tickers.json").read_bytes()
),
),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "CIK0000012345: no submissions, no entity" in res.output
assert T.read_entities(c) == []
# classification: the local model, resumable through the ledger
class _Ask:
model = "qwen-test"
def __init__(self, answers):
self.answers = answers
self.calls = 0
def __call__(self, system, user, schema):
self.calls += 1
return self.answers.pop(0)
TWO = [
HITS[0],
_hit_json("0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"),
]
TWO_DOCS = {"a10k.htm": SNIPPET, "b.htm": SNIPPET}
def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / f"{task}-{fam}.jsonl"
)
# a passage of a term outside this run's scope is never sent to the model
T.write_passages(
c, [T.PassageRow("X:y.htm:1", "9", "X", "y.htm", "other term", 1, "z")]
)
ask = _Ask(
[
{"label": "revenue_exposure", "reason": "r"},
None,
{"label": "incidental", "reason": "i"},
]
)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
labelled = [p for p in T.read_passages(c) if p.label]
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0 and ask.calls == 2
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
# a rerun without --classify keeps what is stored
res = runner.invoke(app, [*FTS, "--write"])
assert len([p for p in T.read_passages(c) if p.label]) == 1
got = sorted(l.strength for l in T.read_links(c, basis="fts"))
assert got == ["mention", "revenue_exposure"]
def test_fts_cli_incidental_label_makes_no_link(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask([{"label": "incidental", "reason": "i"}] * 2)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=TWO, docs=TWO_DOCS)
res = runner.invoke(app, [*FTS, "--classify", "--write"])
assert res.exit_code == 0, res.output
assert T.read_links(c, basis="fts") == [] and len(T.read_passages(c)) == 2
def test_asker_builds_json_extractor(monkeypatch):
sentinel = object()
seen = {}
monkeypatch.setattr("llm.config.load", lambda: "cfg")
monkeypatch.setattr(
"llm.pool.HostPool.from_config", classmethod(lambda cls, cfg: ("pool", cfg))
)
def make(cfg, pool):
seen["args"] = (cfg, pool)
return sentinel
monkeypatch.setattr("llm.structured.json_extractor", make)
assert mk._asker() is sentinel
assert seen["args"] == ("cfg", ("pool", "cfg"))
def test_gap_note_truncates():
note = mk._gap_note([f"i{n}" for n in range(20)])
assert note.startswith("20 piece(s) missing: i0; i1") and "... 5 more" in note
def test_fts_cli_submissions_refusal_is_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1])
_online(
monkeypatch,
[
(
r"efts\.sec\.gov/.*",
lambda m: httpx.Response(
200,
json={"hits": {"total": {"value": 1}, "hits": HITS[:1]}}
if "Medicare%20Physician" in m.group(0)
or "Medicare+Physician" in m.group(0)
else {"hits": {"total": {"value": 0}, "hits": []}},
),
),
(r"a10k\.htm$", lambda m: httpx.Response(200, content=SNIPPET)),
(r"CIK\d{10}\.json", lambda m: httpx.Response(403)),
(r"company_tickers\.json", lambda m: httpx.Response(403)),
],
)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts:" in res.output and "403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"]
assert _count(c, "market.filing_hit") == 0 and published == [True]
def test_fts_cli_long_gap_list_is_abridged(env, monkeypatch):
c, _, _, _ = env
hits = [
_hit_json("0000012346", f"0000012346-23-{n:06d}", "x.pdf", ["8082"])
for n in range(22)
]
_fts_net(monkeypatch, hits=hits)
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: ... 2 more" in res.output
assert "... 7 more" in T.read_gaps(c)[0].note
def test_fts_cli_flaky_document_is_skipped(env, monkeypatch):
hits = [
HITS[0],
_hit_json(
"0000012349", "0000012349-23-000001", "b.htm", ["8082"], "2022-03-01"
),
]
_fts_net(
monkeypatch, hits=hits, docs={"a10k.htm": SNIPPET}, broken={"b.htm": "timeout"}
)
monkeypatch.setattr("time.sleep", lambda s: None)
res = runner.invoke(app, FTS)
assert res.exit_code == 0, res.output
assert "b.htm: sec-edgar: HTTP 0 ReadTimeout" in res.output
assert "passages kept 1," in res.output
def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
c, _, published, _ = env
_fts_net(monkeypatch, hits=HITS[:1], broken={"a10k.htm": 403})
res = runner.invoke(app, [*FTS, "--write"])
assert res.exit_code == 0, res.output
assert "gap: sec-edgar:fts: sec-edgar: HTTP 403" in res.output
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
True
]

View File

@@ -0,0 +1,140 @@
{
"hits": {
"total": {
"value": 57,
"relation": "eq"
},
"max_score": 8.786024,
"hits": [
{
"_index": "edgar_file",
"_id": "0000790526-23-000003:rdnt-20221231.htm",
"_score": 8.786024,
"_source": {
"ciks": [
"0000790526"
],
"period_ending": "2022-12-31",
"file_num": [
"001-33307"
],
"display_names": [
"RadNet, Inc. (RDNT) (CIK 0000790526)"
],
"xsl": null,
"sequence": 1,
"root_forms": [
"10-K"
],
"file_date": "2023-03-01",
"biz_states": [
"CA"
],
"sics": [
"8071"
],
"form": "10-K",
"adsh": "0000790526-23-000003",
"film_num": [
"23695229"
],
"biz_locations": [
"Los Angeles, CA"
],
"file_type": "10-K",
"file_description": "10-K",
"inc_states": [
"DE"
],
"items": []
}
},
{
"_index": "edgar_file",
"_id": "0001798270-23-000007:ionm-20221231x10k.htm",
"_score": 8.094215,
"_source": {
"ciks": [
"0001798270"
],
"period_ending": "2022-12-31",
"file_num": [
"001-40785"
],
"display_names": [
"Assure Holdings Corp. (IONM) (CIK 0001798270)"
],
"xsl": null,
"sequence": 1,
"root_forms": [
"10-K"
],
"file_date": "2023-03-31",
"biz_states": [
"CO"
],
"sics": [
"3841"
],
"form": "10-K",
"adsh": "0001798270-23-000007",
"film_num": [
"23788435"
],
"biz_locations": [
"Greenwood Village, CO"
],
"file_type": "10-K",
"file_description": "10-K",
"inc_states": [
"NV"
],
"items": []
}
},
{
"_index": "edgar_file",
"_id": "0000355019-23-000044:fonar_10-k.htm",
"_score": 7.894161,
"_source": {
"ciks": [
"0000355019"
],
"period_ending": "2023-06-30",
"file_num": [
"000-10248"
],
"display_names": [
"FONAR CORP (FONR) (CIK 0000355019)"
],
"xsl": null,
"sequence": 1,
"root_forms": [
"10-K"
],
"file_date": "2023-09-28",
"biz_states": [
"NY"
],
"sics": [
"3845"
],
"form": "10-K",
"adsh": "0000355019-23-000044",
"film_num": [
"231289120"
],
"biz_locations": [
"Melville, NY"
],
"file_type": "10-K",
"file_description": "FONAR CORPORATION FORM 10-K AT JUNE 30, 2023",
"inc_states": [
"DE"
],
"items": []
}
}
]
}
}

View File

@@ -0,0 +1,7 @@
<html><head><style>p { color: red; }</style></head><body>
<ix:header><ix:hidden>HIDDEN XBRL TEXT physician fee schedule</ix:hidden></ix:header>
<p>Item 1. Business. We operate imaging centers across several states.</p>
<p>Substantially all of our Medicare revenue is paid under the Medicare Physician Fee Schedule, and the Centers for Medicare &amp; Medicaid Services&#8217; annual updates to the conversion factor affect our results. We cannot predict future rates.</p>
<p>Our contracts with commercial payors are described in Item 7.</p>
<p>Changes to the physician fee schedule could reduce our reimbursement.</p>
</body></html>

387
tests/market/test_fts.py Normal file
View File

@@ -0,0 +1,387 @@
"""market.edgar full-text search: queries, hits, passages, the AMA guard,
the local classifier and the links they produce."""
from __future__ import annotations
import json
from datetime import date
from pathlib import Path
import httpx
import pytest
from bib.store import Store
from market import edgar, http
from market import tables as T
from market.seed import PROGRAMS
FX = Path(__file__).parent / "fixtures"
class _Pacer:
def wait(self):
pass
@pytest.fixture
def hoard(monkeypatch, tmp_path):
monkeypatch.setattr(http, "hoard_root", lambda: tmp_path / "market")
return tmp_path / "market"
def test_fts_params_one_year():
p = edgar.fts_params("Medicare Physician Fee Schedule", "10-K", 2023, 0)
assert p["q"] == '"Medicare Physician Fee Schedule"'
assert p["startdt"] == "2023-01-01" and p["enddt"] == "2023-12-31"
assert p["from"] == 0 and p["forms"] == "10-K" and p["dateRange"] == "custom"
def test_parse_hits():
payload = json.loads((FX / "efts_mpfs_2023.json").read_text())
rows = edgar.parse_hits(
payload, term="Medicare Physician Fee Schedule", scope="program", target="PFS"
)
assert len(rows) == 3
assert rows[0] == T.HitRow(
term="Medicare Physician Fee Schedule",
scope="program",
target="PFS",
cik="0000790526",
adsh="0000790526-23-000003",
filename="rdnt-20221231.htm",
form="10-K",
file_date=date(2023, 3, 1),
period_ending=date(2022, 12, 31),
sics="8071",
display_name="RadNet, Inc. (RDNT) (CIK 0000790526)",
)
def test_parse_hits_without_optional_fields():
payload = {
"hits": {
"hits": [
{
"_id": "0000000001-23-000001:a.htm",
"_source": {"ciks": ["1"], "file_date": "2023-01-02"},
}
]
}
}
(h,) = edgar.parse_hits(payload, term="t", scope="family", target="CCM")
assert (h.cik, h.period_ending, h.sics, h.display_name, h.form) == (
"0000000001",
None,
"",
"",
"",
)
assert edgar.parse_hits({}, term="t", scope="program", target="PFS") == []
def _pages(total):
def handler(request: httpx.Request):
q = dict(request.url.params)
return httpx.Response(
200,
json={
"hits": {
"total": {"value": total},
"hits": [
{
"_id": f"0000000001-23-00000{q['from']}:a.htm",
"_source": {"ciks": ["1"], "file_date": "2023-01-02"},
}
],
}
},
)
return handler
def test_fts_pages_until_total(hoard):
cl = httpx.Client(transport=httpx.MockTransport(_pages(150)))
got = list(edgar.fetch_fts(cl, "RBRVS", "10-K", 2023, pacer=_Pacer()))
assert [off for off, _ in got] == [0, 100]
cached = hoard / "sec" / "fts" / "rbrvs" / "10-K" / "2023.100.json"
assert cached.is_file()
assert edgar.fts_cached("RBRVS", "10-K", 2023)
assert not edgar.fts_cached("RBRVS", "10-K", 2022)
def test_fts_pages_come_from_the_hoard_and_offline_stops(hoard):
cl = httpx.Client(transport=httpx.MockTransport(_pages(50)))
first = list(edgar.fetch_fts(cl, "RBRVS", "10-K/A", 2023, pacer=_Pacer()))
assert (hoard / "sec/fts/rbrvs/10-K_A/2023.0.json").is_file()
again = list(
edgar.fetch_fts(None, "RBRVS", "10-K/A", 2023, pacer=_Pacer(), offline=True)
)
assert again == first
assert (
list(edgar.fetch_fts(None, "RBRVS", "10-K", 1999, pacer=_Pacer(), offline=True))
== []
)
def test_doc_url():
assert (
edgar.doc_url("0000790526", "0000790526-23-000003", "rdnt-20221231.htm")
== "https://www.sec.gov/Archives/edgar/data/790526/000079052623000003/"
"rdnt-20221231.htm"
)
def test_family_terms():
from pfs.families import FAMILIES
terms = edgar.family_terms(FAMILIES["HOME-VISITS"])
low = [t.lower() for t in terms]
assert "house calls" in low and "home or residence" in low
assert "domiciliary" not in low
assert len(low) == len(set(low)) and terms[0] == FAMILIES["HOME-VISITS"].name
def test_program_terms_keys_are_programs():
assert set(edgar.PROGRAM_TERMS) == set(PROGRAMS)
assert "RBRVS" in edgar.PROGRAM_TERMS["PFS"]
def test_passages_window_and_cap():
text = (
"x " * 500
+ "We bill under the Medicare Physician Fee Schedule today. "
+ "y " * 500
)
at = text.index("Medicare Physician")
(one, *_) = edgar.passages(text, "medicare physician fee schedule")
start, p = one
assert start <= at < start + len(p) and len(p) <= 400
assert "Medicare Physician Fee Schedule" in p
assert text[start : start + len(p)] == p
# a short sentence is returned whole
short = "Intro. Our pay follows RBRVS rules. Next."
assert edgar.passages(short, "rbrvs") == [(7, "Our pay follows RBRVS rules.")]
many = " ".join(f"{'filler ' * 200} RBRVS applies." for _ in range(5))
got = edgar.passages(many, "rbrvs")
assert len(got) == 3 and all(len(p) <= 400 for _, p in got)
assert edgar.passages("nothing here", "rbrvs") == []
def test_passages_dedupe_same_sentence():
got = edgar.passages("Both RBRVS and RBRVS appear here.", "rbrvs")
assert got == [(0, "Both RBRVS and RBRVS appear here.")]
def test_normalize_and_snippet_passages():
text = edgar.normalize_text((FX / "filing_snippet.htm").read_text())
assert "HIDDEN" not in text and "color: red" not in text
assert "Medicaid Services' annual" in text # entity + curly quote decoded
(a, b) = edgar.passages(text, "physician fee schedule")
assert a[1].startswith("Substantially all of our Medicare revenue")
assert (
b[1] == "Changes to the physician fee schedule could reduce our reimbursement."
)
assert edgar.quote_in(text, a[1])
DESC = "Office or other outpatient visit for the evaluation and management of an established patient"
def test_ama_guard_matches_shares_run():
from pfs.origin import shares_run
pairs = [
("We see patients: " + DESC + " each day.", DESC),
("Office visits are our business and management of patients too", DESC),
("short", DESC),
]
for passage, d in pairs:
assert edgar.ama_safe(passage, edgar.cpt_grams([d])) == (
not shares_run(passage, d)
)
assert edgar.cpt_grams(["a b c"], 8) == frozenset()
def test_safe_passages_retry_narrower_then_withhold():
grams = edgar.cpt_grams([DESC])
long = f"{'pad ' * 60}Our term RBRVS is used. {DESC} is one service. {'pad ' * 60}"
kept, held = edgar.safe_passages(long, "rbrvs", grams)
assert held == 0 and kept and "RBRVS" in kept[0][1]
assert all(edgar.ama_safe(p, grams) for _, p in kept)
# a term inside the descriptor cannot be separated from it
inside = f"{DESC} for RBRVS"
short = f"The text {DESC}."
assert edgar.safe_passages(short, "evaluation", grams) == ([], 1)
assert edgar.safe_passages(inside, "rbrvs", grams)[1] in (0, 1)
def test_passage_quoting_descriptor_without_code_withheld():
grams = edgar.cpt_grams([DESC])
text = f"We provide {DESC}. This is billed under RBRVS."
assert edgar.safe_passages(text, "established patient", grams) == ([], 1)
h = T.HitRow(
"established patient",
"program",
"PFS",
"0000790526",
"0000790526-23-000003",
"a.htm",
"10-K",
date(2023, 3, 1),
None,
"8071",
"RadNet",
)
(link,) = edgar.fts_links([h], [], [], [])
assert link.passage_id == "" and link.strength == "mention"
assert link.cite_ref == "0000790526-23-000003:a.htm"
def _hit(cik="0000790526", sics="8071", adsh="0000790526-23-000003", **kw):
base = dict(
term="rbrvs",
scope="program",
target="PFS",
cik=cik,
adsh=adsh,
filename="a.htm",
form="10-K",
file_date=date(2023, 3, 1),
period_ending=None,
sics=sics,
display_name="X",
)
base.update(kw)
return T.HitRow(**base)
def _passage(h, label="", start=10):
return T.PassageRow(
f"{h.adsh}:{h.filename}:{start}",
h.cik,
h.adsh,
h.filename,
h.term,
start,
"text",
label,
)
def test_fts_links_require_health_sic_or_seed():
ok, other = _hit(), _hit(cik="0000000002", sics="6798")
links = edgar.fts_links([ok, other], [], [], [])
assert [l.entity_key for l in links] == ["0000790526"]
assert links[0].basis == "fts" and links[0].cite_kind == "adsh"
# a seed CIK links whatever its SIC; an entity's own SIC counts too
assert (
edgar.fts_links([other], [], [], ["0000000002"])[0].entity_key == "0000000002"
)
ent = T.EntityRow("0000000002", "E", "company", cik="0000000002", sic="8082")
assert len(edgar.fts_links([_hit(cik="0000000002", sics="")], [], [ent], [])) == 1
assert edgar.fts_links([_hit(cik="0000000002", sics="")], [], [], []) == []
def test_link_strength_from_label():
h = _hit()
for label, want in (("", "mention"), ("revenue_exposure", "revenue_exposure")):
(l,) = edgar.fts_links([h], [_passage(h, label)], [], [])
assert l.strength == want and l.passage_id == f"{h.adsh}:a.htm:10"
assert edgar.fts_links([h], [_passage(h, "incidental")], [], []) == []
def test_best_evidence_wins_per_target():
old = _hit(adsh="0000790526-20-000001", file_date=date(2020, 3, 1))
new = _hit(adsh="0000790526-23-000003")
got = edgar.fts_links(
[old, new],
[_passage(old, "risk_factor"), _passage(new, "revenue_exposure")],
[],
[],
)
assert len(got) == 1 and got[0].strength == "revenue_exposure"
# equal strength: the newest filing
got = edgar.fts_links([old, new], [_passage(old), _passage(new)], [], [])
assert got[0].cite_ref.startswith("0000790526-23")
# a labelled passage beats an unlabelled sibling
got = edgar.fts_links(
[new], [_passage(new, "", 5), _passage(new, "risk_factor", 9)], [], []
)
assert got[0].strength == "risk_factor"
def test_classify_passage_closed_labels():
seen = []
def ask(system, user, schema):
seen.append((system, user, schema))
return {"label": "revenue_exposure", "reason": "paid under it"}
assert edgar.classify_passage("p", "rbrvs", ask) == "revenue_exposure"
assert "rbrvs" in seen[0][1] and seen[0][2]["properties"]["label"]["enum"] == list(
edgar.EXPOSURE_LABELS
)
assert edgar.classify_passage("p", "t", lambda *a: {"label": "bogus"}) is None
assert edgar.classify_passage("p", "t", lambda *a: None) is None
assert edgar.EXPOSURE_PROMPT_VERSION == "exposure-v1"
def test_fetch_doc_hoards_and_reads(hoard):
h = _hit()
calls = []
def handler(request):
calls.append(str(request.url))
if request.url.path.endswith("gone.htm"):
return httpx.Response(404)
return httpx.Response(200, content=(FX / "filing_snippet.htm").read_bytes())
cl = httpx.Client(transport=httpx.MockTransport(handler))
path = edgar.fetch_doc(cl, h, pacer=_Pacer())
assert path == hoard / "sec/docs/0000790526/0000790526-23-000003/a.htm"
assert "Substantially all" in edgar.read_doc(path)
again = edgar.fetch_doc(cl, h, pacer=_Pacer())
assert again == path and len(calls) == 1 # never refetched
gone = edgar.fetch_doc(cl, _hit(filename="gone.htm"), pacer=_Pacer())
assert gone is None
assert (
edgar.fetch_doc(None, _hit(filename="x.htm"), pacer=_Pacer(), offline=True)
is None
)
def test_cite_filing_attaches_document(tmp_path):
store = Store(":memory:", storage_dir=tmp_path / "st")
doc = tmp_path / "a.htm"
doc.write_text("<html>doc</html>")
h = _hit(display_name="RadNet, Inc.")
key = edgar.cite_filing(store, h, doc)
assert edgar.cite_filing(store, h, doc) == key
item = store.get(key)
assert item.title == (
"RadNet, Inc. Form 10-K filed 2023-03-01 (accession 0000790526-23-000003)"
)
assert item.website_title == "U.S. Securities and Exchange Commission"
names = {t["name"] if isinstance(t, dict) else t[0] for t in store.list_tags()}
assert {"module:market", "source:sec-edgar", "year:2023"} <= names
assert "table:market.filing_passage" in names
n = store._con().execute("SELECT count(*) FROM attachments").fetchone()[0]
assert n == 1
store.close()
def test_window_snaps_to_word_boundaries_in_a_long_sentence():
words = " ".join(f"word{i}" for i in range(400)) # one 2,800-char sentence
mid = words.index("word200")
(start, p), *_ = edgar.passages(words + ".", "word200")
assert len(p) <= 400 and p.startswith("word") and p.split()[-1].startswith("word")
assert start <= mid and "word200" in p
# a run with no spaces cannot be snapped: still capped, term kept
blob = "z" * 900 + "RBRVS" + "z" * 900
(start, p), *_ = edgar.passages(blob, "rbrvs")
assert len(p) == 400 and "RBRVS" in p and blob[start : start + 400] == p
# near the end of the text the window is pulled back to full size
tail = ("q " * 600) + "RBRVS"
(start, p), *_ = edgar.passages(tail, "rbrvs")
assert p.endswith("RBRVS") and len(p) <= 400

View File

@@ -109,6 +109,42 @@ def test_get_5xx_exhausted_raises_refused():
assert ei.value.status == 502 and slept == [2.0, 4.0]
def test_get_retries_transport_errors_then_refuses():
slept = []
calls = []
def boom(request):
calls.append(1)
raise httpx.ReadTimeout("slow", request=request)
with pytest.raises(Refused) as ei:
get(
_cl(boom),
"https://x.test/a",
pacer=_Pacer(),
source="s",
sleep=slept.append,
retries=2,
)
assert ei.value.status == 0 and "ReadTimeout" in str(ei.value)
assert slept == [2.0, 4.0] and len(calls) == 3
def test_get_transport_error_then_ok():
state = []
def flaky(request):
if not state:
state.append(1)
raise httpx.ConnectError("reset", request=request)
return httpx.Response(200, text="ok")
r = get(
_cl(flaky), "https://x.test/a", pacer=_Pacer(), source="s", sleep=lambda s: None
)
assert r.text == "ok"
def test_get_429_retry_after_waits_once():
slept = []
cl = _cl(

View File

@@ -36,6 +36,8 @@ def test_ensure_tables_idempotent():
"security",
"entity_link",
"filing",
"filing_hit",
"filing_passage",
"source_gap",
"price_daily",
"series",
@@ -75,6 +77,31 @@ FILING = T.FilingRow(
"d.htm",
)
GAP = T.GapRow("sec-edgar:entities", "403", dt.date.today())
HIT = T.HitRow(
"RBRVS",
"program",
"PFS",
"0000790526",
"0000790526-23-000003",
"rdnt-20221231.htm",
"10-K",
date(2023, 3, 1),
date(2022, 12, 31),
"8071",
"RadNet, Inc. (RDNT)",
)
PASSAGE = T.PassageRow(
"0000790526-23-000003:rdnt-20221231.htm:42",
"0000790526",
"0000790526-23-000003",
"rdnt-20221231.htm",
"RBRVS",
42,
"Our pay follows RBRVS rules.",
"revenue_exposure",
"model-reading exposure-v1 m",
"K",
)
PRICE = TI.PriceRow(
"OSH",
@@ -121,6 +148,16 @@ ROUNDTRIP = {
lambda c, r: TI.write_series(c, r.series_id, [r]),
lambda c: TI.read_series(c, ["OSH"]),
),
"filing_hit": (
HIT,
lambda c, r: T.write_hits(c, [r]),
lambda c: T.read_hits(c),
),
"filing_passage": (
PASSAGE,
lambda c, r: T.write_passages(c, [r]),
lambda c: T.read_passages(c),
),
"gap": (
GAP,
lambda c, r: T.write_gap(c, r.source, r.note),
@@ -350,3 +387,24 @@ def test_ensure_tables_adds_priced_to_an_existing_security_table():
(old,) = T.read_securities(c)
assert old.priced is True
c.close()
def test_hits_and_passages_upsert_by_their_keys(con):
T.write_hits(con, [HIT, dataclasses_replace(HIT, term="x")])
T.write_hits(con, [dataclasses_replace(HIT, sics="8082")])
assert sorted((h.term, h.sics) for h in T.read_hits(con)) == [
("RBRVS", "8082"),
("x", "8071"),
]
sibling = dataclasses_replace(PASSAGE, term="x", label="")
T.write_passages(con, [PASSAGE, sibling])
T.write_passages(con, [dataclasses_replace(PASSAGE, passage="new text", label="")])
got = {p.term: p.passage for p in T.read_passages(con)}
assert got == {"RBRVS": "new text", "x": PASSAGE.passage}
assert T.write_hits(con, []) == 0 and T.write_passages(con, []) == 0
def dataclasses_replace(row, **kw):
import dataclasses
return dataclasses.replace(row, **kw)