fix(market): fts strength gate and common-share universe, classify links in rounds, remembered 404s, pre-2002 full-text fallback, plain-frozenset AMA check
This commit is contained in:
@@ -642,13 +642,12 @@ def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRo
|
||||
|
||||
|
||||
def _classify(
|
||||
kept: list[PassageRow], limit: int
|
||||
kept: list[PassageRow], limit: int, ask: Any
|
||||
) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]:
|
||||
"""Label *kept* with the local model, resumably (``pfs.runlog``). Returns
|
||||
the ledger outputs by unit, the run counts and the model name."""
|
||||
"""Label *kept* with the local model *ask*, resumably (``pfs.runlog``).
|
||||
Returns the ledger outputs by unit, the run counts and the model name."""
|
||||
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
|
||||
|
||||
ask = _asker()
|
||||
model = getattr(ask, "model", "local")
|
||||
version = edgar.EXPOSURE_PROMPT_VERSION
|
||||
log = RunLog(ledger_path("market-exposure", "all"))
|
||||
@@ -668,6 +667,55 @@ def _classify(
|
||||
return log.outputs(model, version), counts, model
|
||||
|
||||
|
||||
def _classify_links(
|
||||
merged: dict[tuple[str, str], PassageRow],
|
||||
derive: Any,
|
||||
hits: list[HitRow],
|
||||
owners: dict[str, Any],
|
||||
limit: int,
|
||||
) -> None:
|
||||
"""Label the passage behind every live link, in rounds: an ``incidental``
|
||||
label retires that passage, the link falls back to the next candidate
|
||||
passage, and the next round reads that one, until every link rests on a
|
||||
labelled passage or the candidates run out. *limit* caps the model calls
|
||||
(0 = no cap); a rerun resumes from the ledger."""
|
||||
ask = _asker()
|
||||
filed = {(h.adsh, h.filename): h.file_date for h in hits}
|
||||
total: Counter[str] = Counter()
|
||||
rounds = 0
|
||||
while True:
|
||||
want = {lk.passage_id for lk in derive()}
|
||||
todo = sorted(
|
||||
(
|
||||
p
|
||||
for p in merged.values()
|
||||
if p.passage_id in want and not p.label and p.term in owners
|
||||
),
|
||||
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
|
||||
)
|
||||
if not todo:
|
||||
break
|
||||
remaining = limit - total["done"] - total["failed"] if limit else 0
|
||||
if limit and remaining <= 0:
|
||||
break
|
||||
outputs, counts, model = _classify(todo, remaining, ask)
|
||||
total.update(counts)
|
||||
rounds += 1
|
||||
fresh = 0
|
||||
for k, p in list(merged.items()):
|
||||
got = outputs.get(f"{p.passage_id}|{p.term}")
|
||||
if got and not p.label:
|
||||
fresh += 1
|
||||
merged[k] = dataclasses.replace(
|
||||
p,
|
||||
label=got["label"],
|
||||
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
|
||||
)
|
||||
if not fresh:
|
||||
break
|
||||
typer.echo(f"classified in {rounds} round(s): {dict(total)}")
|
||||
|
||||
|
||||
def _verified(
|
||||
links: list[LinkRow], passage_rows: dict[Any, PassageRow]
|
||||
) -> tuple[list[LinkRow], int]:
|
||||
@@ -681,10 +729,8 @@ def _verified(
|
||||
if p is None:
|
||||
continue
|
||||
if (p.adsh, p.filename) not in texts:
|
||||
path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename))
|
||||
texts[(p.adsh, p.filename)] = (
|
||||
edgar.read_doc(path) if path.is_file() else None
|
||||
)
|
||||
path = edgar.hoarded_doc(p.cik, p.adsh, p.filename)
|
||||
texts[(p.adsh, p.filename)] = edgar.read_doc(path) if path else None
|
||||
text = texts[(p.adsh, p.filename)]
|
||||
if text is not None and len(p.passage) <= 400 and p.passage in text:
|
||||
kept.append(lk)
|
||||
@@ -797,6 +843,7 @@ def fts(
|
||||
seed_ciks = set(seeds)
|
||||
health = set(edgar.HEALTH_SICS)
|
||||
known = {e.key for e in prior_entities}
|
||||
stored_passages = {(p.adsh, p.filename, p.term) for p in prior_passages}
|
||||
|
||||
new_hits: dict[tuple[str, str, str, str, str], HitRow] = {}
|
||||
new_passages: list[PassageRow] = []
|
||||
@@ -837,11 +884,14 @@ def fts(
|
||||
order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k))
|
||||
for cik, adsh, filename in order:
|
||||
hs = docs[(cik, adsh, filename)]
|
||||
if not refresh and all(
|
||||
(adsh, filename, h.term) in stored_passages for h in hs
|
||||
):
|
||||
continue # already read: its passages are stored
|
||||
if not filename.lower().endswith(edgar.READABLE):
|
||||
issues.append(f"{adsh}:{filename}: format not readable as text")
|
||||
continue
|
||||
path = http.hoard("sec", "docs", cik, adsh, filename)
|
||||
have = http.newest(path).is_file()
|
||||
have = edgar.hoarded_doc(cik, adsh, filename) is not None
|
||||
if not have and ((limit and fetched >= limit) or out_of_time()):
|
||||
pending += 1
|
||||
continue
|
||||
@@ -903,31 +953,21 @@ def fts(
|
||||
)
|
||||
merged[(p.passage_id, p.term)] = p
|
||||
|
||||
if classify:
|
||||
filed = {(h.adsh, h.filename): h.file_date for h in hits.values()}
|
||||
todo = sorted(
|
||||
(p for p in merged.values() if p.term in owners),
|
||||
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
|
||||
)
|
||||
outputs, counts, model = _classify(todo, limit)
|
||||
typer.echo(f"classified with {model}: {counts}")
|
||||
for k, p in list(merged.items()):
|
||||
got = outputs.get(f"{p.passage_id}|{p.term}")
|
||||
if got:
|
||||
merged[k] = dataclasses.replace(
|
||||
p,
|
||||
label=got["label"],
|
||||
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
|
||||
)
|
||||
entities = list(prior_entities)
|
||||
all_hits = list(hits.values())
|
||||
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
|
||||
linkable = [
|
||||
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
|
||||
]
|
||||
links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
|
||||
mine = {(sc, t) for sc, t, _ in scope}
|
||||
links = [lk for lk in links if (lk.scope, lk.target) in mine]
|
||||
|
||||
def derive() -> list[LinkRow]:
|
||||
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
|
||||
linkable = [
|
||||
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
|
||||
]
|
||||
found = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
|
||||
return [lk for lk in found if (lk.scope, lk.target) in mine]
|
||||
|
||||
if classify:
|
||||
_classify_links(merged, derive, all_hits, owners, limit)
|
||||
links = derive()
|
||||
links, unverified = _verified(links, merged)
|
||||
if unverified:
|
||||
issues.append(f"{unverified} link(s) dropped: quote not found in the hoard")
|
||||
@@ -966,7 +1006,7 @@ def fts(
|
||||
if (adsh, filename) not in item_of:
|
||||
h = docs_of[(adsh, filename)]
|
||||
item_of[(adsh, filename)] = edgar.cite_filing(
|
||||
store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename))
|
||||
store, h, edgar.hoarded_doc(h.cik, adsh, filename)
|
||||
)
|
||||
links = [
|
||||
dataclasses.replace(
|
||||
|
||||
@@ -525,7 +525,22 @@ class AmaIndex(frozenset):
|
||||
for d in descriptors:
|
||||
w = _WORD.findall(d.lower())
|
||||
grams.update(tuple(w[i : i + n]) for i in range(len(w) - n + 1))
|
||||
return cls(grams)
|
||||
index = cls(grams)
|
||||
index.plain = frozenset(grams)
|
||||
return index
|
||||
|
||||
@property
|
||||
def plain(self) -> frozenset:
|
||||
"""The grams as a plain ``frozenset``: ``isdisjoint`` against a
|
||||
frozenset subclass takes a slow path (3,213 strings: 187 s)."""
|
||||
got = self.__dict__.get("_plain")
|
||||
if got is None:
|
||||
got = self.__dict__["_plain"] = frozenset(self)
|
||||
return got
|
||||
|
||||
@plain.setter
|
||||
def plain(self, value: frozenset) -> None:
|
||||
self.__dict__["_plain"] = value
|
||||
|
||||
|
||||
def shares_ama_run(
|
||||
@@ -543,7 +558,7 @@ def shares_ama_run(
|
||||
if isinstance(descriptors, AmaIndex)
|
||||
else AmaIndex.of(descriptors, n)
|
||||
)
|
||||
return not grams.isdisjoint(index)
|
||||
return not grams.isdisjoint(index.plain)
|
||||
|
||||
|
||||
# ── full-text search ─────────────────────────────────────────────────
|
||||
@@ -671,6 +686,30 @@ def fetch_fts(
|
||||
READABLE = (".htm", ".html", ".txt")
|
||||
|
||||
|
||||
#: filings filed before this year may exist only as the full submission text
|
||||
FULL_TEXT_BEFORE = 2002
|
||||
|
||||
|
||||
def doc_paths(cik: str, adsh: str, filename: str) -> tuple[Path, Path]:
|
||||
"""Where a hit's document is hoarded: its own file, or (an early filing)
|
||||
the full-submission text ``<adsh>.txt``."""
|
||||
base = ("sec", "docs", cik, adsh)
|
||||
return http.hoard(*base, filename), http.hoard(*base, f"{adsh}.txt")
|
||||
|
||||
|
||||
def hoarded_doc(cik: str, adsh: str, filename: str) -> Path | None:
|
||||
"""The newest hoarded copy of a hit's document, if any."""
|
||||
for path in doc_paths(cik, adsh, filename):
|
||||
have = http.newest(path)
|
||||
if have.is_file():
|
||||
return have
|
||||
return None
|
||||
|
||||
|
||||
def _marker(path: Path) -> Path:
|
||||
return path.with_name(path.name + ".404")
|
||||
|
||||
|
||||
def fetch_doc(
|
||||
cl: httpx.Client | None,
|
||||
hit: HitRow,
|
||||
@@ -680,18 +719,45 @@ def fetch_doc(
|
||||
offline: bool = False,
|
||||
) -> Path | None:
|
||||
"""The hoarded copy of the hit's document (``sec/docs/<cik>/<adsh>/<file>``),
|
||||
downloaded when absent and not *offline*; ``None`` when unavailable."""
|
||||
path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename)
|
||||
body = _fetch(
|
||||
cl,
|
||||
doc_url(hit.cik, hit.adsh, hit.filename),
|
||||
path,
|
||||
pacer=pacer,
|
||||
refresh=refresh,
|
||||
offline=offline,
|
||||
ok_404=True,
|
||||
)
|
||||
return http.newest(path) if body is not None else None
|
||||
downloaded when absent and not *offline*. A filing from before 2002 that
|
||||
has no such file is tried as the full submission ``<adsh>.txt``. A 404 is
|
||||
remembered in a ``.404`` marker beside the path and not asked again
|
||||
(unless *refresh*); ``None`` when the document is unavailable."""
|
||||
path, full = doc_paths(hit.cik, hit.adsh, hit.filename)
|
||||
marker = _marker(path)
|
||||
have = hoarded_doc(hit.cik, hit.adsh, hit.filename)
|
||||
if not refresh and (have is not None or marker.is_file()):
|
||||
if have is not None:
|
||||
http.manifest(have, doc_url(hit.cik, hit.adsh, hit.filename))
|
||||
return have
|
||||
attempts = [(doc_url(hit.cik, hit.adsh, hit.filename), path)]
|
||||
if hit.file_date.year < FULL_TEXT_BEFORE:
|
||||
attempts.append(
|
||||
(
|
||||
f"https://www.sec.gov/Archives/edgar/data/{int(hit.cik)}/{hit.adsh}.txt",
|
||||
full,
|
||||
)
|
||||
)
|
||||
for url, target in attempts:
|
||||
body = _fetch(
|
||||
cl,
|
||||
url,
|
||||
target,
|
||||
pacer=pacer,
|
||||
refresh=refresh,
|
||||
offline=offline,
|
||||
ok_404=True,
|
||||
)
|
||||
if body is not None:
|
||||
return http.newest(target)
|
||||
if cl is not None and not offline:
|
||||
marker.parent.mkdir(parents=True, exist_ok=True)
|
||||
marker.write_text(
|
||||
json.dumps(
|
||||
{"url": attempts[0][0], "status": 404, "checked": str(dt.date.today())}
|
||||
)
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def read_doc(path: Path) -> str:
|
||||
|
||||
@@ -58,6 +58,36 @@ PACE_SECONDS = 75.0
|
||||
NOT_SUPPORTED = "not on Tiingo's supported list"
|
||||
NOT_PRICED = "symbol record only (priced=false), never sent to Tiingo"
|
||||
_STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
|
||||
#: exchanges whose common shares may be priced
|
||||
LISTED = ("NYSE", "NASDAQ")
|
||||
#: a 5-letter symbol ending in one of these is a warrant, unit, right or
|
||||
#: preferred line of the 4-letter common symbol
|
||||
_NON_COMMON_END = "WURPZ"
|
||||
#: SICs of healthcare-services companies (80xx, health insurers)
|
||||
SERVICES_SICS = ("6324",)
|
||||
|
||||
|
||||
def is_common_share(ticker: str, exchange: str | None) -> bool:
|
||||
"""A common share on NYSE or Nasdaq: no OTC or empty exchange, no
|
||||
class/series punctuation, no warrant, unit, right or preferred suffix."""
|
||||
if (exchange or "").upper() not in LISTED:
|
||||
return False
|
||||
if not (ticker.isalpha() and ticker.isupper() and len(ticker) <= 5):
|
||||
return False
|
||||
return not (len(ticker) == 5 and ticker[-1] in _NON_COMMON_END)
|
||||
|
||||
|
||||
def is_services_sic(sic: str) -> bool:
|
||||
return sic.startswith("80") or sic in SERVICES_SICS
|
||||
|
||||
|
||||
def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool:
|
||||
"""The strength gate of the filing-text tier: a model-labelled strength of
|
||||
``risk_factor`` or higher, at least two distinct targets, or a
|
||||
healthcare-services SIC. A bare single-target mention never qualifies."""
|
||||
return (
|
||||
best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic)
|
||||
)
|
||||
|
||||
|
||||
def new_pacer() -> http.Pacer:
|
||||
@@ -414,14 +444,27 @@ def universe(
|
||||
"""``(ticker, entity_key)`` in rank order: the benchmarks, the seed
|
||||
securities in seed order, then the securities of entities with a
|
||||
filing-text (``fts``) link, ranked by strongest link, number of distinct
|
||||
link targets and newest 10-K. SIC-only entities and unsupported or
|
||||
unpriced tickers are excluded; at most *cap* symbols not already in
|
||||
link targets and newest 10-K. An fts entity needs a model-labelled
|
||||
strength of risk_factor or higher, two distinct targets or a
|
||||
healthcare-services SIC (a bare single-target mention never prices), and
|
||||
only its one primary NYSE or Nasdaq common share is priced (no OTC,
|
||||
warrant, unit, right or preferred line). SIC-only entities and
|
||||
unsupported or unpriced tickers are excluded; at most *cap* symbols not already in
|
||||
*used* are returned. Each unsupported ticker is recorded in *skipped*
|
||||
(``{ticker: reason}``) so the caller can turn it into a gap."""
|
||||
by_entity: dict[str, list[str]] = defaultdict(list)
|
||||
curated: dict[str, list[str]] = defaultdict(list)
|
||||
commons: dict[str, list[str]] = defaultdict(list)
|
||||
for s in read_securities(con):
|
||||
if s.priced:
|
||||
by_entity[s.entity_key].append(s.ticker)
|
||||
if not s.priced:
|
||||
continue
|
||||
if s.source == "seed":
|
||||
curated[s.entity_key].append(s.ticker)
|
||||
elif is_common_share(s.ticker, s.exchange):
|
||||
commons[s.entity_key].append(s.ticker)
|
||||
for key in {*curated, *commons}:
|
||||
# curated tickers stay; otherwise one primary common share
|
||||
by_entity[key] = curated[key] or [min(commons[key], key=lambda t: (len(t), t))]
|
||||
|
||||
ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS]
|
||||
for key in seed_order if seed_order is not None else _seed_ciks():
|
||||
@@ -433,6 +476,12 @@ def universe(
|
||||
rank = _STRENGTH.get(link.strength, len(_STRENGTH))
|
||||
best[link.entity_key] = min(best.get(link.entity_key, rank), rank)
|
||||
targets[link.entity_key].add(link.target)
|
||||
sic_of = dict(con.execute("SELECT key, sic FROM market.entity").fetchall())
|
||||
best = {
|
||||
k: r
|
||||
for k, r in best.items()
|
||||
if fts_qualifies(r, len(targets[k]), sic_of.get(k) or "")
|
||||
}
|
||||
newest = {
|
||||
cik: filed
|
||||
for cik, filed in con.execute(
|
||||
|
||||
@@ -739,7 +739,10 @@ def test_prices_dry_run_reports_budget(px, monkeypatch):
|
||||
c, _, published, _, seen, _, state = px
|
||||
T.ensure_tables(c)
|
||||
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
|
||||
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
|
||||
for target in ("PFS", "APM"): # two targets pass the fts strength gate
|
||||
T.write_links(
|
||||
c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")]
|
||||
)
|
||||
state.mkdir(parents=True)
|
||||
(state / "tiingo-2026-11.json").write_text(
|
||||
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
|
||||
@@ -1211,11 +1214,14 @@ def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
|
||||
assert published == [True] and store.count() == 1
|
||||
# a rerun is idempotent and downloads nothing new
|
||||
n_docs = _fts_net(monkeypatch, forbid_docs=True)
|
||||
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
|
||||
res2 = runner.invoke(app, [*FTS, "--write"])
|
||||
assert res2.exit_code == 0
|
||||
assert "passages kept 0," in res2.output # stored passages are not re-read
|
||||
assert (
|
||||
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
|
||||
)
|
||||
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
|
||||
assert n_docs == [] # the 404 is remembered, not asked again
|
||||
assert "gone.htm: document not found" in res2.output
|
||||
|
||||
|
||||
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
|
||||
@@ -1306,8 +1312,8 @@ def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
|
||||
"WHERE passage_id = ?",
|
||||
[p.passage_id],
|
||||
)
|
||||
_fts_net(monkeypatch, forbid_docs=True)
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
_fts_net(monkeypatch) # a refresh reads the documents again
|
||||
res = runner.invoke(app, [*FTS, "--refresh", "--write"])
|
||||
assert res.exit_code == 0 and "new entities 0" in res.output
|
||||
got = {q.passage_id: q for q in T.read_passages(c)}
|
||||
assert (
|
||||
@@ -1393,14 +1399,14 @@ def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
|
||||
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
|
||||
assert "classified in 1 round(s): {'done': 1" in res.output and ask.calls == 1
|
||||
labelled = [p for p in T.read_passages(c) if p.label]
|
||||
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
|
||||
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
|
||||
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0 and ask.calls == 2
|
||||
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
|
||||
assert "'failed': 1" in res.output
|
||||
# a rerun without --classify keeps what is stored
|
||||
res = runner.invoke(app, [*FTS, "--write"])
|
||||
assert len([p for p in T.read_passages(c) if p.label]) == 1
|
||||
@@ -1510,3 +1516,51 @@ def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
|
||||
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
|
||||
True
|
||||
]
|
||||
|
||||
|
||||
TWICE = (
|
||||
b"<p>First, we are paid under the Medicare Physician Fee Schedule rates.</p>"
|
||||
b"<p>Second, our risk comes from the Medicare Physician Fee Schedule cuts.</p>"
|
||||
)
|
||||
|
||||
|
||||
def test_fts_cli_classify_falls_back_to_the_next_passage(env, monkeypatch, tmp_path):
|
||||
"""An incidental first passage retires it; the link moves to the next
|
||||
passage, which the next round reads."""
|
||||
c, _, _, _ = env
|
||||
monkeypatch.setattr(
|
||||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||||
)
|
||||
ask = _Ask(
|
||||
[
|
||||
{"label": "incidental", "reason": "i"},
|
||||
{"label": "risk_factor", "reason": "r"},
|
||||
]
|
||||
)
|
||||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert "classified in 2 round(s)" in res.output and ask.calls == 2
|
||||
(link,) = T.read_links(c, basis="fts")
|
||||
assert link.strength == "risk_factor"
|
||||
labels = {p.passage_id: p.label for p in T.read_passages(c)}
|
||||
assert sorted(labels.values()) == ["incidental", "risk_factor"]
|
||||
text = {p.passage_id: p.passage for p in T.read_passages(c)}[link.passage_id]
|
||||
assert text.startswith("Second")
|
||||
|
||||
|
||||
def test_fts_cli_classify_limit_stops_between_rounds(env, monkeypatch, tmp_path):
|
||||
c, _, _, _ = env
|
||||
monkeypatch.setattr(
|
||||
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
|
||||
)
|
||||
ask = _Ask([{"label": "incidental", "reason": "i"}] * 3)
|
||||
monkeypatch.setattr(mk, "_asker", lambda: ask)
|
||||
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
|
||||
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
|
||||
assert res.exit_code == 0, res.output
|
||||
assert ask.calls == 1 and "classified in 1 round(s)" in res.output
|
||||
# the link now rests on the second, still unlabelled passage
|
||||
(link,) = T.read_links(c, basis="fts")
|
||||
assert link.strength == "mention"
|
||||
|
||||
@@ -385,3 +385,68 @@ def test_window_snaps_to_word_boundaries_in_a_long_sentence():
|
||||
tail = ("q " * 600) + "RBRVS"
|
||||
(start, p), *_ = edgar.passages(tail, "rbrvs")
|
||||
assert p.endswith("RBRVS") and len(p) <= 400
|
||||
|
||||
|
||||
def test_fetch_doc_remembers_404_and_refresh_retries(hoard):
|
||||
h = _hit(filename="gone.htm")
|
||||
calls = []
|
||||
|
||||
def handler(request):
|
||||
calls.append(request.url.path)
|
||||
return httpx.Response(404)
|
||||
|
||||
cl = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None
|
||||
assert (hoard / "sec/docs/0000790526/0000790526-23-000003/gone.htm.404").is_file()
|
||||
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None and len(calls) == 1
|
||||
assert edgar.fetch_doc(cl, h, pacer=_Pacer(), refresh=True) is None
|
||||
assert len(calls) == 2
|
||||
# offline asks nothing and leaves no marker for a never-tried document
|
||||
other = _hit(filename="never.htm")
|
||||
assert edgar.fetch_doc(None, other, pacer=_Pacer(), offline=True) is None
|
||||
assert not (
|
||||
hoard / "sec/docs/0000790526/0000790526-23-000003/never.htm.404"
|
||||
).exists()
|
||||
|
||||
|
||||
def test_fetch_doc_early_filing_falls_back_to_full_submission(hoard):
|
||||
h = _hit(
|
||||
adsh="0000790526-01-000009", filename="0001.txt", file_date=date(2001, 3, 1)
|
||||
)
|
||||
calls = []
|
||||
|
||||
def handler(request):
|
||||
calls.append(request.url.path)
|
||||
if request.url.path.endswith("0000790526-01-000009.txt"):
|
||||
return httpx.Response(
|
||||
200, content=b"<TEXT>Medicare Physician Fee Schedule</TEXT>"
|
||||
)
|
||||
return httpx.Response(404)
|
||||
|
||||
cl = httpx.Client(transport=httpx.MockTransport(handler))
|
||||
path = edgar.fetch_doc(cl, h, pacer=_Pacer())
|
||||
assert path.name == "0000790526-01-000009.txt"
|
||||
assert "Medicare Physician Fee Schedule" in edgar.read_doc(path)
|
||||
assert calls[0].endswith("/0001.txt") and len(calls) == 2
|
||||
assert edgar.hoarded_doc(h.cik, h.adsh, h.filename) == path
|
||||
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) == path and len(calls) == 2
|
||||
# a later filing is never tried as a full submission
|
||||
later = _hit(
|
||||
adsh="0000790526-05-000001", filename="x.htm", file_date=date(2005, 1, 1)
|
||||
)
|
||||
calls.clear()
|
||||
assert edgar.fetch_doc(cl, later, pacer=_Pacer()) is None and len(calls) == 1
|
||||
assert edgar.hoarded_doc("1", "A", "b.htm") is None
|
||||
|
||||
|
||||
def test_ama_check_runs_against_a_plain_frozenset(monkeypatch):
|
||||
"""isdisjoint against a frozenset subclass takes a slow path (187 s for
|
||||
3,213 strings): the check must use a plain frozenset."""
|
||||
idx = edgar.AmaIndex.of([DESC])
|
||||
assert type(idx.plain) is frozenset and idx.plain == frozenset(idx)
|
||||
direct = edgar.AmaIndex({("a",) * 8}) # built without .of: converted once
|
||||
assert type(direct.plain) is frozenset and direct.plain is direct.plain
|
||||
assert edgar.shares_ama_run("x " + DESC, idx)
|
||||
assert edgar.shares_ama_run("x " + DESC, [DESC])
|
||||
# the public API still takes descriptors or an index
|
||||
assert not edgar.shares_ama_run("short text", idx)
|
||||
|
||||
@@ -149,9 +149,14 @@ def _universe_db():
|
||||
con = duckdb.connect(":memory:")
|
||||
T.ensure_tables(con)
|
||||
|
||||
def sec(key, ticker, priced=True):
|
||||
def sec(key, ticker, priced=True, exchange="NYSE", source="sec"):
|
||||
T.write_securities(
|
||||
con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)]
|
||||
con,
|
||||
key,
|
||||
[
|
||||
*T.read_securities(con, key),
|
||||
T.SecurityRow(key, ticker, exchange, None, None, source, priced),
|
||||
],
|
||||
)
|
||||
|
||||
def link(key, basis, target, strength):
|
||||
@@ -164,20 +169,36 @@ def _universe_db():
|
||||
con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")]
|
||||
)
|
||||
|
||||
sec("SEED", "SEEDCO")
|
||||
sec("OLD", "OLDCO", priced=False)
|
||||
sec("REV", "REVCO")
|
||||
sec("MEN", "MENCO")
|
||||
sec("MEN2", "MEN2CO")
|
||||
sec("SIC", "SICCO")
|
||||
sec("GONE", "GONECO")
|
||||
def entity(key, sic):
|
||||
T.write_entities(con, [T.EntityRow(key, key, "company", cik=key, sic=sic)])
|
||||
|
||||
sec("SEED", "SEEDC", source="seed")
|
||||
sec("OLD", "OLDC", priced=False)
|
||||
sec("REV", "REVC")
|
||||
sec("REV", "REVCW") # a warrant of REV
|
||||
sec("REV", "REVCP", exchange="Nasdaq") # a preferred line
|
||||
sec("MEN", "MENC")
|
||||
sec("MEN2", "MENB") # one bare mention, pharma SIC: never priced
|
||||
sec("SVC", "SVCC", exchange="NASDAQ") # one mention, healthcare services
|
||||
sec("RISK", "RISK") # one model-labelled risk_factor
|
||||
sec("OTC", "OTCC", exchange="OTC")
|
||||
sec("NOEX", "NOEX", exchange="")
|
||||
sec("SIC", "SICC")
|
||||
sec("GONE", "GONEC")
|
||||
link("SEED", "curated", "PFS", "curated")
|
||||
link("REV", "fts", "PFS", "revenue_exposure")
|
||||
link("MEN", "fts", "PFS", "mention")
|
||||
link("MEN", "fts", "APM", "mention")
|
||||
link("MEN2", "fts", "PFS", "mention")
|
||||
link("SVC", "fts", "PFS", "mention")
|
||||
link("RISK", "fts", "PFS", "risk_factor")
|
||||
link("OTC", "fts", "PFS", "revenue_exposure")
|
||||
link("NOEX", "fts", "PFS", "revenue_exposure")
|
||||
link("SIC", "sic", "8082", "sector")
|
||||
link("GONE", "fts", "PFS", "revenue_exposure")
|
||||
entity("MEN2", "2834")
|
||||
entity("SVC", "8082")
|
||||
entity("RISK", "3841")
|
||||
tenk("MEN", date(2020, 1, 1))
|
||||
tenk("MEN2", date(2025, 1, 1))
|
||||
return con
|
||||
@@ -189,12 +210,18 @@ SUPPORTED = {
|
||||
"SPY",
|
||||
"XLV",
|
||||
"IHF",
|
||||
"SEEDCO",
|
||||
"REVCO",
|
||||
"MENCO",
|
||||
"MEN2CO",
|
||||
"SICCO",
|
||||
"OLDCO",
|
||||
"SEEDC",
|
||||
"REVC",
|
||||
"REVCW",
|
||||
"REVCP",
|
||||
"MENC",
|
||||
"MENB",
|
||||
"SVCC",
|
||||
"RISK",
|
||||
"OTCC",
|
||||
"NOEX",
|
||||
"SICC",
|
||||
"OLDC",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -204,14 +231,66 @@ def test_universe_ranked_capped_excludes_sic_only():
|
||||
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
|
||||
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
|
||||
assert got[:3] == ["SPY", "XLV", "IHF"]
|
||||
assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"]
|
||||
assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got
|
||||
# ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC
|
||||
assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"]
|
||||
for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"):
|
||||
assert out not in got
|
||||
keys = dict(TI.universe(con, used=set(), **kw))
|
||||
assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY"
|
||||
assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY"
|
||||
assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5]
|
||||
capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)]
|
||||
assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6
|
||||
assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped
|
||||
assert "SEEDC" in capped and "REVC" in capped and "SVCC" not in capped
|
||||
|
||||
|
||||
def test_is_common_share():
|
||||
ok = TI.is_common_share
|
||||
assert ok("RDNT", "Nasdaq") and ok("OSH", "NYSE") and ok("A", "NASDAQ")
|
||||
assert not ok("RSMDF", "OTC") and not ok("ABCD", "") and not ok("ABCD", None)
|
||||
for bad in (
|
||||
"PIIIW",
|
||||
"BTSGU",
|
||||
"XYZAR",
|
||||
"CTRVP",
|
||||
"LGNDZ",
|
||||
"BRK-A",
|
||||
"BF.B",
|
||||
"ab",
|
||||
"TOOLONG",
|
||||
):
|
||||
assert not ok(bad, "Nasdaq")
|
||||
assert ok("ABCDE", "NYSE") # a 5-letter common ending in another letter
|
||||
|
||||
|
||||
def test_fts_gate():
|
||||
q = TI.fts_qualifies
|
||||
assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher
|
||||
assert q(2, 2, "2834") # two targets
|
||||
assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071")
|
||||
assert not q(2, 1, "2834") and not q(2, 1, "")
|
||||
|
||||
|
||||
def test_universe_primary_ticker_and_curated_tickers():
|
||||
con = _universe_db()
|
||||
T.write_securities(
|
||||
con,
|
||||
"SEED",
|
||||
[
|
||||
T.SecurityRow("SEED", "SEEDC", "NYSE", None, None, "seed"),
|
||||
T.SecurityRow("SEED", "SEEDW", "NYSE", None, None, "sec"),
|
||||
],
|
||||
)
|
||||
T.write_securities(
|
||||
con,
|
||||
"REV",
|
||||
[
|
||||
T.SecurityRow("REV", "REVCC", "NYSE", None, None, "sec"),
|
||||
T.SecurityRow("REV", "REVC", "NYSE", None, None, "sec"),
|
||||
],
|
||||
)
|
||||
got = dict(TI.universe(con, used=set(), seed_order=["SEED"]))
|
||||
assert "SEEDC" in got and "SEEDW" not in got
|
||||
assert "REVC" in got and "REVCC" not in got # the shortest common symbol
|
||||
|
||||
|
||||
def test_universe_default_seed_order_reads_the_seed():
|
||||
@@ -462,11 +541,11 @@ def test_plan_refuses_unpriced_and_unsupported_named_tickers():
|
||||
def test_universe_records_unsupported_priced_seed_tickers():
|
||||
con = _universe_db()
|
||||
skipped: dict[str, str] = {}
|
||||
supported = {k: v for k, v in SUPPORTED.items() if k != "REVCO"}
|
||||
supported = {k: v for k, v in SUPPORTED.items() if k != "REVC"}
|
||||
TI.universe(
|
||||
con, used=set(), supported=supported, seed_order=["SEED"], skipped=skipped
|
||||
)
|
||||
assert skipped == {"REVCO": TI.NOT_SUPPORTED, "GONECO": TI.NOT_SUPPORTED}
|
||||
assert skipped == {"REVC": TI.NOT_SUPPORTED, "GONEC": TI.NOT_SUPPORTED}
|
||||
|
||||
|
||||
def test_clip_windows_nulls_first_return_after_each_gap():
|
||||
@@ -497,4 +576,4 @@ def test_universe_dedupes_repeated_seed_entries():
|
||||
con, used=set(), supported=SUPPORTED, seed_order=["SEED", "SEED"]
|
||||
)
|
||||
]
|
||||
assert got.count("SEEDCO") == 1
|
||||
assert got.count("SEEDC") == 1
|
||||
|
||||
Reference in New Issue
Block a user