fix(market): fts strength gate and common-share universe, classify links in rounds, remembered 404s, pre-2002 full-text fallback, plain-frozenset AMA check

This commit is contained in:
kert
2026-11-13 03:09:53 -05:00
parent 9ca93c65a6
commit faf0a86ac6
6 changed files with 433 additions and 80 deletions

View File

@@ -642,13 +642,12 @@ def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRo
def _classify(
kept: list[PassageRow], limit: int
kept: list[PassageRow], limit: int, ask: Any
) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]:
"""Label *kept* with the local model, resumably (``pfs.runlog``). Returns
the ledger outputs by unit, the run counts and the model name."""
"""Label *kept* with the local model *ask*, resumably (``pfs.runlog``).
Returns the ledger outputs by unit, the run counts and the model name."""
from pfs.runlog import RunLog, input_hash, ledger_path, run_units
ask = _asker()
model = getattr(ask, "model", "local")
version = edgar.EXPOSURE_PROMPT_VERSION
log = RunLog(ledger_path("market-exposure", "all"))
@@ -668,6 +667,55 @@ def _classify(
return log.outputs(model, version), counts, model
def _classify_links(
merged: dict[tuple[str, str], PassageRow],
derive: Any,
hits: list[HitRow],
owners: dict[str, Any],
limit: int,
) -> None:
"""Label the passage behind every live link, in rounds: an ``incidental``
label retires that passage, the link falls back to the next candidate
passage, and the next round reads that one, until every link rests on a
labelled passage or the candidates run out. *limit* caps the model calls
(0 = no cap); a rerun resumes from the ledger."""
ask = _asker()
filed = {(h.adsh, h.filename): h.file_date for h in hits}
total: Counter[str] = Counter()
rounds = 0
while True:
want = {lk.passage_id for lk in derive()}
todo = sorted(
(
p
for p in merged.values()
if p.passage_id in want and not p.label and p.term in owners
),
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
)
if not todo:
break
remaining = limit - total["done"] - total["failed"] if limit else 0
if limit and remaining <= 0:
break
outputs, counts, model = _classify(todo, remaining, ask)
total.update(counts)
rounds += 1
fresh = 0
for k, p in list(merged.items()):
got = outputs.get(f"{p.passage_id}|{p.term}")
if got and not p.label:
fresh += 1
merged[k] = dataclasses.replace(
p,
label=got["label"],
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
)
if not fresh:
break
typer.echo(f"classified in {rounds} round(s): {dict(total)}")
def _verified(
links: list[LinkRow], passage_rows: dict[Any, PassageRow]
) -> tuple[list[LinkRow], int]:
@@ -681,10 +729,8 @@ def _verified(
if p is None:
continue
if (p.adsh, p.filename) not in texts:
path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename))
texts[(p.adsh, p.filename)] = (
edgar.read_doc(path) if path.is_file() else None
)
path = edgar.hoarded_doc(p.cik, p.adsh, p.filename)
texts[(p.adsh, p.filename)] = edgar.read_doc(path) if path else None
text = texts[(p.adsh, p.filename)]
if text is not None and len(p.passage) <= 400 and p.passage in text:
kept.append(lk)
@@ -797,6 +843,7 @@ def fts(
seed_ciks = set(seeds)
health = set(edgar.HEALTH_SICS)
known = {e.key for e in prior_entities}
stored_passages = {(p.adsh, p.filename, p.term) for p in prior_passages}
new_hits: dict[tuple[str, str, str, str, str], HitRow] = {}
new_passages: list[PassageRow] = []
@@ -837,11 +884,14 @@ def fts(
order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k))
for cik, adsh, filename in order:
hs = docs[(cik, adsh, filename)]
if not refresh and all(
(adsh, filename, h.term) in stored_passages for h in hs
):
continue # already read: its passages are stored
if not filename.lower().endswith(edgar.READABLE):
issues.append(f"{adsh}:{filename}: format not readable as text")
continue
path = http.hoard("sec", "docs", cik, adsh, filename)
have = http.newest(path).is_file()
have = edgar.hoarded_doc(cik, adsh, filename) is not None
if not have and ((limit and fetched >= limit) or out_of_time()):
pending += 1
continue
@@ -903,31 +953,21 @@ def fts(
)
merged[(p.passage_id, p.term)] = p
if classify:
filed = {(h.adsh, h.filename): h.file_date for h in hits.values()}
todo = sorted(
(p for p in merged.values() if p.term in owners),
key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id),
)
outputs, counts, model = _classify(todo, limit)
typer.echo(f"classified with {model}: {counts}")
for k, p in list(merged.items()):
got = outputs.get(f"{p.passage_id}|{p.term}")
if got:
merged[k] = dataclasses.replace(
p,
label=got["label"],
label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}",
)
entities = list(prior_entities)
all_hits = list(hits.values())
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
linkable = [
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
]
links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
mine = {(sc, t) for sc, t, _ in scope}
links = [lk for lk in links if (lk.scope, lk.target) in mine]
def derive() -> list[LinkRow]:
with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()}
linkable = [
h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage
]
found = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks)
return [lk for lk in found if (lk.scope, lk.target) in mine]
if classify:
_classify_links(merged, derive, all_hits, owners, limit)
links = derive()
links, unverified = _verified(links, merged)
if unverified:
issues.append(f"{unverified} link(s) dropped: quote not found in the hoard")
@@ -966,7 +1006,7 @@ def fts(
if (adsh, filename) not in item_of:
h = docs_of[(adsh, filename)]
item_of[(adsh, filename)] = edgar.cite_filing(
store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename))
store, h, edgar.hoarded_doc(h.cik, adsh, filename)
)
links = [
dataclasses.replace(

View File

@@ -525,7 +525,22 @@ class AmaIndex(frozenset):
for d in descriptors:
w = _WORD.findall(d.lower())
grams.update(tuple(w[i : i + n]) for i in range(len(w) - n + 1))
return cls(grams)
index = cls(grams)
index.plain = frozenset(grams)
return index
@property
def plain(self) -> frozenset:
"""The grams as a plain ``frozenset``: ``isdisjoint`` against a
frozenset subclass takes a slow path (3,213 strings: 187 s)."""
got = self.__dict__.get("_plain")
if got is None:
got = self.__dict__["_plain"] = frozenset(self)
return got
@plain.setter
def plain(self, value: frozenset) -> None:
self.__dict__["_plain"] = value
def shares_ama_run(
@@ -543,7 +558,7 @@ def shares_ama_run(
if isinstance(descriptors, AmaIndex)
else AmaIndex.of(descriptors, n)
)
return not grams.isdisjoint(index)
return not grams.isdisjoint(index.plain)
# ── full-text search ─────────────────────────────────────────────────
@@ -671,6 +686,30 @@ def fetch_fts(
READABLE = (".htm", ".html", ".txt")
#: filings filed before this year may exist only as the full submission text
FULL_TEXT_BEFORE = 2002
def doc_paths(cik: str, adsh: str, filename: str) -> tuple[Path, Path]:
"""Where a hit's document is hoarded: its own file, or (an early filing)
the full-submission text ``<adsh>.txt``."""
base = ("sec", "docs", cik, adsh)
return http.hoard(*base, filename), http.hoard(*base, f"{adsh}.txt")
def hoarded_doc(cik: str, adsh: str, filename: str) -> Path | None:
"""The newest hoarded copy of a hit's document, if any."""
for path in doc_paths(cik, adsh, filename):
have = http.newest(path)
if have.is_file():
return have
return None
def _marker(path: Path) -> Path:
return path.with_name(path.name + ".404")
def fetch_doc(
cl: httpx.Client | None,
hit: HitRow,
@@ -680,18 +719,45 @@ def fetch_doc(
offline: bool = False,
) -> Path | None:
"""The hoarded copy of the hit's document (``sec/docs/<cik>/<adsh>/<file>``),
downloaded when absent and not *offline*; ``None`` when unavailable."""
path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename)
body = _fetch(
cl,
doc_url(hit.cik, hit.adsh, hit.filename),
path,
pacer=pacer,
refresh=refresh,
offline=offline,
ok_404=True,
)
return http.newest(path) if body is not None else None
downloaded when absent and not *offline*. A filing from before 2002 that
has no such file is tried as the full submission ``<adsh>.txt``. A 404 is
remembered in a ``.404`` marker beside the path and not asked again
(unless *refresh*); ``None`` when the document is unavailable."""
path, full = doc_paths(hit.cik, hit.adsh, hit.filename)
marker = _marker(path)
have = hoarded_doc(hit.cik, hit.adsh, hit.filename)
if not refresh and (have is not None or marker.is_file()):
if have is not None:
http.manifest(have, doc_url(hit.cik, hit.adsh, hit.filename))
return have
attempts = [(doc_url(hit.cik, hit.adsh, hit.filename), path)]
if hit.file_date.year < FULL_TEXT_BEFORE:
attempts.append(
(
f"https://www.sec.gov/Archives/edgar/data/{int(hit.cik)}/{hit.adsh}.txt",
full,
)
)
for url, target in attempts:
body = _fetch(
cl,
url,
target,
pacer=pacer,
refresh=refresh,
offline=offline,
ok_404=True,
)
if body is not None:
return http.newest(target)
if cl is not None and not offline:
marker.parent.mkdir(parents=True, exist_ok=True)
marker.write_text(
json.dumps(
{"url": attempts[0][0], "status": 404, "checked": str(dt.date.today())}
)
)
return None
def read_doc(path: Path) -> str:

View File

@@ -58,6 +58,36 @@ PACE_SECONDS = 75.0
NOT_SUPPORTED = "not on Tiingo's supported list"
NOT_PRICED = "symbol record only (priced=false), never sent to Tiingo"
_STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2}
#: exchanges whose common shares may be priced
LISTED = ("NYSE", "NASDAQ")
#: a 5-letter symbol ending in one of these is a warrant, unit, right or
#: preferred line of the 4-letter common symbol
_NON_COMMON_END = "WURPZ"
#: SICs of healthcare-services companies (80xx, health insurers)
SERVICES_SICS = ("6324",)
def is_common_share(ticker: str, exchange: str | None) -> bool:
"""A common share on NYSE or Nasdaq: no OTC or empty exchange, no
class/series punctuation, no warrant, unit, right or preferred suffix."""
if (exchange or "").upper() not in LISTED:
return False
if not (ticker.isalpha() and ticker.isupper() and len(ticker) <= 5):
return False
return not (len(ticker) == 5 and ticker[-1] in _NON_COMMON_END)
def is_services_sic(sic: str) -> bool:
return sic.startswith("80") or sic in SERVICES_SICS
def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool:
"""The strength gate of the filing-text tier: a model-labelled strength of
``risk_factor`` or higher, at least two distinct targets, or a
healthcare-services SIC. A bare single-target mention never qualifies."""
return (
best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic)
)
def new_pacer() -> http.Pacer:
@@ -414,14 +444,27 @@ def universe(
"""``(ticker, entity_key)`` in rank order: the benchmarks, the seed
securities in seed order, then the securities of entities with a
filing-text (``fts``) link, ranked by strongest link, number of distinct
link targets and newest 10-K. SIC-only entities and unsupported or
unpriced tickers are excluded; at most *cap* symbols not already in
link targets and newest 10-K. An fts entity needs a model-labelled
strength of risk_factor or higher, two distinct targets or a
healthcare-services SIC (a bare single-target mention never prices), and
only its one primary NYSE or Nasdaq common share is priced (no OTC,
warrant, unit, right or preferred line). SIC-only entities and
unsupported or unpriced tickers are excluded; at most *cap* symbols not already in
*used* are returned. Each unsupported ticker is recorded in *skipped*
(``{ticker: reason}``) so the caller can turn it into a gap."""
by_entity: dict[str, list[str]] = defaultdict(list)
curated: dict[str, list[str]] = defaultdict(list)
commons: dict[str, list[str]] = defaultdict(list)
for s in read_securities(con):
if s.priced:
by_entity[s.entity_key].append(s.ticker)
if not s.priced:
continue
if s.source == "seed":
curated[s.entity_key].append(s.ticker)
elif is_common_share(s.ticker, s.exchange):
commons[s.entity_key].append(s.ticker)
for key in {*curated, *commons}:
# curated tickers stay; otherwise one primary common share
by_entity[key] = curated[key] or [min(commons[key], key=lambda t: (len(t), t))]
ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS]
for key in seed_order if seed_order is not None else _seed_ciks():
@@ -433,6 +476,12 @@ def universe(
rank = _STRENGTH.get(link.strength, len(_STRENGTH))
best[link.entity_key] = min(best.get(link.entity_key, rank), rank)
targets[link.entity_key].add(link.target)
sic_of = dict(con.execute("SELECT key, sic FROM market.entity").fetchall())
best = {
k: r
for k, r in best.items()
if fts_qualifies(r, len(targets[k]), sic_of.get(k) or "")
}
newest = {
cik: filed
for cik, filed in con.execute(

View File

@@ -739,7 +739,10 @@ def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")])
for target in ("PFS", "APM"): # two targets pass the fts strength gate
T.write_links(
c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")]
)
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
@@ -1211,11 +1214,14 @@ def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch):
assert published == [True] and store.count() == 1
# a rerun is idempotent and downloads nothing new
n_docs = _fts_net(monkeypatch, forbid_docs=True)
assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0
res2 = runner.invoke(app, [*FTS, "--write"])
assert res2.exit_code == 0
assert "passages kept 0," in res2.output # stored passages are not re-read
assert (
_count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1
)
assert n_docs == ["gone.htm"] # a 404 is retried, never stored
assert n_docs == [] # the 404 is remembered, not asked again
assert "gone.htm: document not found" in res2.output
def test_fts_cli_refusal_prints_gap(env, monkeypatch):
@@ -1306,8 +1312,8 @@ def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch):
"WHERE passage_id = ?",
[p.passage_id],
)
_fts_net(monkeypatch, forbid_docs=True)
res = runner.invoke(app, [*FTS, "--write"])
_fts_net(monkeypatch) # a refresh reads the documents again
res = runner.invoke(app, [*FTS, "--refresh", "--write"])
assert res.exit_code == 0 and "new entities 0" in res.output
got = {q.passage_id: q for q in T.read_passages(c)}
assert (
@@ -1393,14 +1399,14 @@ def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path):
assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1
assert "classified in 1 round(s): {'done': 1" in res.output and ask.calls == 1
labelled = [p for p in T.read_passages(c) if p.label]
assert len(labelled) == 1 and labelled[0].label == "revenue_exposure"
assert labelled[0].label_version == "model-reading exposure-v1 qwen-test"
# resume: the done unit is skipped; the next call fails (None) and stays unlabelled
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0 and ask.calls == 2
assert "'skipped': 1" in res.output and "'failed': 1" in res.output
assert "'failed': 1" in res.output
# a rerun without --classify keeps what is stored
res = runner.invoke(app, [*FTS, "--write"])
assert len([p for p in T.read_passages(c) if p.label]) == 1
@@ -1510,3 +1516,51 @@ def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch):
assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [
True
]
TWICE = (
b"<p>First, we are paid under the Medicare Physician Fee Schedule rates.</p>"
b"<p>Second, our risk comes from the Medicare Physician Fee Schedule cuts.</p>"
)
def test_fts_cli_classify_falls_back_to_the_next_passage(env, monkeypatch, tmp_path):
"""An incidental first passage retires it; the link moves to the next
passage, which the next round reads."""
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask(
[
{"label": "incidental", "reason": "i"},
{"label": "risk_factor", "reason": "r"},
]
)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
res = runner.invoke(app, [*FTS, "--classify", "--write"])
assert res.exit_code == 0, res.output
assert "classified in 2 round(s)" in res.output and ask.calls == 2
(link,) = T.read_links(c, basis="fts")
assert link.strength == "risk_factor"
labels = {p.passage_id: p.label for p in T.read_passages(c)}
assert sorted(labels.values()) == ["incidental", "risk_factor"]
text = {p.passage_id: p.passage for p in T.read_passages(c)}[link.passage_id]
assert text.startswith("Second")
def test_fts_cli_classify_limit_stops_between_rounds(env, monkeypatch, tmp_path):
c, _, _, _ = env
monkeypatch.setattr(
"pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl"
)
ask = _Ask([{"label": "incidental", "reason": "i"}] * 3)
monkeypatch.setattr(mk, "_asker", lambda: ask)
_fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE})
res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"])
assert res.exit_code == 0, res.output
assert ask.calls == 1 and "classified in 1 round(s)" in res.output
# the link now rests on the second, still unlabelled passage
(link,) = T.read_links(c, basis="fts")
assert link.strength == "mention"

View File

@@ -385,3 +385,68 @@ def test_window_snaps_to_word_boundaries_in_a_long_sentence():
tail = ("q " * 600) + "RBRVS"
(start, p), *_ = edgar.passages(tail, "rbrvs")
assert p.endswith("RBRVS") and len(p) <= 400
def test_fetch_doc_remembers_404_and_refresh_retries(hoard):
h = _hit(filename="gone.htm")
calls = []
def handler(request):
calls.append(request.url.path)
return httpx.Response(404)
cl = httpx.Client(transport=httpx.MockTransport(handler))
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None
assert (hoard / "sec/docs/0000790526/0000790526-23-000003/gone.htm.404").is_file()
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None and len(calls) == 1
assert edgar.fetch_doc(cl, h, pacer=_Pacer(), refresh=True) is None
assert len(calls) == 2
# offline asks nothing and leaves no marker for a never-tried document
other = _hit(filename="never.htm")
assert edgar.fetch_doc(None, other, pacer=_Pacer(), offline=True) is None
assert not (
hoard / "sec/docs/0000790526/0000790526-23-000003/never.htm.404"
).exists()
def test_fetch_doc_early_filing_falls_back_to_full_submission(hoard):
h = _hit(
adsh="0000790526-01-000009", filename="0001.txt", file_date=date(2001, 3, 1)
)
calls = []
def handler(request):
calls.append(request.url.path)
if request.url.path.endswith("0000790526-01-000009.txt"):
return httpx.Response(
200, content=b"<TEXT>Medicare Physician Fee Schedule</TEXT>"
)
return httpx.Response(404)
cl = httpx.Client(transport=httpx.MockTransport(handler))
path = edgar.fetch_doc(cl, h, pacer=_Pacer())
assert path.name == "0000790526-01-000009.txt"
assert "Medicare Physician Fee Schedule" in edgar.read_doc(path)
assert calls[0].endswith("/0001.txt") and len(calls) == 2
assert edgar.hoarded_doc(h.cik, h.adsh, h.filename) == path
assert edgar.fetch_doc(cl, h, pacer=_Pacer()) == path and len(calls) == 2
# a later filing is never tried as a full submission
later = _hit(
adsh="0000790526-05-000001", filename="x.htm", file_date=date(2005, 1, 1)
)
calls.clear()
assert edgar.fetch_doc(cl, later, pacer=_Pacer()) is None and len(calls) == 1
assert edgar.hoarded_doc("1", "A", "b.htm") is None
def test_ama_check_runs_against_a_plain_frozenset(monkeypatch):
"""isdisjoint against a frozenset subclass takes a slow path (187 s for
3,213 strings): the check must use a plain frozenset."""
idx = edgar.AmaIndex.of([DESC])
assert type(idx.plain) is frozenset and idx.plain == frozenset(idx)
direct = edgar.AmaIndex({("a",) * 8}) # built without .of: converted once
assert type(direct.plain) is frozenset and direct.plain is direct.plain
assert edgar.shares_ama_run("x " + DESC, idx)
assert edgar.shares_ama_run("x " + DESC, [DESC])
# the public API still takes descriptors or an index
assert not edgar.shares_ama_run("short text", idx)

View File

@@ -149,9 +149,14 @@ def _universe_db():
con = duckdb.connect(":memory:")
T.ensure_tables(con)
def sec(key, ticker, priced=True):
def sec(key, ticker, priced=True, exchange="NYSE", source="sec"):
T.write_securities(
con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)]
con,
key,
[
*T.read_securities(con, key),
T.SecurityRow(key, ticker, exchange, None, None, source, priced),
],
)
def link(key, basis, target, strength):
@@ -164,20 +169,36 @@ def _universe_db():
con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")]
)
sec("SEED", "SEEDCO")
sec("OLD", "OLDCO", priced=False)
sec("REV", "REVCO")
sec("MEN", "MENCO")
sec("MEN2", "MEN2CO")
sec("SIC", "SICCO")
sec("GONE", "GONECO")
def entity(key, sic):
T.write_entities(con, [T.EntityRow(key, key, "company", cik=key, sic=sic)])
sec("SEED", "SEEDC", source="seed")
sec("OLD", "OLDC", priced=False)
sec("REV", "REVC")
sec("REV", "REVCW") # a warrant of REV
sec("REV", "REVCP", exchange="Nasdaq") # a preferred line
sec("MEN", "MENC")
sec("MEN2", "MENB") # one bare mention, pharma SIC: never priced
sec("SVC", "SVCC", exchange="NASDAQ") # one mention, healthcare services
sec("RISK", "RISK") # one model-labelled risk_factor
sec("OTC", "OTCC", exchange="OTC")
sec("NOEX", "NOEX", exchange="")
sec("SIC", "SICC")
sec("GONE", "GONEC")
link("SEED", "curated", "PFS", "curated")
link("REV", "fts", "PFS", "revenue_exposure")
link("MEN", "fts", "PFS", "mention")
link("MEN", "fts", "APM", "mention")
link("MEN2", "fts", "PFS", "mention")
link("SVC", "fts", "PFS", "mention")
link("RISK", "fts", "PFS", "risk_factor")
link("OTC", "fts", "PFS", "revenue_exposure")
link("NOEX", "fts", "PFS", "revenue_exposure")
link("SIC", "sic", "8082", "sector")
link("GONE", "fts", "PFS", "revenue_exposure")
entity("MEN2", "2834")
entity("SVC", "8082")
entity("RISK", "3841")
tenk("MEN", date(2020, 1, 1))
tenk("MEN2", date(2025, 1, 1))
return con
@@ -189,12 +210,18 @@ SUPPORTED = {
"SPY",
"XLV",
"IHF",
"SEEDCO",
"REVCO",
"MENCO",
"MEN2CO",
"SICCO",
"OLDCO",
"SEEDC",
"REVC",
"REVCW",
"REVCP",
"MENC",
"MENB",
"SVCC",
"RISK",
"OTCC",
"NOEX",
"SICC",
"OLDC",
)
}
@@ -204,14 +231,66 @@ def test_universe_ranked_capped_excludes_sic_only():
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
assert got[:3] == ["SPY", "XLV", "IHF"]
assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"]
assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got
# ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC
assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"]
for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"):
assert out not in got
keys = dict(TI.universe(con, used=set(), **kw))
assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY"
assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY"
assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5]
capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)]
assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6
assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped
assert "SEEDC" in capped and "REVC" in capped and "SVCC" not in capped
def test_is_common_share():
ok = TI.is_common_share
assert ok("RDNT", "Nasdaq") and ok("OSH", "NYSE") and ok("A", "NASDAQ")
assert not ok("RSMDF", "OTC") and not ok("ABCD", "") and not ok("ABCD", None)
for bad in (
"PIIIW",
"BTSGU",
"XYZAR",
"CTRVP",
"LGNDZ",
"BRK-A",
"BF.B",
"ab",
"TOOLONG",
):
assert not ok(bad, "Nasdaq")
assert ok("ABCDE", "NYSE") # a 5-letter common ending in another letter
def test_fts_gate():
q = TI.fts_qualifies
assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher
assert q(2, 2, "2834") # two targets
assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071")
assert not q(2, 1, "2834") and not q(2, 1, "")
def test_universe_primary_ticker_and_curated_tickers():
con = _universe_db()
T.write_securities(
con,
"SEED",
[
T.SecurityRow("SEED", "SEEDC", "NYSE", None, None, "seed"),
T.SecurityRow("SEED", "SEEDW", "NYSE", None, None, "sec"),
],
)
T.write_securities(
con,
"REV",
[
T.SecurityRow("REV", "REVCC", "NYSE", None, None, "sec"),
T.SecurityRow("REV", "REVC", "NYSE", None, None, "sec"),
],
)
got = dict(TI.universe(con, used=set(), seed_order=["SEED"]))
assert "SEEDC" in got and "SEEDW" not in got
assert "REVC" in got and "REVCC" not in got # the shortest common symbol
def test_universe_default_seed_order_reads_the_seed():
@@ -462,11 +541,11 @@ def test_plan_refuses_unpriced_and_unsupported_named_tickers():
def test_universe_records_unsupported_priced_seed_tickers():
con = _universe_db()
skipped: dict[str, str] = {}
supported = {k: v for k, v in SUPPORTED.items() if k != "REVCO"}
supported = {k: v for k, v in SUPPORTED.items() if k != "REVC"}
TI.universe(
con, used=set(), supported=supported, seed_order=["SEED"], skipped=skipped
)
assert skipped == {"REVCO": TI.NOT_SUPPORTED, "GONECO": TI.NOT_SUPPORTED}
assert skipped == {"REVC": TI.NOT_SUPPORTED, "GONEC": TI.NOT_SUPPORTED}
def test_clip_windows_nulls_first_return_after_each_gap():
@@ -497,4 +576,4 @@ def test_universe_dedupes_repeated_seed_entries():
con, used=set(), supported=SUPPORTED, seed_order=["SEED", "SEED"]
)
]
assert got.count("SEEDCO") == 1
assert got.count("SEEDC") == 1