From faf0a86ac6a3c60f683ff493d0faabe98201d61c Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 03:09:53 -0500 Subject: [PATCH] fix(market): fts strength gate and common-share universe, classify links in rounds, remembered 404s, pre-2002 full-text fallback, plain-frozenset AMA check --- src/cli/market.py | 106 ++++++++++++++++++++---------- src/market/edgar.py | 94 ++++++++++++++++++++++---- src/market/tiingo.py | 57 ++++++++++++++-- tests/cli/test_market_cli.py | 68 +++++++++++++++++-- tests/market/test_fts.py | 65 ++++++++++++++++++ tests/market/test_tiingo.py | 123 ++++++++++++++++++++++++++++------- 6 files changed, 433 insertions(+), 80 deletions(-) diff --git a/src/cli/market.py b/src/cli/market.py index e672bd2..49b3642 100644 --- a/src/cli/market.py +++ b/src/cli/market.py @@ -642,13 +642,12 @@ def _prior(con_read: Any) -> tuple[list[HitRow], list[PassageRow], list[EntityRo def _classify( - kept: list[PassageRow], limit: int + kept: list[PassageRow], limit: int, ask: Any ) -> tuple[dict[str, dict[str, Any]], dict[str, int], str]: - """Label *kept* with the local model, resumably (``pfs.runlog``). Returns - the ledger outputs by unit, the run counts and the model name.""" + """Label *kept* with the local model *ask*, resumably (``pfs.runlog``). + Returns the ledger outputs by unit, the run counts and the model name.""" from pfs.runlog import RunLog, input_hash, ledger_path, run_units - ask = _asker() model = getattr(ask, "model", "local") version = edgar.EXPOSURE_PROMPT_VERSION log = RunLog(ledger_path("market-exposure", "all")) @@ -668,6 +667,55 @@ def _classify( return log.outputs(model, version), counts, model +def _classify_links( + merged: dict[tuple[str, str], PassageRow], + derive: Any, + hits: list[HitRow], + owners: dict[str, Any], + limit: int, +) -> None: + """Label the passage behind every live link, in rounds: an ``incidental`` + label retires that passage, the link falls back to the next candidate + passage, and the next round reads that one, until every link rests on a + labelled passage or the candidates run out. *limit* caps the model calls + (0 = no cap); a rerun resumes from the ledger.""" + ask = _asker() + filed = {(h.adsh, h.filename): h.file_date for h in hits} + total: Counter[str] = Counter() + rounds = 0 + while True: + want = {lk.passage_id for lk in derive()} + todo = sorted( + ( + p + for p in merged.values() + if p.passage_id in want and not p.label and p.term in owners + ), + key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id), + ) + if not todo: + break + remaining = limit - total["done"] - total["failed"] if limit else 0 + if limit and remaining <= 0: + break + outputs, counts, model = _classify(todo, remaining, ask) + total.update(counts) + rounds += 1 + fresh = 0 + for k, p in list(merged.items()): + got = outputs.get(f"{p.passage_id}|{p.term}") + if got and not p.label: + fresh += 1 + merged[k] = dataclasses.replace( + p, + label=got["label"], + label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}", + ) + if not fresh: + break + typer.echo(f"classified in {rounds} round(s): {dict(total)}") + + def _verified( links: list[LinkRow], passage_rows: dict[Any, PassageRow] ) -> tuple[list[LinkRow], int]: @@ -681,10 +729,8 @@ def _verified( if p is None: continue if (p.adsh, p.filename) not in texts: - path = http.newest(http.hoard("sec", "docs", p.cik, p.adsh, p.filename)) - texts[(p.adsh, p.filename)] = ( - edgar.read_doc(path) if path.is_file() else None - ) + path = edgar.hoarded_doc(p.cik, p.adsh, p.filename) + texts[(p.adsh, p.filename)] = edgar.read_doc(path) if path else None text = texts[(p.adsh, p.filename)] if text is not None and len(p.passage) <= 400 and p.passage in text: kept.append(lk) @@ -797,6 +843,7 @@ def fts( seed_ciks = set(seeds) health = set(edgar.HEALTH_SICS) known = {e.key for e in prior_entities} + stored_passages = {(p.adsh, p.filename, p.term) for p in prior_passages} new_hits: dict[tuple[str, str, str, str, str], HitRow] = {} new_passages: list[PassageRow] = [] @@ -837,11 +884,14 @@ def fts( order = sorted(docs, key=lambda k: (-docs[k][0].file_date.toordinal(), k)) for cik, adsh, filename in order: hs = docs[(cik, adsh, filename)] + if not refresh and all( + (adsh, filename, h.term) in stored_passages for h in hs + ): + continue # already read: its passages are stored if not filename.lower().endswith(edgar.READABLE): issues.append(f"{adsh}:{filename}: format not readable as text") continue - path = http.hoard("sec", "docs", cik, adsh, filename) - have = http.newest(path).is_file() + have = edgar.hoarded_doc(cik, adsh, filename) is not None if not have and ((limit and fetched >= limit) or out_of_time()): pending += 1 continue @@ -903,31 +953,21 @@ def fts( ) merged[(p.passage_id, p.term)] = p - if classify: - filed = {(h.adsh, h.filename): h.file_date for h in hits.values()} - todo = sorted( - (p for p in merged.values() if p.term in owners), - key=lambda p: (-filed[(p.adsh, p.filename)].toordinal(), p.passage_id), - ) - outputs, counts, model = _classify(todo, limit) - typer.echo(f"classified with {model}: {counts}") - for k, p in list(merged.items()): - got = outputs.get(f"{p.passage_id}|{p.term}") - if got: - merged[k] = dataclasses.replace( - p, - label=got["label"], - label_version=f"model-reading {edgar.EXPOSURE_PROMPT_VERSION} {model}", - ) entities = list(prior_entities) all_hits = list(hits.values()) - with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()} - linkable = [ - h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage - ] - links = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks) mine = {(sc, t) for sc, t, _ in scope} - links = [lk for lk in links if (lk.scope, lk.target) in mine] + + def derive() -> list[LinkRow]: + with_passage = {(p.cik, p.adsh, p.filename, p.term) for p in merged.values()} + linkable = [ + h for h in all_hits if (h.cik, h.adsh, h.filename, h.term) in with_passage + ] + found = edgar.fts_links(linkable, list(merged.values()), entities, seed_ciks) + return [lk for lk in found if (lk.scope, lk.target) in mine] + + if classify: + _classify_links(merged, derive, all_hits, owners, limit) + links = derive() links, unverified = _verified(links, merged) if unverified: issues.append(f"{unverified} link(s) dropped: quote not found in the hoard") @@ -966,7 +1006,7 @@ def fts( if (adsh, filename) not in item_of: h = docs_of[(adsh, filename)] item_of[(adsh, filename)] = edgar.cite_filing( - store, h, http.newest(http.hoard("sec", "docs", h.cik, adsh, filename)) + store, h, edgar.hoarded_doc(h.cik, adsh, filename) ) links = [ dataclasses.replace( diff --git a/src/market/edgar.py b/src/market/edgar.py index ce22f69..0627735 100644 --- a/src/market/edgar.py +++ b/src/market/edgar.py @@ -525,7 +525,22 @@ class AmaIndex(frozenset): for d in descriptors: w = _WORD.findall(d.lower()) grams.update(tuple(w[i : i + n]) for i in range(len(w) - n + 1)) - return cls(grams) + index = cls(grams) + index.plain = frozenset(grams) + return index + + @property + def plain(self) -> frozenset: + """The grams as a plain ``frozenset``: ``isdisjoint`` against a + frozenset subclass takes a slow path (3,213 strings: 187 s).""" + got = self.__dict__.get("_plain") + if got is None: + got = self.__dict__["_plain"] = frozenset(self) + return got + + @plain.setter + def plain(self, value: frozenset) -> None: + self.__dict__["_plain"] = value def shares_ama_run( @@ -543,7 +558,7 @@ def shares_ama_run( if isinstance(descriptors, AmaIndex) else AmaIndex.of(descriptors, n) ) - return not grams.isdisjoint(index) + return not grams.isdisjoint(index.plain) # ── full-text search ───────────────────────────────────────────────── @@ -671,6 +686,30 @@ def fetch_fts( READABLE = (".htm", ".html", ".txt") +#: filings filed before this year may exist only as the full submission text +FULL_TEXT_BEFORE = 2002 + + +def doc_paths(cik: str, adsh: str, filename: str) -> tuple[Path, Path]: + """Where a hit's document is hoarded: its own file, or (an early filing) + the full-submission text ``.txt``.""" + base = ("sec", "docs", cik, adsh) + return http.hoard(*base, filename), http.hoard(*base, f"{adsh}.txt") + + +def hoarded_doc(cik: str, adsh: str, filename: str) -> Path | None: + """The newest hoarded copy of a hit's document, if any.""" + for path in doc_paths(cik, adsh, filename): + have = http.newest(path) + if have.is_file(): + return have + return None + + +def _marker(path: Path) -> Path: + return path.with_name(path.name + ".404") + + def fetch_doc( cl: httpx.Client | None, hit: HitRow, @@ -680,18 +719,45 @@ def fetch_doc( offline: bool = False, ) -> Path | None: """The hoarded copy of the hit's document (``sec/docs///``), - downloaded when absent and not *offline*; ``None`` when unavailable.""" - path = http.hoard("sec", "docs", hit.cik, hit.adsh, hit.filename) - body = _fetch( - cl, - doc_url(hit.cik, hit.adsh, hit.filename), - path, - pacer=pacer, - refresh=refresh, - offline=offline, - ok_404=True, - ) - return http.newest(path) if body is not None else None + downloaded when absent and not *offline*. A filing from before 2002 that + has no such file is tried as the full submission ``.txt``. A 404 is + remembered in a ``.404`` marker beside the path and not asked again + (unless *refresh*); ``None`` when the document is unavailable.""" + path, full = doc_paths(hit.cik, hit.adsh, hit.filename) + marker = _marker(path) + have = hoarded_doc(hit.cik, hit.adsh, hit.filename) + if not refresh and (have is not None or marker.is_file()): + if have is not None: + http.manifest(have, doc_url(hit.cik, hit.adsh, hit.filename)) + return have + attempts = [(doc_url(hit.cik, hit.adsh, hit.filename), path)] + if hit.file_date.year < FULL_TEXT_BEFORE: + attempts.append( + ( + f"https://www.sec.gov/Archives/edgar/data/{int(hit.cik)}/{hit.adsh}.txt", + full, + ) + ) + for url, target in attempts: + body = _fetch( + cl, + url, + target, + pacer=pacer, + refresh=refresh, + offline=offline, + ok_404=True, + ) + if body is not None: + return http.newest(target) + if cl is not None and not offline: + marker.parent.mkdir(parents=True, exist_ok=True) + marker.write_text( + json.dumps( + {"url": attempts[0][0], "status": 404, "checked": str(dt.date.today())} + ) + ) + return None def read_doc(path: Path) -> str: diff --git a/src/market/tiingo.py b/src/market/tiingo.py index 8d94298..b0c4cca 100644 --- a/src/market/tiingo.py +++ b/src/market/tiingo.py @@ -58,6 +58,36 @@ PACE_SECONDS = 75.0 NOT_SUPPORTED = "not on Tiingo's supported list" NOT_PRICED = "symbol record only (priced=false), never sent to Tiingo" _STRENGTH = {"revenue_exposure": 0, "risk_factor": 1, "mention": 2} +#: exchanges whose common shares may be priced +LISTED = ("NYSE", "NASDAQ") +#: a 5-letter symbol ending in one of these is a warrant, unit, right or +#: preferred line of the 4-letter common symbol +_NON_COMMON_END = "WURPZ" +#: SICs of healthcare-services companies (80xx, health insurers) +SERVICES_SICS = ("6324",) + + +def is_common_share(ticker: str, exchange: str | None) -> bool: + """A common share on NYSE or Nasdaq: no OTC or empty exchange, no + class/series punctuation, no warrant, unit, right or preferred suffix.""" + if (exchange or "").upper() not in LISTED: + return False + if not (ticker.isalpha() and ticker.isupper() and len(ticker) <= 5): + return False + return not (len(ticker) == 5 and ticker[-1] in _NON_COMMON_END) + + +def is_services_sic(sic: str) -> bool: + return sic.startswith("80") or sic in SERVICES_SICS + + +def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool: + """The strength gate of the filing-text tier: a model-labelled strength of + ``risk_factor`` or higher, at least two distinct targets, or a + healthcare-services SIC. A bare single-target mention never qualifies.""" + return ( + best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic) + ) def new_pacer() -> http.Pacer: @@ -414,14 +444,27 @@ def universe( """``(ticker, entity_key)`` in rank order: the benchmarks, the seed securities in seed order, then the securities of entities with a filing-text (``fts``) link, ranked by strongest link, number of distinct - link targets and newest 10-K. SIC-only entities and unsupported or - unpriced tickers are excluded; at most *cap* symbols not already in + link targets and newest 10-K. An fts entity needs a model-labelled + strength of risk_factor or higher, two distinct targets or a + healthcare-services SIC (a bare single-target mention never prices), and + only its one primary NYSE or Nasdaq common share is priced (no OTC, + warrant, unit, right or preferred line). SIC-only entities and + unsupported or unpriced tickers are excluded; at most *cap* symbols not already in *used* are returned. Each unsupported ticker is recorded in *skipped* (``{ticker: reason}``) so the caller can turn it into a gap.""" by_entity: dict[str, list[str]] = defaultdict(list) + curated: dict[str, list[str]] = defaultdict(list) + commons: dict[str, list[str]] = defaultdict(list) for s in read_securities(con): - if s.priced: - by_entity[s.entity_key].append(s.ticker) + if not s.priced: + continue + if s.source == "seed": + curated[s.entity_key].append(s.ticker) + elif is_common_share(s.ticker, s.exchange): + commons[s.entity_key].append(s.ticker) + for key in {*curated, *commons}: + # curated tickers stay; otherwise one primary common share + by_entity[key] = curated[key] or [min(commons[key], key=lambda t: (len(t), t))] ranked: list[tuple[str, str]] = [(t, f"index:{t}") for t in BENCHMARKS] for key in seed_order if seed_order is not None else _seed_ciks(): @@ -433,6 +476,12 @@ def universe( rank = _STRENGTH.get(link.strength, len(_STRENGTH)) best[link.entity_key] = min(best.get(link.entity_key, rank), rank) targets[link.entity_key].add(link.target) + sic_of = dict(con.execute("SELECT key, sic FROM market.entity").fetchall()) + best = { + k: r + for k, r in best.items() + if fts_qualifies(r, len(targets[k]), sic_of.get(k) or "") + } newest = { cik: filed for cik, filed in con.execute( diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index f89588f..14944b7 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -739,7 +739,10 @@ def test_prices_dry_run_reports_budget(px, monkeypatch): c, _, published, _, seen, _, state = px T.ensure_tables(c) T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")]) - T.write_links(c, [T.LinkRow("K1", "program", "PFS", "fts", "mention", "adsh", "a")]) + for target in ("PFS", "APM"): # two targets pass the fts strength gate + T.write_links( + c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")] + ) state.mkdir(parents=True) (state / "tiingo-2026-11.json").write_text( json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}}) @@ -1211,11 +1214,14 @@ def test_fts_cli_write_adds_new_cik_entity_and_sic_link(env, monkeypatch): assert published == [True] and store.count() == 1 # a rerun is idempotent and downloads nothing new n_docs = _fts_net(monkeypatch, forbid_docs=True) - assert runner.invoke(app, [*FTS, "--write"]).exit_code == 0 + res2 = runner.invoke(app, [*FTS, "--write"]) + assert res2.exit_code == 0 + assert "passages kept 0," in res2.output # stored passages are not re-read assert ( _count(c, "market.entity_link") == 3 and _count(c, "market.filing_passage") == 1 ) - assert n_docs == ["gone.htm"] # a 404 is retried, never stored + assert n_docs == [] # the 404 is remembered, not asked again + assert "gone.htm: document not found" in res2.output def test_fts_cli_refusal_prints_gap(env, monkeypatch): @@ -1306,8 +1312,8 @@ def test_fts_cli_existing_entity_and_prior_labels_carry_over(env, monkeypatch): "WHERE passage_id = ?", [p.passage_id], ) - _fts_net(monkeypatch, forbid_docs=True) - res = runner.invoke(app, [*FTS, "--write"]) + _fts_net(monkeypatch) # a refresh reads the documents again + res = runner.invoke(app, [*FTS, "--refresh", "--write"]) assert res.exit_code == 0 and "new entities 0" in res.output got = {q.passage_id: q for q in T.read_passages(c)} assert ( @@ -1393,14 +1399,14 @@ def test_fts_cli_classify_resumes_from_ledger(env, monkeypatch, tmp_path): assert runner.invoke(app, FTS).exit_code == 0 # hoard the documents first res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"]) assert res.exit_code == 0, res.output - assert "classified with qwen-test: {'done': 1" in res.output and ask.calls == 1 + assert "classified in 1 round(s): {'done': 1" in res.output and ask.calls == 1 labelled = [p for p in T.read_passages(c) if p.label] assert len(labelled) == 1 and labelled[0].label == "revenue_exposure" assert labelled[0].label_version == "model-reading exposure-v1 qwen-test" # resume: the done unit is skipped; the next call fails (None) and stays unlabelled res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"]) assert res.exit_code == 0 and ask.calls == 2 - assert "'skipped': 1" in res.output and "'failed': 1" in res.output + assert "'failed': 1" in res.output # a rerun without --classify keeps what is stored res = runner.invoke(app, [*FTS, "--write"]) assert len([p for p in T.read_passages(c) if p.label]) == 1 @@ -1510,3 +1516,51 @@ def test_fts_cli_document_403_stops_as_a_gap(env, monkeypatch): assert [g.source for g in T.read_gaps(c)] == ["sec-edgar:fts"] and published == [ True ] + + +TWICE = ( + b"

First, we are paid under the Medicare Physician Fee Schedule rates.

" + b"

Second, our risk comes from the Medicare Physician Fee Schedule cuts.

" +) + + +def test_fts_cli_classify_falls_back_to_the_next_passage(env, monkeypatch, tmp_path): + """An incidental first passage retires it; the link moves to the next + passage, which the next round reads.""" + c, _, _, _ = env + monkeypatch.setattr( + "pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl" + ) + ask = _Ask( + [ + {"label": "incidental", "reason": "i"}, + {"label": "risk_factor", "reason": "r"}, + ] + ) + monkeypatch.setattr(mk, "_asker", lambda: ask) + _fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE}) + res = runner.invoke(app, [*FTS, "--classify", "--write"]) + assert res.exit_code == 0, res.output + assert "classified in 2 round(s)" in res.output and ask.calls == 2 + (link,) = T.read_links(c, basis="fts") + assert link.strength == "risk_factor" + labels = {p.passage_id: p.label for p in T.read_passages(c)} + assert sorted(labels.values()) == ["incidental", "risk_factor"] + text = {p.passage_id: p.passage for p in T.read_passages(c)}[link.passage_id] + assert text.startswith("Second") + + +def test_fts_cli_classify_limit_stops_between_rounds(env, monkeypatch, tmp_path): + c, _, _, _ = env + monkeypatch.setattr( + "pfs.runlog.ledger_path", lambda task, fam: tmp_path / "l.jsonl" + ) + ask = _Ask([{"label": "incidental", "reason": "i"}] * 3) + monkeypatch.setattr(mk, "_asker", lambda: ask) + _fts_net(monkeypatch, hits=HITS[:1], docs={"a10k.htm": TWICE}) + res = runner.invoke(app, [*FTS, "--classify", "--limit", "1", "--write"]) + assert res.exit_code == 0, res.output + assert ask.calls == 1 and "classified in 1 round(s)" in res.output + # the link now rests on the second, still unlabelled passage + (link,) = T.read_links(c, basis="fts") + assert link.strength == "mention" diff --git a/tests/market/test_fts.py b/tests/market/test_fts.py index ef27d3b..36727cb 100644 --- a/tests/market/test_fts.py +++ b/tests/market/test_fts.py @@ -385,3 +385,68 @@ def test_window_snaps_to_word_boundaries_in_a_long_sentence(): tail = ("q " * 600) + "RBRVS" (start, p), *_ = edgar.passages(tail, "rbrvs") assert p.endswith("RBRVS") and len(p) <= 400 + + +def test_fetch_doc_remembers_404_and_refresh_retries(hoard): + h = _hit(filename="gone.htm") + calls = [] + + def handler(request): + calls.append(request.url.path) + return httpx.Response(404) + + cl = httpx.Client(transport=httpx.MockTransport(handler)) + assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None + assert (hoard / "sec/docs/0000790526/0000790526-23-000003/gone.htm.404").is_file() + assert edgar.fetch_doc(cl, h, pacer=_Pacer()) is None and len(calls) == 1 + assert edgar.fetch_doc(cl, h, pacer=_Pacer(), refresh=True) is None + assert len(calls) == 2 + # offline asks nothing and leaves no marker for a never-tried document + other = _hit(filename="never.htm") + assert edgar.fetch_doc(None, other, pacer=_Pacer(), offline=True) is None + assert not ( + hoard / "sec/docs/0000790526/0000790526-23-000003/never.htm.404" + ).exists() + + +def test_fetch_doc_early_filing_falls_back_to_full_submission(hoard): + h = _hit( + adsh="0000790526-01-000009", filename="0001.txt", file_date=date(2001, 3, 1) + ) + calls = [] + + def handler(request): + calls.append(request.url.path) + if request.url.path.endswith("0000790526-01-000009.txt"): + return httpx.Response( + 200, content=b"Medicare Physician Fee Schedule" + ) + return httpx.Response(404) + + cl = httpx.Client(transport=httpx.MockTransport(handler)) + path = edgar.fetch_doc(cl, h, pacer=_Pacer()) + assert path.name == "0000790526-01-000009.txt" + assert "Medicare Physician Fee Schedule" in edgar.read_doc(path) + assert calls[0].endswith("/0001.txt") and len(calls) == 2 + assert edgar.hoarded_doc(h.cik, h.adsh, h.filename) == path + assert edgar.fetch_doc(cl, h, pacer=_Pacer()) == path and len(calls) == 2 + # a later filing is never tried as a full submission + later = _hit( + adsh="0000790526-05-000001", filename="x.htm", file_date=date(2005, 1, 1) + ) + calls.clear() + assert edgar.fetch_doc(cl, later, pacer=_Pacer()) is None and len(calls) == 1 + assert edgar.hoarded_doc("1", "A", "b.htm") is None + + +def test_ama_check_runs_against_a_plain_frozenset(monkeypatch): + """isdisjoint against a frozenset subclass takes a slow path (187 s for + 3,213 strings): the check must use a plain frozenset.""" + idx = edgar.AmaIndex.of([DESC]) + assert type(idx.plain) is frozenset and idx.plain == frozenset(idx) + direct = edgar.AmaIndex({("a",) * 8}) # built without .of: converted once + assert type(direct.plain) is frozenset and direct.plain is direct.plain + assert edgar.shares_ama_run("x " + DESC, idx) + assert edgar.shares_ama_run("x " + DESC, [DESC]) + # the public API still takes descriptors or an index + assert not edgar.shares_ama_run("short text", idx) diff --git a/tests/market/test_tiingo.py b/tests/market/test_tiingo.py index a592ec2..997c21d 100644 --- a/tests/market/test_tiingo.py +++ b/tests/market/test_tiingo.py @@ -149,9 +149,14 @@ def _universe_db(): con = duckdb.connect(":memory:") T.ensure_tables(con) - def sec(key, ticker, priced=True): + def sec(key, ticker, priced=True, exchange="NYSE", source="sec"): T.write_securities( - con, key, [T.SecurityRow(key, ticker, "NYSE", None, None, "s", priced)] + con, + key, + [ + *T.read_securities(con, key), + T.SecurityRow(key, ticker, exchange, None, None, source, priced), + ], ) def link(key, basis, target, strength): @@ -164,20 +169,36 @@ def _universe_db(): con, key, [T.FilingRow(key, f"A-{key}", "10-K", filed, "", None, "", "")] ) - sec("SEED", "SEEDCO") - sec("OLD", "OLDCO", priced=False) - sec("REV", "REVCO") - sec("MEN", "MENCO") - sec("MEN2", "MEN2CO") - sec("SIC", "SICCO") - sec("GONE", "GONECO") + def entity(key, sic): + T.write_entities(con, [T.EntityRow(key, key, "company", cik=key, sic=sic)]) + + sec("SEED", "SEEDC", source="seed") + sec("OLD", "OLDC", priced=False) + sec("REV", "REVC") + sec("REV", "REVCW") # a warrant of REV + sec("REV", "REVCP", exchange="Nasdaq") # a preferred line + sec("MEN", "MENC") + sec("MEN2", "MENB") # one bare mention, pharma SIC: never priced + sec("SVC", "SVCC", exchange="NASDAQ") # one mention, healthcare services + sec("RISK", "RISK") # one model-labelled risk_factor + sec("OTC", "OTCC", exchange="OTC") + sec("NOEX", "NOEX", exchange="") + sec("SIC", "SICC") + sec("GONE", "GONEC") link("SEED", "curated", "PFS", "curated") link("REV", "fts", "PFS", "revenue_exposure") link("MEN", "fts", "PFS", "mention") link("MEN", "fts", "APM", "mention") link("MEN2", "fts", "PFS", "mention") + link("SVC", "fts", "PFS", "mention") + link("RISK", "fts", "PFS", "risk_factor") + link("OTC", "fts", "PFS", "revenue_exposure") + link("NOEX", "fts", "PFS", "revenue_exposure") link("SIC", "sic", "8082", "sector") link("GONE", "fts", "PFS", "revenue_exposure") + entity("MEN2", "2834") + entity("SVC", "8082") + entity("RISK", "3841") tenk("MEN", date(2020, 1, 1)) tenk("MEN2", date(2025, 1, 1)) return con @@ -189,12 +210,18 @@ SUPPORTED = { "SPY", "XLV", "IHF", - "SEEDCO", - "REVCO", - "MENCO", - "MEN2CO", - "SICCO", - "OLDCO", + "SEEDC", + "REVC", + "REVCW", + "REVCP", + "MENC", + "MENB", + "SVCC", + "RISK", + "OTCC", + "NOEX", + "SICC", + "OLDC", ) } @@ -204,14 +231,66 @@ def test_universe_ranked_capped_excludes_sic_only(): kw = {"supported": SUPPORTED, "seed_order": ["SEED"]} got = [t for t, _ in TI.universe(con, used=set(), **kw)] assert got[:3] == ["SPY", "XLV", "IHF"] - assert got[3:] == ["SEEDCO", "REVCO", "MENCO", "MEN2CO"] - assert "SICCO" not in got and "OLDCO" not in got and "GONECO" not in got + # ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC + assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"] + for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"): + assert out not in got keys = dict(TI.universe(con, used=set(), **kw)) - assert keys["REVCO"] == "REV" and keys["SPY"] == "index:SPY" + assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY" assert [t for t, _ in TI.universe(con, used=set(), cap=5, **kw)] == got[:5] capped = [t for t, _ in TI.universe(con, used={"SPY"}, cap=5, **kw)] assert capped[:2] == ["SPY", "XLV"] and len(capped) == 6 - assert "SEEDCO" in capped and "REVCO" in capped and "MEN2CO" not in capped + assert "SEEDC" in capped and "REVC" in capped and "SVCC" not in capped + + +def test_is_common_share(): + ok = TI.is_common_share + assert ok("RDNT", "Nasdaq") and ok("OSH", "NYSE") and ok("A", "NASDAQ") + assert not ok("RSMDF", "OTC") and not ok("ABCD", "") and not ok("ABCD", None) + for bad in ( + "PIIIW", + "BTSGU", + "XYZAR", + "CTRVP", + "LGNDZ", + "BRK-A", + "BF.B", + "ab", + "TOOLONG", + ): + assert not ok(bad, "Nasdaq") + assert ok("ABCDE", "NYSE") # a 5-letter common ending in another letter + + +def test_fts_gate(): + q = TI.fts_qualifies + assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher + assert q(2, 2, "2834") # two targets + assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071") + assert not q(2, 1, "2834") and not q(2, 1, "") + + +def test_universe_primary_ticker_and_curated_tickers(): + con = _universe_db() + T.write_securities( + con, + "SEED", + [ + T.SecurityRow("SEED", "SEEDC", "NYSE", None, None, "seed"), + T.SecurityRow("SEED", "SEEDW", "NYSE", None, None, "sec"), + ], + ) + T.write_securities( + con, + "REV", + [ + T.SecurityRow("REV", "REVCC", "NYSE", None, None, "sec"), + T.SecurityRow("REV", "REVC", "NYSE", None, None, "sec"), + ], + ) + got = dict(TI.universe(con, used=set(), seed_order=["SEED"])) + assert "SEEDC" in got and "SEEDW" not in got + assert "REVC" in got and "REVCC" not in got # the shortest common symbol def test_universe_default_seed_order_reads_the_seed(): @@ -462,11 +541,11 @@ def test_plan_refuses_unpriced_and_unsupported_named_tickers(): def test_universe_records_unsupported_priced_seed_tickers(): con = _universe_db() skipped: dict[str, str] = {} - supported = {k: v for k, v in SUPPORTED.items() if k != "REVCO"} + supported = {k: v for k, v in SUPPORTED.items() if k != "REVC"} TI.universe( con, used=set(), supported=supported, seed_order=["SEED"], skipped=skipped ) - assert skipped == {"REVCO": TI.NOT_SUPPORTED, "GONECO": TI.NOT_SUPPORTED} + assert skipped == {"REVC": TI.NOT_SUPPORTED, "GONEC": TI.NOT_SUPPORTED} def test_clip_windows_nulls_first_return_after_each_gap(): @@ -497,4 +576,4 @@ def test_universe_dedupes_repeated_seed_entries(): con, used=set(), supported=SUPPORTED, seed_order=["SEED", "SEED"] ) ] - assert got.count("SEEDCO") == 1 + assert got.count("SEEDC") == 1