From 7a5f6df1157d7bad190b9532573f80efbb3f2c40 Mon Sep 17 00:00:00 2001 From: kert Date: Fri, 13 Nov 2026 03:20:05 -0500 Subject: [PATCH] fix(market): M17 payment-sentence retirement and stricter pharma/device and other-SIC gates for the fts tier --- src/market/edgar.py | 36 +++++++++++++++++++++++---- src/market/tiingo.py | 25 ++++++++++++++----- tests/cli/test_market_cli.py | 7 +++--- tests/market/test_fts.py | 48 +++++++++++++++++++++++++++++++++++- tests/market/test_tiingo.py | 33 +++++++++++++++++++------ 5 files changed, 125 insertions(+), 24 deletions(-) diff --git a/src/market/edgar.py b/src/market/edgar.py index 0627735..0d04faa 100644 --- a/src/market/edgar.py +++ b/src/market/edgar.py @@ -851,6 +851,26 @@ def ama_safe(passage: str, grams: AmaIndex) -> bool: return not shares_ama_run(passage, grams) +_SENTENCE = re.compile(r"(?<=[.!?])\s+") +_PAYMENT = re.compile( + r"\b(pay(?:s|ing|ment|ments|ed)?|paid|reimburs\w*|fee schedules?|" + r"relative value|conversion factor|medicare|medicaid|cms|" + r"centers for medicare|capitat\w*|bill(?:s|ed|ing)?)\b", + re.I, +) + + +def payment_context(passage: str, term: str) -> bool: + """M17: the sentence(s) holding *term* talk about payment, + reimbursement, the fee schedule, Medicare or CMS. A phrase match such as + employees driving to "home visits" has no such context. Deterministic.""" + needle = term.lower() + return any( + needle in sentence.lower() and _PAYMENT.search(sentence) + for sentence in _SENTENCE.split(passage) + ) + + # ── local classification ───────────────────────────────────────────── EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental") @@ -900,8 +920,10 @@ def fts_links( """``basis="fts"`` links, one per (cik, scope, target): the strongest evidence, then the newest filing. A hit counts when its SIC is a health SIC or its CIK is in the seed. The strength is the passage label, or - ``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit - without a kept passage links with ``passage_id=""``.""" + ``mention`` for an unlabelled passage; ``incidental`` makes no link, and + neither does a passage whose sentence is not about payment (M17, + :func:`payment_context`). A hit with no stored passage links with + ``passage_id=""``.""" seed = set(seed_ciks) sic_of = {e.cik: e.sic for e in entities} by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {} @@ -912,10 +934,14 @@ def fts_links( sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")} if h.cik not in seed and not (sics & set(HEALTH_SICS)): continue - kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), []) + stored = by_doc.get((h.cik, h.adsh, h.filename, h.term), []) cands: list[tuple[str, str]] = [ - (p.label or "mention", p.passage_id) for p in kept - ] or [("mention", "")] + (p.label or "mention", p.passage_id) + for p in stored + if payment_context(p.passage, h.term) + ] + if not stored: + cands = [("mention", "")] for strength, pid in cands: if strength == "incidental": continue diff --git a/src/market/tiingo.py b/src/market/tiingo.py index b0c4cca..55256ef 100644 --- a/src/market/tiingo.py +++ b/src/market/tiingo.py @@ -81,13 +81,26 @@ def is_services_sic(sic: str) -> bool: return sic.startswith("80") or sic in SERVICES_SICS +#: pharma and device SICs: broad phrase matches, so they need more evidence +PHARMA_DEVICE_SICS = ("2834", "2835", "2836", "3841", "3842", "3844", "3845") + + def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool: - """The strength gate of the filing-text tier: a model-labelled strength of - ``risk_factor`` or higher, at least two distinct targets, or a - healthcare-services SIC. A bare single-target mention never qualifies.""" - return ( - best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic) - ) + """The strength gate of the filing-text tier (M16, M17). + + - A healthcare-services SIC (80xx, 6324) qualifies. + - A pharma or device SIC (:data:`PHARMA_DEVICE_SICS`) needs a + ``revenue_exposure`` link or at least two distinct targets. + - Any other SIC needs a ``revenue_exposure`` link. + + A bare single-target mention, and a generic ``risk_factor`` line from a + company that sells no Medicare-paid service, never qualify.""" + revenue = best_rank <= _STRENGTH["revenue_exposure"] + if is_services_sic(sic): + return True + if sic in PHARMA_DEVICE_SICS: + return revenue or n_targets >= 2 + return revenue def new_pacer() -> http.Pacer: diff --git a/tests/cli/test_market_cli.py b/tests/cli/test_market_cli.py index 14944b7..15a5645 100644 --- a/tests/cli/test_market_cli.py +++ b/tests/cli/test_market_cli.py @@ -739,10 +739,9 @@ def test_prices_dry_run_reports_budget(px, monkeypatch): c, _, published, _, seen, _, state = px T.ensure_tables(c) T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")]) - for target in ("PFS", "APM"): # two targets pass the fts strength gate - T.write_links( - c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")] - ) + T.write_links( # revenue_exposure passes the strength gate for any SIC + c, [T.LinkRow("K1", "program", "PFS", "fts", "revenue_exposure", "adsh", "a")] + ) state.mkdir(parents=True) (state / "tiingo-2026-11.json").write_text( json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}}) diff --git a/tests/market/test_fts.py b/tests/market/test_fts.py index 36727cb..7516151 100644 --- a/tests/market/test_fts.py +++ b/tests/market/test_fts.py @@ -264,7 +264,7 @@ def _passage(h, label="", start=10): h.filename, h.term, start, - "text", + "Medicare pays under rbrvs.", label, ) @@ -450,3 +450,49 @@ def test_ama_check_runs_against_a_plain_frozenset(monkeypatch): assert edgar.shares_ama_run("x " + DESC, [DESC]) # the public API still takes descriptors or an index assert not edgar.shares_ama_run("short text", idx) + + +VEHICLE = ( + "Our employees drive company vehicles to home visits, and a collision " + "could expose us to claims." +) +PAID = "Medicare pays us for home visits under the physician fee schedule." + + +def test_payment_context_retires_vehicle_claim_shaped_passages(): + assert not edgar.payment_context(VEHICLE, "home visits") + assert edgar.payment_context(PAID, "home visits") + # the term must sit in the sentence about payment, not a neighbour + both = f"{VEHICLE} {PAID.replace('home visits', 'services')}" + assert not edgar.payment_context(both, "home visits") + assert edgar.payment_context( + "CMS revises the Physician Fee Schedule each year.", "physician fee schedule" + ) + assert edgar.payment_context("It reimburses home-based care.", "home-based care") + assert not edgar.payment_context("A drug for home-based care.", "home-based care") + assert not edgar.payment_context("no term here, but Medicare", "home visits") + + +def test_fts_links_skip_passages_outside_a_payment_sentence(): + h = _hit(term="home visits", target="HOME-VISITS", scope="family") + + def p(text, start, label): + return T.PassageRow( + f"{h.adsh}:a.htm:{start}", + h.cik, + h.adsh, + "a.htm", + h.term, + start, + text, + label, + ) + + bad = p(VEHICLE, 1, "risk_factor") + good = p(PAID, 2, "") + assert edgar.fts_links([h], [bad], [], []) == [] # retired, not a bare mention + (link,) = edgar.fts_links([h], [bad, good], [], []) + assert link.strength == "mention" and link.passage_id.endswith(":2") + # a hit with no stored passage keeps the brief's bare-mention behaviour + (bare,) = edgar.fts_links([h], [], [], []) + assert bare.passage_id == "" diff --git a/tests/market/test_tiingo.py b/tests/market/test_tiingo.py index 997c21d..32b034a 100644 --- a/tests/market/test_tiingo.py +++ b/tests/market/test_tiingo.py @@ -198,7 +198,8 @@ def _universe_db(): link("GONE", "fts", "PFS", "revenue_exposure") entity("MEN2", "2834") entity("SVC", "8082") - entity("RISK", "3841") + entity("RISK", "3841") # a device maker with one generic risk_factor + entity("MEN", "2834") # pharma with two targets tenk("MEN", date(2020, 1, 1)) tenk("MEN2", date(2025, 1, 1)) return con @@ -231,9 +232,19 @@ def test_universe_ranked_capped_excludes_sic_only(): kw = {"supported": SUPPORTED, "seed_order": ["SEED"]} got = [t for t, _ in TI.universe(con, used=set(), **kw)] assert got[:3] == ["SPY", "XLV", "IHF"] - # ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC - assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"] - for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"): + # ranked by strongest link, then targets: REV, MEN (2 targets), SVC + assert got[3:] == ["SEEDC", "REVC", "MENC", "SVCC"] + for out in ( + "SICC", + "OLDC", + "GONEC", + "MENB", + "OTCC", + "NOEX", + "REVCW", + "REVCP", + "RISK", + ): assert out not in got keys = dict(TI.universe(con, used=set(), **kw)) assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY" @@ -264,10 +275,16 @@ def test_is_common_share(): def test_fts_gate(): q = TI.fts_qualifies - assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher - assert q(2, 2, "2834") # two targets - assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071") - assert not q(2, 1, "2834") and not q(2, 1, "") + for sic in ("2834", "2835", "2836", "3841", "3842", "3844", "3845"): + # pharma and device: revenue_exposure or two targets + assert q(0, 1, sic) and q(2, 2, sic) and q(1, 2, sic) + assert not q(1, 1, sic) and not q(2, 1, sic) # generic risk_factor + for sic in ("", "5047", "6798", "2711"): + # any other SIC: revenue_exposure only, however many targets + assert q(0, 1, sic) and not q(1, 1, sic) and not q(1, 3, sic) + assert not q(2, 5, sic) + for sic in ("8082", "6324", "8071", "8062", "8011"): + assert q(2, 1, sic) and q(0, 1, sic) # healthcare services def test_universe_primary_ticker_and_curated_tickers():