fix(market): M17 payment-sentence retirement and stricter pharma/device and other-SIC gates for the fts tier

This commit is contained in:
kert
2026-11-13 03:20:05 -05:00
parent 9c93a6b379
commit 7a5f6df115
5 changed files with 125 additions and 24 deletions

View File

@@ -851,6 +851,26 @@ def ama_safe(passage: str, grams: AmaIndex) -> bool:
return not shares_ama_run(passage, grams)
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
_PAYMENT = re.compile(
r"\b(pay(?:s|ing|ment|ments|ed)?|paid|reimburs\w*|fee schedules?|"
r"relative value|conversion factor|medicare|medicaid|cms|"
r"centers for medicare|capitat\w*|bill(?:s|ed|ing)?)\b",
re.I,
)
def payment_context(passage: str, term: str) -> bool:
"""M17: the sentence(s) holding *term* talk about payment,
reimbursement, the fee schedule, Medicare or CMS. A phrase match such as
employees driving to "home visits" has no such context. Deterministic."""
needle = term.lower()
return any(
needle in sentence.lower() and _PAYMENT.search(sentence)
for sentence in _SENTENCE.split(passage)
)
# ── local classification ─────────────────────────────────────────────
EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental")
@@ -900,8 +920,10 @@ def fts_links(
"""``basis="fts"`` links, one per (cik, scope, target): the strongest
evidence, then the newest filing. A hit counts when its SIC is a health
SIC or its CIK is in the seed. The strength is the passage label, or
``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit
without a kept passage links with ``passage_id=""``."""
``mention`` for an unlabelled passage; ``incidental`` makes no link, and
neither does a passage whose sentence is not about payment (M17,
:func:`payment_context`). A hit with no stored passage links with
``passage_id=""``."""
seed = set(seed_ciks)
sic_of = {e.cik: e.sic for e in entities}
by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {}
@@ -912,10 +934,14 @@ def fts_links(
sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")}
if h.cik not in seed and not (sics & set(HEALTH_SICS)):
continue
kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
stored = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
cands: list[tuple[str, str]] = [
(p.label or "mention", p.passage_id) for p in kept
] or [("mention", "")]
(p.label or "mention", p.passage_id)
for p in stored
if payment_context(p.passage, h.term)
]
if not stored:
cands = [("mention", "")]
for strength, pid in cands:
if strength == "incidental":
continue

View File

@@ -81,13 +81,26 @@ def is_services_sic(sic: str) -> bool:
return sic.startswith("80") or sic in SERVICES_SICS
#: pharma and device SICs: broad phrase matches, so they need more evidence
PHARMA_DEVICE_SICS = ("2834", "2835", "2836", "3841", "3842", "3844", "3845")
def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool:
"""The strength gate of the filing-text tier: a model-labelled strength of
``risk_factor`` or higher, at least two distinct targets, or a
healthcare-services SIC. A bare single-target mention never qualifies."""
return (
best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic)
)
"""The strength gate of the filing-text tier (M16, M17).
- A healthcare-services SIC (80xx, 6324) qualifies.
- A pharma or device SIC (:data:`PHARMA_DEVICE_SICS`) needs a
``revenue_exposure`` link or at least two distinct targets.
- Any other SIC needs a ``revenue_exposure`` link.
A bare single-target mention, and a generic ``risk_factor`` line from a
company that sells no Medicare-paid service, never qualify."""
revenue = best_rank <= _STRENGTH["revenue_exposure"]
if is_services_sic(sic):
return True
if sic in PHARMA_DEVICE_SICS:
return revenue or n_targets >= 2
return revenue
def new_pacer() -> http.Pacer:

View File

@@ -739,10 +739,9 @@ def test_prices_dry_run_reports_budget(px, monkeypatch):
c, _, published, _, seen, _, state = px
T.ensure_tables(c)
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
for target in ("PFS", "APM"): # two targets pass the fts strength gate
T.write_links(
c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")]
)
T.write_links( # revenue_exposure passes the strength gate for any SIC
c, [T.LinkRow("K1", "program", "PFS", "fts", "revenue_exposure", "adsh", "a")]
)
state.mkdir(parents=True)
(state / "tiingo-2026-11.json").write_text(
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})

View File

@@ -264,7 +264,7 @@ def _passage(h, label="", start=10):
h.filename,
h.term,
start,
"text",
"Medicare pays under rbrvs.",
label,
)
@@ -450,3 +450,49 @@ def test_ama_check_runs_against_a_plain_frozenset(monkeypatch):
assert edgar.shares_ama_run("x " + DESC, [DESC])
# the public API still takes descriptors or an index
assert not edgar.shares_ama_run("short text", idx)
VEHICLE = (
"Our employees drive company vehicles to home visits, and a collision "
"could expose us to claims."
)
PAID = "Medicare pays us for home visits under the physician fee schedule."
def test_payment_context_retires_vehicle_claim_shaped_passages():
assert not edgar.payment_context(VEHICLE, "home visits")
assert edgar.payment_context(PAID, "home visits")
# the term must sit in the sentence about payment, not a neighbour
both = f"{VEHICLE} {PAID.replace('home visits', 'services')}"
assert not edgar.payment_context(both, "home visits")
assert edgar.payment_context(
"CMS revises the Physician Fee Schedule each year.", "physician fee schedule"
)
assert edgar.payment_context("It reimburses home-based care.", "home-based care")
assert not edgar.payment_context("A drug for home-based care.", "home-based care")
assert not edgar.payment_context("no term here, but Medicare", "home visits")
def test_fts_links_skip_passages_outside_a_payment_sentence():
h = _hit(term="home visits", target="HOME-VISITS", scope="family")
def p(text, start, label):
return T.PassageRow(
f"{h.adsh}:a.htm:{start}",
h.cik,
h.adsh,
"a.htm",
h.term,
start,
text,
label,
)
bad = p(VEHICLE, 1, "risk_factor")
good = p(PAID, 2, "")
assert edgar.fts_links([h], [bad], [], []) == [] # retired, not a bare mention
(link,) = edgar.fts_links([h], [bad, good], [], [])
assert link.strength == "mention" and link.passage_id.endswith(":2")
# a hit with no stored passage keeps the brief's bare-mention behaviour
(bare,) = edgar.fts_links([h], [], [], [])
assert bare.passage_id == ""

View File

@@ -198,7 +198,8 @@ def _universe_db():
link("GONE", "fts", "PFS", "revenue_exposure")
entity("MEN2", "2834")
entity("SVC", "8082")
entity("RISK", "3841")
entity("RISK", "3841") # a device maker with one generic risk_factor
entity("MEN", "2834") # pharma with two targets
tenk("MEN", date(2020, 1, 1))
tenk("MEN2", date(2025, 1, 1))
return con
@@ -231,9 +232,19 @@ def test_universe_ranked_capped_excludes_sic_only():
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
assert got[:3] == ["SPY", "XLV", "IHF"]
# ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC
assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"]
for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"):
# ranked by strongest link, then targets: REV, MEN (2 targets), SVC
assert got[3:] == ["SEEDC", "REVC", "MENC", "SVCC"]
for out in (
"SICC",
"OLDC",
"GONEC",
"MENB",
"OTCC",
"NOEX",
"REVCW",
"REVCP",
"RISK",
):
assert out not in got
keys = dict(TI.universe(con, used=set(), **kw))
assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY"
@@ -264,10 +275,16 @@ def test_is_common_share():
def test_fts_gate():
q = TI.fts_qualifies
assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher
assert q(2, 2, "2834") # two targets
assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071")
assert not q(2, 1, "2834") and not q(2, 1, "")
for sic in ("2834", "2835", "2836", "3841", "3842", "3844", "3845"):
# pharma and device: revenue_exposure or two targets
assert q(0, 1, sic) and q(2, 2, sic) and q(1, 2, sic)
assert not q(1, 1, sic) and not q(2, 1, sic) # generic risk_factor
for sic in ("", "5047", "6798", "2711"):
# any other SIC: revenue_exposure only, however many targets
assert q(0, 1, sic) and not q(1, 1, sic) and not q(1, 3, sic)
assert not q(2, 5, sic)
for sic in ("8082", "6324", "8071", "8062", "8011"):
assert q(2, 1, sic) and q(0, 1, sic) # healthcare services
def test_universe_primary_ticker_and_curated_tickers():