fix(market): M17 payment-sentence retirement and stricter pharma/device and other-SIC gates for the fts tier
This commit is contained in:
@@ -851,6 +851,26 @@ def ama_safe(passage: str, grams: AmaIndex) -> bool:
|
||||
return not shares_ama_run(passage, grams)
|
||||
|
||||
|
||||
_SENTENCE = re.compile(r"(?<=[.!?])\s+")
|
||||
_PAYMENT = re.compile(
|
||||
r"\b(pay(?:s|ing|ment|ments|ed)?|paid|reimburs\w*|fee schedules?|"
|
||||
r"relative value|conversion factor|medicare|medicaid|cms|"
|
||||
r"centers for medicare|capitat\w*|bill(?:s|ed|ing)?)\b",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def payment_context(passage: str, term: str) -> bool:
|
||||
"""M17: the sentence(s) holding *term* talk about payment,
|
||||
reimbursement, the fee schedule, Medicare or CMS. A phrase match such as
|
||||
employees driving to "home visits" has no such context. Deterministic."""
|
||||
needle = term.lower()
|
||||
return any(
|
||||
needle in sentence.lower() and _PAYMENT.search(sentence)
|
||||
for sentence in _SENTENCE.split(passage)
|
||||
)
|
||||
|
||||
|
||||
# ── local classification ─────────────────────────────────────────────
|
||||
|
||||
EXPOSURE_LABELS = ("revenue_exposure", "risk_factor", "incidental")
|
||||
@@ -900,8 +920,10 @@ def fts_links(
|
||||
"""``basis="fts"`` links, one per (cik, scope, target): the strongest
|
||||
evidence, then the newest filing. A hit counts when its SIC is a health
|
||||
SIC or its CIK is in the seed. The strength is the passage label, or
|
||||
``mention`` for an unlabelled passage; ``incidental`` makes no link. A hit
|
||||
without a kept passage links with ``passage_id=""``."""
|
||||
``mention`` for an unlabelled passage; ``incidental`` makes no link, and
|
||||
neither does a passage whose sentence is not about payment (M17,
|
||||
:func:`payment_context`). A hit with no stored passage links with
|
||||
``passage_id=""``."""
|
||||
seed = set(seed_ciks)
|
||||
sic_of = {e.cik: e.sic for e in entities}
|
||||
by_doc: dict[tuple[str, str, str, str], list[PassageRow]] = {}
|
||||
@@ -912,10 +934,14 @@ def fts_links(
|
||||
sics = set(h.sics.split(",")) | {sic_of.get(h.cik, "")}
|
||||
if h.cik not in seed and not (sics & set(HEALTH_SICS)):
|
||||
continue
|
||||
kept = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
|
||||
stored = by_doc.get((h.cik, h.adsh, h.filename, h.term), [])
|
||||
cands: list[tuple[str, str]] = [
|
||||
(p.label or "mention", p.passage_id) for p in kept
|
||||
] or [("mention", "")]
|
||||
(p.label or "mention", p.passage_id)
|
||||
for p in stored
|
||||
if payment_context(p.passage, h.term)
|
||||
]
|
||||
if not stored:
|
||||
cands = [("mention", "")]
|
||||
for strength, pid in cands:
|
||||
if strength == "incidental":
|
||||
continue
|
||||
|
||||
@@ -81,13 +81,26 @@ def is_services_sic(sic: str) -> bool:
|
||||
return sic.startswith("80") or sic in SERVICES_SICS
|
||||
|
||||
|
||||
#: pharma and device SICs: broad phrase matches, so they need more evidence
|
||||
PHARMA_DEVICE_SICS = ("2834", "2835", "2836", "3841", "3842", "3844", "3845")
|
||||
|
||||
|
||||
def fts_qualifies(best_rank: int, n_targets: int, sic: str) -> bool:
|
||||
"""The strength gate of the filing-text tier: a model-labelled strength of
|
||||
``risk_factor`` or higher, at least two distinct targets, or a
|
||||
healthcare-services SIC. A bare single-target mention never qualifies."""
|
||||
return (
|
||||
best_rank <= _STRENGTH["risk_factor"] or n_targets >= 2 or is_services_sic(sic)
|
||||
)
|
||||
"""The strength gate of the filing-text tier (M16, M17).
|
||||
|
||||
- A healthcare-services SIC (80xx, 6324) qualifies.
|
||||
- A pharma or device SIC (:data:`PHARMA_DEVICE_SICS`) needs a
|
||||
``revenue_exposure`` link or at least two distinct targets.
|
||||
- Any other SIC needs a ``revenue_exposure`` link.
|
||||
|
||||
A bare single-target mention, and a generic ``risk_factor`` line from a
|
||||
company that sells no Medicare-paid service, never qualify."""
|
||||
revenue = best_rank <= _STRENGTH["revenue_exposure"]
|
||||
if is_services_sic(sic):
|
||||
return True
|
||||
if sic in PHARMA_DEVICE_SICS:
|
||||
return revenue or n_targets >= 2
|
||||
return revenue
|
||||
|
||||
|
||||
def new_pacer() -> http.Pacer:
|
||||
|
||||
@@ -739,10 +739,9 @@ def test_prices_dry_run_reports_budget(px, monkeypatch):
|
||||
c, _, published, _, seen, _, state = px
|
||||
T.ensure_tables(c)
|
||||
T.write_securities(c, "K1", [T.SecurityRow("K1", "AAA", "NYSE", None, None, "s")])
|
||||
for target in ("PFS", "APM"): # two targets pass the fts strength gate
|
||||
T.write_links(
|
||||
c, [T.LinkRow("K1", "program", target, "fts", "mention", "adsh", "a")]
|
||||
)
|
||||
T.write_links( # revenue_exposure passes the strength gate for any SIC
|
||||
c, [T.LinkRow("K1", "program", "PFS", "fts", "revenue_exposure", "adsh", "a")]
|
||||
)
|
||||
state.mkdir(parents=True)
|
||||
(state / "tiingo-2026-11.json").write_text(
|
||||
json.dumps({"month": "2026-11", "symbols": ["SPY"], "day_counts": {}})
|
||||
|
||||
@@ -264,7 +264,7 @@ def _passage(h, label="", start=10):
|
||||
h.filename,
|
||||
h.term,
|
||||
start,
|
||||
"text",
|
||||
"Medicare pays under rbrvs.",
|
||||
label,
|
||||
)
|
||||
|
||||
@@ -450,3 +450,49 @@ def test_ama_check_runs_against_a_plain_frozenset(monkeypatch):
|
||||
assert edgar.shares_ama_run("x " + DESC, [DESC])
|
||||
# the public API still takes descriptors or an index
|
||||
assert not edgar.shares_ama_run("short text", idx)
|
||||
|
||||
|
||||
VEHICLE = (
|
||||
"Our employees drive company vehicles to home visits, and a collision "
|
||||
"could expose us to claims."
|
||||
)
|
||||
PAID = "Medicare pays us for home visits under the physician fee schedule."
|
||||
|
||||
|
||||
def test_payment_context_retires_vehicle_claim_shaped_passages():
|
||||
assert not edgar.payment_context(VEHICLE, "home visits")
|
||||
assert edgar.payment_context(PAID, "home visits")
|
||||
# the term must sit in the sentence about payment, not a neighbour
|
||||
both = f"{VEHICLE} {PAID.replace('home visits', 'services')}"
|
||||
assert not edgar.payment_context(both, "home visits")
|
||||
assert edgar.payment_context(
|
||||
"CMS revises the Physician Fee Schedule each year.", "physician fee schedule"
|
||||
)
|
||||
assert edgar.payment_context("It reimburses home-based care.", "home-based care")
|
||||
assert not edgar.payment_context("A drug for home-based care.", "home-based care")
|
||||
assert not edgar.payment_context("no term here, but Medicare", "home visits")
|
||||
|
||||
|
||||
def test_fts_links_skip_passages_outside_a_payment_sentence():
|
||||
h = _hit(term="home visits", target="HOME-VISITS", scope="family")
|
||||
|
||||
def p(text, start, label):
|
||||
return T.PassageRow(
|
||||
f"{h.adsh}:a.htm:{start}",
|
||||
h.cik,
|
||||
h.adsh,
|
||||
"a.htm",
|
||||
h.term,
|
||||
start,
|
||||
text,
|
||||
label,
|
||||
)
|
||||
|
||||
bad = p(VEHICLE, 1, "risk_factor")
|
||||
good = p(PAID, 2, "")
|
||||
assert edgar.fts_links([h], [bad], [], []) == [] # retired, not a bare mention
|
||||
(link,) = edgar.fts_links([h], [bad, good], [], [])
|
||||
assert link.strength == "mention" and link.passage_id.endswith(":2")
|
||||
# a hit with no stored passage keeps the brief's bare-mention behaviour
|
||||
(bare,) = edgar.fts_links([h], [], [], [])
|
||||
assert bare.passage_id == ""
|
||||
|
||||
@@ -198,7 +198,8 @@ def _universe_db():
|
||||
link("GONE", "fts", "PFS", "revenue_exposure")
|
||||
entity("MEN2", "2834")
|
||||
entity("SVC", "8082")
|
||||
entity("RISK", "3841")
|
||||
entity("RISK", "3841") # a device maker with one generic risk_factor
|
||||
entity("MEN", "2834") # pharma with two targets
|
||||
tenk("MEN", date(2020, 1, 1))
|
||||
tenk("MEN2", date(2025, 1, 1))
|
||||
return con
|
||||
@@ -231,9 +232,19 @@ def test_universe_ranked_capped_excludes_sic_only():
|
||||
kw = {"supported": SUPPORTED, "seed_order": ["SEED"]}
|
||||
got = [t for t, _ in TI.universe(con, used=set(), **kw)]
|
||||
assert got[:3] == ["SPY", "XLV", "IHF"]
|
||||
# ranked by strongest link, then targets: REV, RISK, MEN (2 targets), SVC
|
||||
assert got[3:] == ["SEEDC", "REVC", "RISK", "MENC", "SVCC"]
|
||||
for out in ("SICC", "OLDC", "GONEC", "MENB", "OTCC", "NOEX", "REVCW", "REVCP"):
|
||||
# ranked by strongest link, then targets: REV, MEN (2 targets), SVC
|
||||
assert got[3:] == ["SEEDC", "REVC", "MENC", "SVCC"]
|
||||
for out in (
|
||||
"SICC",
|
||||
"OLDC",
|
||||
"GONEC",
|
||||
"MENB",
|
||||
"OTCC",
|
||||
"NOEX",
|
||||
"REVCW",
|
||||
"REVCP",
|
||||
"RISK",
|
||||
):
|
||||
assert out not in got
|
||||
keys = dict(TI.universe(con, used=set(), **kw))
|
||||
assert keys["REVC"] == "REV" and keys["SPY"] == "index:SPY"
|
||||
@@ -264,10 +275,16 @@ def test_is_common_share():
|
||||
|
||||
def test_fts_gate():
|
||||
q = TI.fts_qualifies
|
||||
assert q(1, 1, "2834") and q(0, 1, "2834") # risk_factor or higher
|
||||
assert q(2, 2, "2834") # two targets
|
||||
assert q(2, 1, "8082") and q(2, 1, "6324") and q(2, 1, "8071")
|
||||
assert not q(2, 1, "2834") and not q(2, 1, "")
|
||||
for sic in ("2834", "2835", "2836", "3841", "3842", "3844", "3845"):
|
||||
# pharma and device: revenue_exposure or two targets
|
||||
assert q(0, 1, sic) and q(2, 2, sic) and q(1, 2, sic)
|
||||
assert not q(1, 1, sic) and not q(2, 1, sic) # generic risk_factor
|
||||
for sic in ("", "5047", "6798", "2711"):
|
||||
# any other SIC: revenue_exposure only, however many targets
|
||||
assert q(0, 1, sic) and not q(1, 1, sic) and not q(1, 3, sic)
|
||||
assert not q(2, 5, sic)
|
||||
for sic in ("8082", "6324", "8071", "8062", "8011"):
|
||||
assert q(2, 1, sic) and q(0, 1, sic) # healthcare services
|
||||
|
||||
|
||||
def test_universe_primary_ticker_and_curated_tickers():
|
||||
|
||||
Reference in New Issue
Block a user