From 79743fea4d948bb56c889ba0d4e8840992c0f7a5 Mon Sep 17 00:00:00 2001 From: kert Date: Thu, 12 Nov 2026 19:12:28 -0500 Subject: [PATCH] =?UTF-8?q?fix(pfs):=20PSPS=20counts=20survive=20suppressi?= =?UTF-8?q?on=20=E2=80=94=20services=5Freported/n=5Fsuppressed=20beside=20?= =?UTF-8?q?the=20strict=20sum,=20money=20flagged=20as=20a=20lower=20bound?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/cli/pfs.py | 7 +++++- src/pfs/codetables.py | 19 ++++++++++++++++- src/pfs/psps.py | 12 ++++++++++- src/pfs/stakes.py | 30 ++++++++++++++++++-------- tests/cli/test_pfs_cli.py | 4 +++- tests/pfs/test_psps.py | 45 +++++++++++++++++++++++++++++++++++++++ tests/pfs/test_stakes.py | 19 +++++++++++++++++ 7 files changed, 123 insertions(+), 13 deletions(-) diff --git a/src/cli/pfs.py b/src/cli/pfs.py index 57d5776..b018571 100644 --- a/src/cli/pfs.py +++ b/src/cli/pfs.py @@ -1150,7 +1150,12 @@ def psps( con.close() if not write: for r in rows: - svc = f"{r.services:,.0f}" if r.services is not None else "-" + if r.services is not None: + svc = f"{r.services:,.0f}" + elif r.services_reported is not None: + svc = f">={r.services_reported:,.0f}" # suppressed lines omitted + else: + svc = "-" typer.echo( f"{r.year} {r.hcpcs} {r.specialty_cd} {r.specialty:<32} {svc:>12}" ) diff --git a/src/pfs/codetables.py b/src/pfs/codetables.py index 10780be..37c1025 100644 --- a/src/pfs/codetables.py +++ b/src/pfs/codetables.py @@ -515,6 +515,14 @@ class PspsRow: paid: float | None dataset_id: str item_key: str + #: Sum of the lines whose service count was reported (a lower bound on + #: ``services`` when ``n_suppressed`` > 0). In the real file two thirds + #: of the lines of a code are suppressed (< 11 services), so the strict + #: ``services`` is ``None`` for nearly every specialty; this is what the + #: file does state. ``None`` when no line reported a count. + services_reported: float | None = None + #: How many lines had a suppressed service count (``"*"``). + n_suppressed: int = 0 @dataclass(frozen=True) @@ -548,7 +556,7 @@ CREATE SCHEMA IF NOT EXISTS pfs; CREATE TABLE IF NOT EXISTS pfs.utilization_specialty ( year INTEGER, hcpcs VARCHAR, specialty_cd VARCHAR, specialty VARCHAR, services DOUBLE, allowed DOUBLE, paid DOUBLE, dataset_id VARCHAR, - item_key VARCHAR); + item_key VARCHAR, services_reported DOUBLE, n_suppressed INTEGER); CREATE TABLE IF NOT EXISTS pfs.specialty_impact ( rule_year INTEGER, vintage VARCHAR, specialty VARCHAR, setting VARCHAR, allowed_mil DOUBLE, impact_work DOUBLE, impact_pe DOUBLE, impact_mp DOUBLE, @@ -709,6 +717,15 @@ def ensure_tables(con: Any) -> None: "ALTER TABLE pfs.comment_stance ADD COLUMN IF NOT EXISTS reviewed BOOLEAN " "DEFAULT false" ) + # T10: PSPS suppression-aware service counts + con.execute( + "ALTER TABLE pfs.utilization_specialty " + "ADD COLUMN IF NOT EXISTS services_reported DOUBLE" + ) + con.execute( + "ALTER TABLE pfs.utilization_specialty " + "ADD COLUMN IF NOT EXISTS n_suppressed INTEGER" + ) con.execute("UPDATE pfs.commenter SET family = 'HOME-VISITS' WHERE family IS NULL") con.execute( "UPDATE pfs.commenter SET submitter_name = '' WHERE submitter_name IS NULL" diff --git a/src/pfs/psps.py b/src/pfs/psps.py index 416d508..1f29f34 100644 --- a/src/pfs/psps.py +++ b/src/pfs/psps.py @@ -175,7 +175,10 @@ def to_rows( """National totals by ``PROVIDER_SPEC_CD`` for *code*: submitted services, allowed charges and NCH payment summed over every line (carrier, locality, place of service, modifier). Error-flagged lines are skipped; - a suppressed (``"*"``) or blank cell makes that sum ``None``.""" + a suppressed (``"*"``) or blank cell makes that sum ``None``. Because + CMS suppresses every line under 11 services (about two thirds of a + code's lines), ``services_reported`` and ``n_suppressed`` also carry the + sum of the lines that did report and how many did not.""" code = code.upper() buckets: dict[str, list[dict]] = {} for rec in records: @@ -187,6 +190,11 @@ def to_rows( out: list[PspsRow] = [] for cd in sorted(buckets): recs = buckets[cd] + reported = [ + n + for n in (_num(r.get("PSPS_SUBMITTED_SERVICE_CNT")) for r in recs) + if n is not None + ] out.append( PspsRow( year=year, @@ -200,6 +208,8 @@ def to_rows( paid=_sum([_num(r.get("PSPS_NCH_PAYMENT_AMT")) for r in recs]), dataset_id=dataset_id, item_key=item_key, + services_reported=round(sum(reported), 2) if reported else None, + n_suppressed=len(recs) - len(reported), ) ) return out diff --git a/src/pfs/stakes.py b/src/pfs/stakes.py index ab691ae..f009f8b 100644 --- a/src/pfs/stakes.py +++ b/src/pfs/stakes.py @@ -259,7 +259,8 @@ class SpecialtyMoney: ``delta_pct`` is ``delta_total`` over the same codes' year-1 payments, as a fraction. ``proposed_*`` are the same figures from the NPRM's proposed payment (``rvu_proposed``, CY2026-27) against the year-1 final, - kept apart from the final figures. ``new_payment`` is True when a code + kept apart from the final figures. ``lower_bound`` marks a specialty whose + PSPS counts include suppressed lines. ``new_payment`` is True when a code the specialty bills first became payable in *year*.""" specialty: str @@ -272,6 +273,9 @@ class SpecialtyMoney: proposed_delta_per_service: float | None = None proposed_delta_total: float | None = None proposed_delta_pct: float | None = None + #: True when some contributing PSPS row had suppressed lines, so + #: ``services`` (and everything scaled by it) is a lower bound. + lower_bound: bool = False def _latest_services(psps: Sequence[PspsRow], year: int) -> dict[str, dict[str, float]]: @@ -281,13 +285,17 @@ def _latest_services(psps: Sequence[PspsRow], year: int) -> dict[str, dict[str, for r in psps: if r.year <= year - 1: latest[r.hcpcs] = max(latest.get(r.hcpcs, 0), r.year) - out: dict[str, dict[str, float]] = {} + out: dict[str, dict[str, tuple[float, bool]]] = {} for r in psps: - if latest.get(r.hcpcs) != r.year or r.services is None: + if latest.get(r.hcpcs) != r.year: + continue + n = r.services if r.services is not None else r.services_reported + if n is None: continue key = specialty_key(r.specialty) or f"CODE {r.specialty_cd}" d = out.setdefault(r.hcpcs, {}) - d[key] = d.get(key, 0.0) + r.services + prior, lb = d.get(key, (0.0, False)) + d[key] = (prior + n, lb or r.services is None) return out @@ -296,7 +304,7 @@ def billing_share(psps: Sequence[PspsRow], *, year: int) -> dict[str, float]: services, latest PSPS year <= year-1 per code.""" totals: dict[str, float] = {} for per in _latest_services(psps, year).values(): - for k, v in per.items(): + for k, (v, _lb) in per.items(): totals[k] = totals.get(k, 0.0) + v whole = sum(totals.values()) if whole <= 0: @@ -328,12 +336,15 @@ def money_by_specialty( is_new = now is not None and not any( pay is not None for (c, y), pay in final.items() if c == code and y < year ) - for spec, n in per.items(): + for spec, (n, lb) in per.items(): a = acc.setdefault( spec, - dict(svc=0.0, d=None, base=None, pd=None, pbase=None, new=False), + dict( + svc=0.0, d=None, base=None, pd=None, pbase=None, new=False, lb=False + ), ) a["svc"] += n + a["lb"] = a["lb"] or lb a["new"] = a["new"] or is_new if now is not None and before is not None: a["d"] = (a["d"] or 0.0) + (now - before) * n @@ -352,7 +363,7 @@ def money_by_specialty( d_svc = sum( n for code, per in services.items() - for s, n in per.items() + for s, (n, _lb) in per.items() if s == spec and final.get((code, year)) is not None and final.get((code, year - 1)) is not None @@ -360,7 +371,7 @@ def money_by_specialty( p_svc = sum( n for code, per in services.items() - for s, n in per.items() + for s, (n, _lb) in per.items() if s == spec and proposed.get(code) is not None and final.get((code, year - 1)) is not None @@ -374,6 +385,7 @@ def money_by_specialty( share=a["svc"] / whole if whole else 0.0, delta_pct=ratio(a["d"], a["base"]), new_payment=a["new"], + lower_bound=a["lb"], proposed_delta_per_service=ratio(a["pd"], p_svc), proposed_delta_total=a["pd"], proposed_delta_pct=ratio(a["pd"], a["pbase"]), diff --git a/tests/cli/test_pfs_cli.py b/tests/cli/test_pfs_cli.py index e1ede6e..4420754 100644 --- a/tests/cli/test_pfs_cli.py +++ b/tests/cli/test_pfs_cli.py @@ -2943,10 +2943,12 @@ class TestPsps: def test_read_prints_dash_for_suppressed_and_reports_missing_table(self, con, env): con.execute( "INSERT INTO pfs.utilization_specialty VALUES " - "(2023, '99350', '50', 'NP', NULL, NULL, NULL, 'd', 'K')" + "(2023, '99350', '50', 'NP', NULL, NULL, NULL, 'd', 'K', NULL, 2), " + "(2023, '99350', '08', 'FP', NULL, NULL, NULL, 'd', 'K', 1234, 2)" ) res = runner.invoke(app, ["pfs", "psps", "--family", "HOME-VISITS"]) assert res.exit_code == 0 and " -" in res.output + assert ">=1,234" in res.output con.execute("DROP TABLE pfs.utilization_specialty") res = runner.invoke(app, ["pfs", "psps", "--family", "HOME-VISITS"]) assert "no PSPS table yet" in res.output diff --git a/tests/pfs/test_psps.py b/tests/pfs/test_psps.py index 6386f57..d021452 100644 --- a/tests/pfs/test_psps.py +++ b/tests/pfs/test_psps.py @@ -75,6 +75,34 @@ class TestToRows: assert fp.services == 20.0 assert fp.allowed is None # a suppressed part makes the sum None + def test_reported_lines_and_suppression_count_kept_beside_the_strict_sum(self): + records = [ + rec(), + rec(PSPS_SUBMITTED_SERVICE_CNT="*", PSPS_ALLOWED_CHARGE_AMT="*"), + rec(PROVIDER_SPEC_CD="08", PSPS_SUBMITTED_SERVICE_CNT="*"), + ] + by = { + r.specialty_cd: r + for r in to_rows( + 2023, "99350", records, dataset_id="d", item_key="K", names=NAMES + ) + } + np_, fp = by["50"], by["08"] + assert np_.services is None and np_.services_reported == 10.0 + assert np_.n_suppressed == 1 + assert fp.services is None and fp.services_reported is None + assert fp.n_suppressed == 1 + clean = to_rows(2023, "99350", [rec()], dataset_id="d", item_key="K", names={}) + assert ( + clean[0].services, + clean[0].services_reported, + clean[0].n_suppressed, + ) == ( + 10.0, + 10.0, + 0, + ) + def test_unknown_specialty_name_blank_and_bad_numbers_none(self): rows = to_rows( 2023, @@ -235,6 +263,23 @@ class TestWriteRead: (2023, "99350", 9.0), ] + def test_old_shape_table_gains_the_suppression_columns(self): + c = duckdb.connect(":memory:") + c.execute("CREATE SCHEMA pfs") + c.execute( + "CREATE TABLE pfs.utilization_specialty (year INTEGER, hcpcs VARCHAR, " + "specialty_cd VARCHAR, specialty VARCHAR, services DOUBLE, allowed DOUBLE, " + "paid DOUBLE, dataset_id VARCHAR, item_key VARCHAR)" + ) + c.execute( + "INSERT INTO pfs.utilization_specialty VALUES (2020,'A','1','X',1,2,3,'d','K')" + ) + row = PspsRow(2021, "A", "1", "X", None, None, 3.0, "d", "K", 4.0, 2) + assert write(c, 2021, "A", [row]) == 1 + got = read_by_specialty(c, ["A"]) + assert got[0].services_reported is None and got[1] == row + c.close() + def test_read_filters_years_and_handles_empty(self, con): write(con, 2023, "99350", [self._row(2023, "99350")]) write(con, 2022, "99350", [self._row(2022, "99350")]) diff --git a/tests/pfs/test_stakes.py b/tests/pfs/test_stakes.py index f7315ae..ff288e2 100644 --- a/tests/pfs/test_stakes.py +++ b/tests/pfs/test_stakes.py @@ -354,6 +354,25 @@ def test_proposed_vs_final_delta_kept_apart(): assert m.proposed_delta_pct == pytest.approx(-0.2) +def test_suppressed_lines_make_a_lower_bound_not_a_none(): + payment = [val("A", 2022, 100.0), val("A", 2023, 90.0)] + psps = [ + PspsRow(2022, "A", "08", "Family Practice", None, None, 5.0, "d", "K", 12.0, 3), + PspsRow(2022, "A", "50", "Nurse Practitioner", 8.0, 1.0, 5.0, "d", "K", 8.0, 0), + PspsRow(2022, "A", "06", "Cardiology", None, None, 5.0, "d", "K", None, 2), + ] + by = {m.specialty: m for m in money_by_specialty(payment, psps, year=2023)} + assert set(by) == {"FAMILY PRACTICE", "NURSE PRACTITIONER"} + assert by["FAMILY PRACTICE"].services == 12.0 + assert by["FAMILY PRACTICE"].lower_bound is True + assert by["FAMILY PRACTICE"].delta_total == -120.0 + assert by["NURSE PRACTITIONER"].lower_bound is False + assert billing_share(psps, year=2023) == { + "FAMILY PRACTICE": 0.6, + "NURSE PRACTITIONER": 0.4, + } + + def test_unnamed_specialty_falls_back_to_code_and_empty_inputs(): (m,) = money_by_specialty( [val("A", 2022, 1.0), val("A", 2023, 2.0)],