stack/dev/scripts/add_pprrvu_historical_to_zotero.py

"""Backfill ALL quarterly PPRRVU releases for CY2015–CY2024 into Zotero.

CMS publishes a fresh PPRRVU file each calendar quarter.  Sometimes
the CF and/or RVUs carry the same values across all four quarters;
sometimes a mid-year CAA retroactive update changes them.  Example:

    2015-2023: CF constant across Q1/Q2/Q3/Q4 for a given year.
    2024:      Q1 CF = 32.7442 (original Final Rule),
               Q2/Q3/Q4 CF = 33.2875 (after CAA 2024 adjustment).

We archive ALL four quarters per year to preserve provenance and let
downstream code pick the release that matches a given carrier file's
publication date.  See :pincite:`VVBEVYLC` — the AMA CF history table
— for the authoritative summary across years.

Each year's four releases are added to Zotero as a SINGLE parent
webpage item with ``year:YYYY`` + ``file:rvu`` + ``source:cms-website``
+ ``release:q1/q2/q3/q4`` tags; every release's xlsx/csv/txt trio
becomes an attachment on that parent.

Source zip URLs live in ``/tmp/rvu_urls_final.txt`` (tab-separated
``year\\tquarter\\turl``) produced by the CMS PFS RVU index scraper.

Usage:
    uv run python dev/scripts/add_pprrvu_historical_to_zotero.py

Assumes the zips have already been downloaded + extracted to
``/tmp/pprrvu_dl/all/extracted/{year}{q}/``.
"""

from __future__ import annotations

import subprocess
from pathlib import Path

from conf import path as _conf_path
from zot.db import TYPE_MAP, Db, generate_key, now_iso

ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = str(_conf_path("storage.zotero"))

URL_TSV = Path("/tmp/rvu_urls_final.txt")
EXTRACTED_ROOT = Path("/tmp/pprrvu_dl/all/extracted")

QUARTER_LABEL = {"a": "Q1", "b": "Q2", "c": "Q3", "d": "Q4"}


def _load_url_map() -> dict[int, dict[str, str]]:
    """Parse the tab-separated URL list into {year: {quarter: url}}."""
    urls: dict[int, dict[str, str]] = {}
    for line in URL_TSV.read_text().splitlines():
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        parts = line.split("\t")
        if len(parts) != 3:
            continue
        year, q, url = parts
        urls.setdefault(int(year), {})[q] = url
    return urls


def add_quarter(db: Db, year: int, q: str, url: str) -> int:
    """Create one parent item per (year, quarter).

    Unambiguous ``release:q1..q4`` tag on the parent is what the
    pipe loader uses to dedupe to the latest quarter per year.
    """
    now = now_iso()
    parent_id = db.create_item(TYPE_MAP["webpage"], now=now)
    db.set_fields(
        parent_id,
        {
            "title": (
                f"CY {year} PFS Q{'abcd'.index(q) + 1} — PPRRVU "
                f"({QUARTER_LABEL[q]} release)"
            ),
            "date": f"{year}-01-01",
            "url": url,
            "accessDate": now,
            "websiteType": "Government Data Portal",
            "websiteTitle": "Centers for Medicare & Medicaid Services",
        },
    )
    db.sync_tags(
        parent_id,
        [
            "module:pfs",
            "file:rvu",
            "source:cms-website",
            f"year:{year}",
            f"release:{QUARTER_LABEL[q].lower()}",
        ],
    )

    extracted_dir = EXTRACTED_ROOT / f"{year}{q}"
    if not extracted_dir.exists():
        print(f"  {year} {QUARTER_LABEL[q]}: no extracted dir")
        return parent_id

    attached = 0
    # Walk recursively; older zips nest files under a subfolder.
    for filepath in sorted(extracted_dir.rglob("*")):
        if not filepath.is_file():
            continue
        fn = filepath.name
        if "PPRRVU" not in fn.upper():
            continue
        if filepath.suffix.lower() not in (".xlsx", ".csv", ".txt"):
            continue

        att_key = generate_key()
        storage_dir = Path(ZOTERO_STORAGE) / att_key
        subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
        dest = storage_dir / fn
        subprocess.run(["sudo", "cp", str(filepath), str(dest)], check=True)
        subprocess.run(
            ["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
            check=True,
        )

        ext = filepath.suffix.lower()
        content_type = {
            ".xlsx": (
                "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
            ),
            ".csv": "text/csv",
            ".txt": "text/plain",
        }[ext]

        att_id = db.add_attachment(
            parent_id,
            key=att_key,
            content_type=content_type,
            path=f"storage:{fn}",
        )
        db.set_field(att_id, "title", fn)
        attached += 1

    print(f"  {year} {QUARTER_LABEL[q]}: parent {parent_id}, {attached} files")
    return parent_id


def main() -> None:
    urls = _load_url_map()
    if not urls:
        raise SystemExit(f"No URLs parsed from {URL_TSV}")

    if not EXTRACTED_ROOT.exists():
        raise SystemExit(
            f"Extracted dir {EXTRACTED_ROOT} not found — download+unzip the zips first."
        )

    with Db(ZOTERO_DB) as db:
        for year in sorted(urls):
            for q in sorted(urls[year]):
                add_quarter(db, year, q, urls[year][q])
        db.commit()

    print("\nDone.  Restart Zotero + re-run pfs ingestion.")


if __name__ == "__main__":
    main()