wire stack.toml into dev scripts and notebooks refs #1

Replace hardcoded paths in all dev/scripts/ (register_seeds,
add_carrier, add_zipcode, migrate_zotero, generate_models,
generate_expressions, generate_zot_tables, quality measure scripts)
and notebooks (acodb_explorer, bib_explorer, cms_quality_measures,
pfs_calcs) with conf.path() lookups.  Eliminates all hardcoded
absolute /home/kert/ paths from notebooks.

dev/scripts/add_carrier_to_zotero.py

@@ -17,8 +17,10 @@ import zipfile
 from datetime import datetime, timezone
 from pathlib import Path
 
-ZOTERO_DB = "zotero/data/zotero.sqlite"
+from conf import path as _conf_path
-ZOTERO_STORAGE = "zotero/data/storage"
+
+ZOTERO_DB = str(_conf_path("db.zotero"))
+ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
 
 # Files already downloaded to /tmp
 CARRIER_FILES = {

dev/scripts/add_zipcode_to_zotero.py

@@ -15,8 +15,10 @@ import zipfile
 from datetime import datetime, timezone
 from pathlib import Path
 
-ZOTERO_DB = "zotero/data/zotero.sqlite"
+from conf import path as _conf_path
-ZOTERO_STORAGE = "zotero/data/storage"
+
+ZOTERO_DB = str(_conf_path("db.zotero"))
+ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
 
 ZIPCODE_FILES = {
     2016: {

dev/scripts/generate_expressions.py

@@ -1114,10 +1114,16 @@ def main() -> None:
     parser = argparse.ArgumentParser(
         description="Generate narwhals expression modules from DuckDB views"
     )
+    from conf import path as _conf_path
+
     parser.add_argument(
-        "--db", default="../notebooks/aco.duckdb", help="Path to DuckDB file"
+        "--db",
+        default=str(_conf_path("db.aco")),
+        help="Path to DuckDB file",
+    )
+    parser.add_argument(
+        "--out", default="../src/aco/express", help="Output directory"
     )
-    parser.add_argument("--out", default="../src/aco/express", help="Output directory")
     args = parser.parse_args()
 
     db_path = Path(args.db)

dev/scripts/generate_models.py

@@ -209,13 +209,21 @@ def generate_schema_module(
 
 def main() -> None:
     parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
+    from conf import cfg, path as _conf_path
+
     parser.add_argument(
-        "--db", default="../notebooks/aco.duckdb", help="Path to DuckDB file"
+        "--db",
+        default=str(_conf_path("db.aco")),
+        help="Path to DuckDB file",
+    )
+    parser.add_argument(
+        "--out",
+        default=str(_conf_path("generate.table_out")),
+        help="Output directory",
     )
-    parser.add_argument("--out", default="../src/aco/table", help="Output directory")
     parser.add_argument(
         "--base-import",
-        default="aco.table.base",
+        default=cfg.generate.base_import,
         help="Import path for the SQLTable base class",
     )
     args = parser.parse_args()

dev/scripts/generate_quality_measure_docs.py

@@ -69,9 +69,11 @@ import pdfplumber
 
 # ── Path constants ────────────────────────────────────────────────────────────
 
-_ROOT = Path(__file__).resolve().parents[2]
+from conf import ROOT as _ROOT
+from conf import path as _conf_path
+
 _SEEDS = _ROOT / "dev" / "seeds"
-_ZOTERO_DEFAULT = _ROOT / "zotero" / "data" / "storage"
+_ZOTERO_DEFAULT = _conf_path("storage.zotero")
 
 _TABLE_PATH = _ROOT / "src" / "aco" / "table" / "cms_quality_measures.py"
 _EXPRESS_PATH = _ROOT / "src" / "aco" / "express" / "cms_quality_measures.py"

dev/scripts/generate_zot_tables.py

@@ -172,9 +172,11 @@ def main() -> None:
     parser = argparse.ArgumentParser(
         description="Generate Pydantic models from Zotero SQLite"
     )
+    from conf import path as _conf_path
+
     parser.add_argument(
         "--db",
-        default="../zotero/data/zotero.sqlite",
+        default=str(_conf_path("db.zotero")),
         help="Path to Zotero SQLite file",
     )
     parser.add_argument(

dev/scripts/migrate_zotero.py

@@ -465,14 +465,16 @@ def main() -> None:
     parser = argparse.ArgumentParser(
         description="Migrate Zotero SQLite to bib SQLite"
     )
+    from conf import path as _conf_path
+
     parser.add_argument(
         "--src",
-        default="zotero/data/zotero.sqlite",
+        default=str(_conf_path("db.zotero")),
         help="Path to Zotero SQLite database",
     )
     parser.add_argument(
         "--dst",
-        default="data/bib.sqlite",
+        default=str(_conf_path("db.bib")),
         help="Path to output bib SQLite database",
     )
     args = parser.parse_args()

dev/scripts/populate_quality_measure_value_sets.py

@@ -61,10 +61,12 @@ import polars as pl
 
 # ── Path constants ────────────────────────────────────────────────────────────
 
-_ROOT = Path(__file__).resolve().parents[2]
+from conf import ROOT as _ROOT
+from conf import path as _conf_path
+
 _SEEDS = _ROOT / "dev" / "seeds"
-_ZOTERO_DEFAULT = _ROOT / "zotero" / "data" / "storage"
+_ZOTERO_DEFAULT = _conf_path("storage.zotero")
-_DB_DEFAULT = _ROOT / "notebooks" / "aco.duckdb"
+_DB_DEFAULT = _conf_path("db.aco")
 
 # ── Value set spec ─────────────────────────────────────────────────────────────
 
@@ -666,6 +668,36 @@ def _parse_sheet(
     # Add performance_year column
     df = df.with_columns(pl.lit(performance_year).alias("performance_year"))
 
+    # ── Data cleanup ──────────────────────────────────────────────────────
+    # CMS Excel workbooks contain artifacts that are not actual value set data.
+
+    # 1. Drop "end of worksheet" sentinel rows (present in some PY2026 sheets).
+    text_cols = [c for c in df.columns if df[c].dtype == pl.Utf8]
+    if text_cols:
+        sentinel_mask = pl.lit(False)
+        for c in text_cols:
+            sentinel_mask = sentinel_mask | pl.col(c).str.to_lowercase().str.contains(
+                "end of worksheet"
+            )
+        df = df.filter(~sentinel_mask)
+
+    # 2. Drop section header rows where all key columns are null.
+    #    Excel workbooks use ALL CAPS group headers (e.g. "ACUTE MYOCARDIAL
+    #    INFARCTION") as visual separators — they have no code values.
+    key_cols_present = [kc for kc in spec.key_cols if kc in df.columns]
+    if key_cols_present:
+        all_keys_null = pl.lit(True)
+        for kc in key_cols_present:
+            all_keys_null = all_keys_null & pl.col(kc).is_null()
+        df = df.filter(~all_keys_null)
+
+    # 3. Normalise ICD-10 codes: strip dots for consistency across years.
+    #    PY2024 workbooks use dotted format (I21.01), PY2025+ use undotted
+    #    (I2101).  Undotted is the canonical CMS billing format.
+    for col in df.columns:
+        if col in ("icd_10_cm", "icd_10_pcs"):
+            df = df.with_columns(pl.col(col).str.replace_all(r"\.", ""))
+
     return df
 
 
@@ -724,9 +756,7 @@ def _delete_year(con: Any, table_name: str, year: int) -> int:
         ).fetchone()
         n = result[0] if result else 0
         if n > 0:
-            con.execute(
+            con.execute(f"DELETE FROM {qualified} WHERE performance_year = {year}")
-                f"DELETE FROM {qualified} WHERE performance_year = {year}"
-            )
         return n
     except Exception:
         return 0
@@ -742,9 +772,7 @@ def _insert_df(con: Any, table_name: str, df: "pl.DataFrame") -> int:
         df = df.with_columns(pl.col("performance_year").cast(pl.Int64))
 
     col_list = ", ".join(f'"{c}"' for c in df.columns)
-    con.execute(
+    con.execute(f"INSERT INTO {qualified} ({col_list}) SELECT {col_list} FROM df")
-        f"INSERT INTO {qualified} ({col_list}) SELECT {col_list} FROM df"
-    )
     return len(df)
 
 
@@ -778,9 +806,7 @@ def _write_sheet(
 
     # Ensure performance_year is present
     if "performance_year" not in df.columns:
-        df = df.with_columns(
+        df = df.with_columns(pl.lit(performance_year).alias("performance_year"))
-            pl.lit(performance_year).alias("performance_year")
-        )
 
     _ensure_table(con, spec.table_name, model_cols)
 
@@ -824,9 +850,7 @@ def _load_workbook(
         )
         return {}
 
-    print(
+    print(f"  📖 {mif.program} {mif.measure} PY{mif.year}: {xlsx_path.name}")
-        f"  📖 {mif.program} {mif.measure} PY{mif.year}: {xlsx_path.name}"
-    )
 
     wb = openpyxl.load_workbook(str(xlsx_path), read_only=True, data_only=True)
     available_sheets = wb.sheetnames
@@ -853,13 +877,8 @@ def _load_workbook(
             )
             results[sheet_spec.table_name] = len(df)
         else:
-            rows = _write_sheet(
+            rows = _write_sheet(con, df, sheet_spec, mif.year, mode=mode)
-                con, df, sheet_spec, mif.year, mode=mode
+            print(f"    ✓  {matched} → {sheet_spec.table_name}: {rows:,} rows")
-            )
-            print(
-                f"    ✓  {matched} → {sheet_spec.table_name}: "
-                f"{rows:,} rows"
-            )
             results[sheet_spec.table_name] = rows
 
     wb.close()
@@ -902,9 +921,7 @@ def _diff_table(
     schema, tbl = table_name.split(".", 1)
     qualified = f'"{schema}"."{tbl}"'
 
-    key_expr = " || '|' || ".join(
+    key_expr = " || '|' || ".join(f"COALESCE(\"{k}\", '')" for k in key_cols)
-        f'COALESCE("{k}", \'\')' for k in key_cols
-    )
 
     try:
         rows_a = con.execute(
@@ -957,9 +974,7 @@ def _compute_diffs(con: Any) -> list[DiffRow]:
         except Exception:
             continue
         for i in range(len(years) - 1):
-            row = _diff_table(
+            row = _diff_table(con, table_name, key_cols, years[i], years[i + 1])
-                con, table_name, key_cols, years[i], years[i + 1]
-            )
             if row is not None:
                 diffs.append(row)
 
@@ -969,10 +984,7 @@ def _compute_diffs(con: Any) -> list[DiffRow]:
 def _print_diffs(diffs: list[DiffRow]) -> None:
     """Pretty-print the diff summary table."""
     if not diffs:
-        print(
+        print("\n  No year-over-year diffs available (need ≥2 years loaded).\n")
-            "\n  No year-over-year diffs available "
-            "(need ≥2 years loaded).\n"
-        )
         return
 
     print("\n  Year-over-year value set diffs:")
@@ -983,11 +995,7 @@ def _print_diffs(diffs: list[DiffRow]) -> None:
     )
     print("  " + "─" * 100)
     for d in diffs:
-        short = (
+        short = d.table_name.split(".", 1)[1] if "." in d.table_name else d.table_name
-            d.table_name.split(".", 1)[1]
-            if "." in d.table_name
-            else d.table_name
-        )
         print(
             f"  {short:<50} {d.year_a}→{d.year_b}  "
             f"{d.codes_in_a:>6} {d.codes_in_b:>6} "
@@ -1116,10 +1124,7 @@ def main() -> None:
             total_tables += 1
             total_rows += rows
 
-    print(
+    print(f"\n  Done: {total_rows:,} total rows across {total_tables} table loads.\n")
-        f"\n  Done: {total_rows:,} total rows across "
-        f"{total_tables} table loads.\n"
-    )
 
     # Year-over-year diffs
     if args.diff and con is not None:

dev/scripts/register_seeds.py

@@ -23,10 +23,12 @@ from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from pathlib import Path
 
+from conf import path as _conf_path
+
 SEEDS_DIR = Path("dev/seeds")
-BIB_DB = "data/bib.sqlite"
+BIB_DB = str(_conf_path("db.bib"))
-ZOTERO_DB = "zotero/data/zotero.sqlite"
+ZOTERO_DB = str(_conf_path("db.zotero"))
-ZOTERO_STORAGE = Path("zotero/data/storage")
+ZOTERO_STORAGE = _conf_path("storage.zotero")
 
 # Zotero schema constants (match add_carrier_to_zotero.py / sync.py)
 ZOTERO_TYPE_WEBPAGE = 40

notebooks/acodb_explorer.py

@@ -31,7 +31,9 @@ def _():
     import duckdb
     import polars as pl
 
-    DB_PATH = "/home/kert/notebooks/aco.duckdb"
+    from conf import path as _conf_path
+
+    DB_PATH = str(_conf_path("db.aco"))
     con = duckdb.connect(DB_PATH, read_only=True)
 
     def q(sql):

notebooks/bib_explorer.py

@@ -36,8 +36,10 @@ def _(mo):
     from bib.store import Store
     from bib.tag import Tag, filter_tags
 
-    BIB_DB = Path("/home/kert/data/bib.sqlite")
+    from conf import path as _conf_path
-    ZOTERO_DB = Path("/home/kert/zotero/zotero.sqlite")
+
+    BIB_DB = _conf_path("db.bib")
+    ZOTERO_DB = _conf_path("db.zotero")
 
     store = Store(str(BIB_DB))
 

notebooks/pfs_calcs.py

@@ -28,11 +28,9 @@ def _():
     import polars as pl
     from pathlib import Path
 
-    _candidates = [
+    from conf import path as _conf_path
-        Path("/home/kert/notebooks/aco.duckdb"),  # container
+
-        Path(__file__).parent / "aco.duckdb",       # local
+    DB_PATH = str(_conf_path("db.aco"))
-    ]
-    DB_PATH = str(next(p for p in _candidates if p.exists()))
     con = duckdb.connect(DB_PATH, read_only=True)
 
     def q(sql):