homelab/stack
1
0
Fork 0
Code Issues 20 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

wire stack.toml into dev scripts and notebooks refs #1

Browse Source 
Replace hardcoded paths in all dev/scripts/ (register_seeds,
add_carrier, add_zipcode, migrate_zotero, generate_models,
generate_expressions, generate_zot_tables, quality measure scripts)
and notebooks (acodb_explorer, bib_explorer, cms_quality_measures,
pfs_calcs) with conf.path() lookups.  Eliminates all hardcoded
absolute /home/kert/ paths from notebooks.
This commit is contained in:
kert
2026-03-12 14:17:13 -04:00
parent 560df768d0
commit 16131e46a5
13 changed files with 844 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
dev/scripts/add_carrier_to_zotero.py
View File

@@ -17,8 +17,10 @@ import zipfile
from datetime import datetime, timezone
from pathlib import Path
ZOTERO_DB = "zotero/data/zotero.sqlite"
ZOTERO_STORAGE = "zotero/data/storage"
from conf import path as _conf_path
ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
# Files already downloaded to /tmp
CARRIER_FILES = {

6
dev/scripts/add_zipcode_to_zotero.py
View File

@@ -15,8 +15,10 @@ import zipfile
from datetime import datetime, timezone
from pathlib import Path
ZOTERO_DB = "zotero/data/zotero.sqlite"
ZOTERO_STORAGE = "zotero/data/storage"
from conf import path as _conf_path
ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = str(_conf_path("storage.zotero"))
ZIPCODE_FILES = {
2016: {

10
dev/scripts/generate_expressions.py
View File

@@ -1114,10 +1114,16 @@ def main() -> None:
parser = argparse.ArgumentParser(
description="Generate narwhals expression modules from DuckDB views"
)
from conf import path as _conf_path
parser.add_argument(
"--db", default="../notebooks/aco.duckdb", help="Path to DuckDB file"
"--db",
default=str(_conf_path("db.aco")),
help="Path to DuckDB file",
)
parser.add_argument(
"--out", default="../src/aco/express", help="Output directory"
)
parser.add_argument("--out", default="../src/aco/express", help="Output directory")
args = parser.parse_args()
db_path = Path(args.db)

14
dev/scripts/generate_models.py
View File

@@ -209,13 +209,21 @@ def generate_schema_module(
def main() -> None:
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
from conf import cfg, path as _conf_path
parser.add_argument(
"--db", default="../notebooks/aco.duckdb", help="Path to DuckDB file"
"--db",
default=str(_conf_path("db.aco")),
help="Path to DuckDB file",
)
parser.add_argument(
"--out",
default=str(_conf_path("generate.table_out")),
help="Output directory",
)
parser.add_argument("--out", default="../src/aco/table", help="Output directory")
parser.add_argument(
"--base-import",
default="aco.table.base",
default=cfg.generate.base_import,
help="Import path for the SQLTable base class",
)
args = parser.parse_args()

6
dev/scripts/generate_quality_measure_docs.py
View File

@@ -69,9 +69,11 @@ import pdfplumber
# ── Path constants ────────────────────────────────────────────────────────────
_ROOT = Path(__file__).resolve().parents[2]
from conf import ROOT as _ROOT
from conf import path as _conf_path
_SEEDS = _ROOT / "dev" / "seeds"
_ZOTERO_DEFAULT = _ROOT / "zotero" / "data" / "storage"
_ZOTERO_DEFAULT = _conf_path("storage.zotero")
_TABLE_PATH = _ROOT / "src" / "aco" / "table" / "cms_quality_measures.py"
_EXPRESS_PATH = _ROOT / "src" / "aco" / "express" / "cms_quality_measures.py"

4
dev/scripts/generate_zot_tables.py
View File

@@ -172,9 +172,11 @@ def main() -> None:
parser = argparse.ArgumentParser(
description="Generate Pydantic models from Zotero SQLite"
)
from conf import path as _conf_path
parser.add_argument(
"--db",
default="../zotero/data/zotero.sqlite",
default=str(_conf_path("db.zotero")),
help="Path to Zotero SQLite file",
)
parser.add_argument(

6
dev/scripts/migrate_zotero.py
View File

@@ -465,14 +465,16 @@ def main() -> None:
parser = argparse.ArgumentParser(
description="Migrate Zotero SQLite to bib SQLite"
)
from conf import path as _conf_path
parser.add_argument(
"--src",
default="zotero/data/zotero.sqlite",
default=str(_conf_path("db.zotero")),
help="Path to Zotero SQLite database",
)
parser.add_argument(
"--dst",
default="data/bib.sqlite",
default=str(_conf_path("db.bib")),
help="Path to output bib SQLite database",
)
args = parser.parse_args()

87
dev/scripts/populate_quality_measure_value_sets.py
View File

@@ -61,10 +61,12 @@ import polars as pl
# ── Path constants ────────────────────────────────────────────────────────────
_ROOT = Path(__file__).resolve().parents[2]
from conf import ROOT as _ROOT
from conf import path as _conf_path
_SEEDS = _ROOT / "dev" / "seeds"
_ZOTERO_DEFAULT = _ROOT / "zotero" / "data" / "storage"
_DB_DEFAULT = _ROOT / "notebooks" / "aco.duckdb"
_ZOTERO_DEFAULT = _conf_path("storage.zotero")
_DB_DEFAULT = _conf_path("db.aco")
# ── Value set spec ─────────────────────────────────────────────────────────────
@@ -666,6 +668,36 @@ def _parse_sheet(
# Add performance_year column
df = df.with_columns(pl.lit(performance_year).alias("performance_year"))
# ── Data cleanup ──────────────────────────────────────────────────────
# CMS Excel workbooks contain artifacts that are not actual value set data.
# 1. Drop "end of worksheet" sentinel rows (present in some PY2026 sheets).
text_cols = [c for c in df.columns if df[c].dtype == pl.Utf8]
if text_cols:
sentinel_mask = pl.lit(False)
for c in text_cols:
sentinel_mask = sentinel_mask | pl.col(c).str.to_lowercase().str.contains(
"end of worksheet"
)
df = df.filter(~sentinel_mask)
# 2. Drop section header rows where all key columns are null.
# Excel workbooks use ALL CAPS group headers (e.g. "ACUTE MYOCARDIAL
# INFARCTION") as visual separators — they have no code values.
key_cols_present = [kc for kc in spec.key_cols if kc in df.columns]
if key_cols_present:
all_keys_null = pl.lit(True)
for kc in key_cols_present:
all_keys_null = all_keys_null & pl.col(kc).is_null()
df = df.filter(~all_keys_null)
# 3. Normalise ICD-10 codes: strip dots for consistency across years.
# PY2024 workbooks use dotted format (I21.01), PY2025+ use undotted
# (I2101). Undotted is the canonical CMS billing format.
for col in df.columns:
if col in ("icd_10_cm", "icd_10_pcs"):
df = df.with_columns(pl.col(col).str.replace_all(r"\.", ""))
return df
@@ -724,9 +756,7 @@ def _delete_year(con: Any, table_name: str, year: int) -> int:
).fetchone()
n = result[0] if result else 0
if n > 0:
con.execute(
f"DELETE FROM {qualified} WHERE performance_year = {year}"
)
con.execute(f"DELETE FROM {qualified} WHERE performance_year = {year}")
return n
except Exception:
return 0
@@ -742,9 +772,7 @@ def _insert_df(con: Any, table_name: str, df: "pl.DataFrame") -> int:
df = df.with_columns(pl.col("performance_year").cast(pl.Int64))
col_list = ", ".join(f'"{c}"' for c in df.columns)
con.execute(
f"INSERT INTO {qualified} ({col_list}) SELECT {col_list} FROM df"
)
con.execute(f"INSERT INTO {qualified} ({col_list}) SELECT {col_list} FROM df")
return len(df)
@@ -778,9 +806,7 @@ def _write_sheet(
# Ensure performance_year is present
if "performance_year" not in df.columns:
df = df.with_columns(
pl.lit(performance_year).alias("performance_year")
)
df = df.with_columns(pl.lit(performance_year).alias("performance_year"))
_ensure_table(con, spec.table_name, model_cols)
@@ -824,9 +850,7 @@ def _load_workbook(
)
return {}
print(
f" 📖 {mif.program} {mif.measure} PY{mif.year}: {xlsx_path.name}"
)
print(f" 📖 {mif.program} {mif.measure} PY{mif.year}: {xlsx_path.name}")
wb = openpyxl.load_workbook(str(xlsx_path), read_only=True, data_only=True)
available_sheets = wb.sheetnames
@@ -853,13 +877,8 @@ def _load_workbook(
)
results[sheet_spec.table_name] = len(df)
else:
rows = _write_sheet(
con, df, sheet_spec, mif.year, mode=mode
)
print(
f" ✓ {matched} → {sheet_spec.table_name}: "
f"{rows:,} rows"
)
rows = _write_sheet(con, df, sheet_spec, mif.year, mode=mode)
print(f" ✓ {matched} → {sheet_spec.table_name}: {rows:,} rows")
results[sheet_spec.table_name] = rows
wb.close()
@@ -902,9 +921,7 @@ def _diff_table(
schema, tbl = table_name.split(".", 1)
qualified = f'"{schema}"."{tbl}"'
key_expr = " || '|' || ".join(
f'COALESCE("{k}", \'\')' for k in key_cols
)
key_expr = " || '|' || ".join(f"COALESCE(\"{k}\", '')" for k in key_cols)
try:
rows_a = con.execute(
@@ -957,9 +974,7 @@ def _compute_diffs(con: Any) -> list[DiffRow]:
except Exception:
continue
for i in range(len(years) - 1):
row = _diff_table(
con, table_name, key_cols, years[i], years[i + 1]
)
row = _diff_table(con, table_name, key_cols, years[i], years[i + 1])
if row is not None:
diffs.append(row)
@@ -969,10 +984,7 @@ def _compute_diffs(con: Any) -> list[DiffRow]:
def _print_diffs(diffs: list[DiffRow]) -> None:
"""Pretty-print the diff summary table."""
if not diffs:
print(
"\n No year-over-year diffs available "
"(need ≥2 years loaded).\n"
)
print("\n No year-over-year diffs available (need ≥2 years loaded).\n")
return
print("\n Year-over-year value set diffs:")
@@ -983,11 +995,7 @@ def _print_diffs(diffs: list[DiffRow]) -> None:
)
print(" " + "─" * 100)
for d in diffs:
short = (
d.table_name.split(".", 1)[1]
if "." in d.table_name
else d.table_name
)
short = d.table_name.split(".", 1)[1] if "." in d.table_name else d.table_name
print(
f" {short:<50} {d.year_a}→{d.year_b} "
f"{d.codes_in_a:>6} {d.codes_in_b:>6} "
@@ -1116,10 +1124,7 @@ def main() -> None:
total_tables += 1
total_rows += rows
print(
f"\n Done: {total_rows:,} total rows across "
f"{total_tables} table loads.\n"
)
print(f"\n Done: {total_rows:,} total rows across {total_tables} table loads.\n")
# Year-over-year diffs
if args.diff and con is not None:

8
dev/scripts/register_seeds.py
View File

@@ -23,10 +23,12 @@ from dataclasses import dataclass, field
from datetime import datetime, timezone
from pathlib import Path
from conf import path as _conf_path
SEEDS_DIR = Path("dev/seeds")
BIB_DB = "data/bib.sqlite"
ZOTERO_DB = "zotero/data/zotero.sqlite"
ZOTERO_STORAGE = Path("zotero/data/storage")
BIB_DB = str(_conf_path("db.bib"))
ZOTERO_DB = str(_conf_path("db.zotero"))
ZOTERO_STORAGE = _conf_path("storage.zotero")
# Zotero schema constants (match add_carrier_to_zotero.py / sync.py)
ZOTERO_TYPE_WEBPAGE = 40

4
notebooks/acodb_explorer.py
View File

@@ -31,7 +31,9 @@ def _():
import duckdb
import polars as pl
DB_PATH = "/home/kert/notebooks/aco.duckdb"
from conf import path as _conf_path
DB_PATH = str(_conf_path("db.aco"))
con = duckdb.connect(DB_PATH, read_only=True)
def q(sql):

6
notebooks/bib_explorer.py
View File

@@ -36,8 +36,10 @@ def _(mo):
from bib.store import Store
from bib.tag import Tag, filter_tags
BIB_DB = Path("/home/kert/data/bib.sqlite")
ZOTERO_DB = Path("/home/kert/zotero/zotero.sqlite")
from conf import path as _conf_path
BIB_DB = _conf_path("db.bib")
ZOTERO_DB = _conf_path("db.zotero")
store = Store(str(BIB_DB))

998
notebooks/cms_quality_measures.py
View File

File diff suppressed because it is too large Load Diff

8
notebooks/pfs_calcs.py
View File

@@ -28,11 +28,9 @@ def _():
import polars as pl
from pathlib import Path
_candidates = [
Path("/home/kert/notebooks/aco.duckdb"), # container
Path(__file__).parent / "aco.duckdb", # local
]
DB_PATH = str(next(p for p in _candidates if p.exists()))
from conf import path as _conf_path
DB_PATH = str(_conf_path("db.aco"))
con = duckdb.connect(DB_PATH, read_only=True)
def q(sql):