chore: clean sweep — lint, format, stale refs, generated artifacts
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled

- Fix all 72 ruff lint errors (unused imports, unused variables, E402)
- Format all 14 unformatted dev/scripts files
- Move generated artifacts to assets/ (dag.html, pfs.html)
- Remove duplicate root coverage.svg (already in assets/icons/)
- Update .dockerignore for infra/ tree layout
- Update .gitignore: add .env.bak, mirrors/, htmlcov/
- Fix stale path refs in coverage_badge.py, woodpecker backend,
  test_network_isolation.sh, docs custom.css
- Add .gitkeep to empty dirs (infra/polaris, cloud/*/terraform)
- Delete 12 stale local branches, 10 stale remote branches
This commit is contained in:
kert
2026-03-24 17:33:55 -04:00
parent 29b302cdf0
commit 85ce5e719d
38 changed files with 340 additions and 304 deletions

View File

@@ -12,13 +12,8 @@ logs/
tuva/
dev/
tests/
grafana/
prometheus/
loki/
trino/
traefik/
nginx/
gitea/
styles/
.woodpecker/
infra/
assets/
cloud/
mirrors/
.claude/

13
.gitignore vendored
View File

@@ -13,17 +13,28 @@ bundle/sql/
# Marimo cache
notebooks/__marimo__/
# Secrets and keys
.deploy_key
.env
.deploy_key.pub
.env
.env.bak
# Python build artifacts
__pycache__/
*.pyc
.coverage
dist/
*.egg-info/
htmlcov/
# IDE / tool settings
.claude/settings.local.json
notebooks/aco.duckdb
# Package mirrors (local cache, not tracked)
mirrors/
# Docs (auto-generated at build time)
docs/docs/api/
docs/static/library.json

View File

View File

View File

View File

@@ -1,20 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="116" height="20">
<linearGradient id="b" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
</linearGradient>
<clipPath id="a">
<rect width="116" height="20" rx="3" fill="#fff"/>
</clipPath>
<g clip-path="url(#a)">
<path fill="#555" d="M0 0h65v20H0z"/>
<path fill="#4c1" d="M65 0h51v20H65z"/>
<path fill="url(#b)" d="M0 0h116v20H0z"/>
</g>
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
<text x="32.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
<text x="32.5" y="14">coverage</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">99%</text>
<text x="90" y="14">99%</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 822 B

View File

@@ -7,11 +7,8 @@ Usage:
uv run python dev/scripts/add_carrier_to_zotero.py
"""
import os
import random
import shutil
import sqlite3
import string
import subprocess
import zipfile
from datetime import datetime, timezone

View File

@@ -9,7 +9,6 @@ Usage:
import random
import sqlite3
import string
import subprocess
import zipfile
from datetime import datetime, timezone

View File

@@ -30,8 +30,11 @@ yaml.add_representer(
def _dump(data: dict) -> str:
return yaml.dump(
data, default_flow_style=False, sort_keys=False,
width=120, allow_unicode=True,
data,
default_flow_style=False,
sort_keys=False,
width=120,
allow_unicode=True,
)
@@ -69,9 +72,7 @@ class SQLPlugin(DabPlugin):
output_mode="ctas",
)
except Exception as e:
result[f"bundle/sql/{pipe_name}/_error.txt"] = (
f"Transpile failed: {e}"
)
result[f"bundle/sql/{pipe_name}/_error.txt"] = f"Transpile failed: {e}"
continue
for expr_name, sql in sql_map.items():
@@ -80,8 +81,7 @@ class SQLPlugin(DabPlugin):
schema, table = expr_name.split(".", 1)
path = f"bundle/sql/{pipe_name}/{table}.sql"
header = (
f"-- {expr_name}\n"
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
f"-- {expr_name}\n-- Generated by gen_config.py — DO NOT EDIT\n\n"
)
result[path] = header + sql + "\n"
@@ -102,27 +102,58 @@ class JobsPlugin(DabPlugin):
job_clusters = []
for profile_name in sorted(used_profiles):
cluster_spec = dict(clusters_cfg.get(profile_name, {}))
job_clusters.append(OrderedDict([
job_clusters.append(
OrderedDict(
[
("job_cluster_key", f"{profile_name}_cluster"),
("new_cluster", OrderedDict([
("spark_version", cluster_spec.get("spark_version", "15.4.x-scala2.12")),
("node_type_id", cluster_spec.get("node_type_id", "i3.xlarge")),
(
"new_cluster",
OrderedDict(
[
(
"spark_version",
cluster_spec.get(
"spark_version", "15.4.x-scala2.12"
),
),
(
"node_type_id",
cluster_spec.get("node_type_id", "i3.xlarge"),
),
("num_workers", cluster_spec.get("num_workers", 1)),
])),
]))
]
),
),
]
)
)
# DDL tasks — one per schema to create tables before SQL runs
ddl_tasks = []
for schema_name in sorted(catalog_schemas):
ddl_tasks.append(OrderedDict([
ddl_tasks.append(
OrderedDict(
[
("task_key", f"ddl_{schema_name}"),
("job_cluster_key", "default_cluster"),
("sql_task", OrderedDict([
("file", OrderedDict([
(
"sql_task",
OrderedDict(
[
(
"file",
OrderedDict(
[
("path", f"bundle/ddl/{schema_name}/"),
])),
])),
]))
]
),
),
]
),
),
]
)
)
# Pipeline tasks — each expression becomes a sql_task
pipeline_tasks = []
@@ -143,12 +174,19 @@ class JobsPlugin(DabPlugin):
# Each pipeline task runs all its SQL files in order
# We use a for_each or sequential notebook; simplest is
# a single SQL file per pipeline that sources all expressions
task["sql_task"] = OrderedDict([
("file", OrderedDict([
task["sql_task"] = OrderedDict(
[
(
"file",
OrderedDict(
[
("path", f"bundle/sql/{pipe_name}/"),
])),
]
),
),
("warehouse_id", "${var.warehouse_id}"),
])
]
)
pipeline_tasks.append(task)
schedule = cfg.get("schedule", "0 0 6 * * ?")
@@ -156,11 +194,15 @@ class JobsPlugin(DabPlugin):
job = OrderedDict()
job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines"
job["description"] = f"Run all {len(registry)} ACO pipelines in dependency order"
job["schedule"] = OrderedDict([
job["description"] = (
f"Run all {len(registry)} ACO pipelines in dependency order"
)
job["schedule"] = OrderedDict(
[
("quartz_cron_expression", schedule),
("timezone_id", timezone),
])
]
)
job["job_clusters"] = job_clusters
job["tasks"] = pipeline_tasks
@@ -175,10 +217,12 @@ class SchemasPlugin(DabPlugin):
return {}
schemas = OrderedDict()
for schema_name in sorted(catalog_schemas):
schemas[schema_name] = OrderedDict([
schemas[schema_name] = OrderedDict(
[
("name", schema_name),
("catalog_name", "${var.catalog}"),
])
]
)
return {"schemas": schemas}
@@ -186,14 +230,23 @@ class VolumesPlugin(DabPlugin):
"""Generate a staging volume for data upload."""
def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict:
return {"volumes": OrderedDict([
("staging", OrderedDict([
return {
"volumes": OrderedDict(
[
(
"staging",
OrderedDict(
[
("name", "staging"),
("catalog_name", "${var.catalog}"),
("schema_name", "default"),
("volume_type", "MANAGED"),
])),
])}
]
),
),
]
)
}
# ── Core generator ───────────────────────────────────────────────
@@ -214,6 +267,7 @@ _DEFAULT_PLUGINS: list[DabPlugin] = [
def _get_catalog_schemas() -> list[str]:
try:
from aco.lake.catalog import Catalog
return Catalog().schemas()
except Exception:
return []
@@ -252,19 +306,33 @@ def emit(cfg_data: dict) -> dict[str, str]:
bundle["bundle"] = OrderedDict([("name", bundle_name)])
# Sync SQL files to workspace
bundle["sync"] = OrderedDict([
bundle["sync"] = OrderedDict(
[
("include", ["bundle/**"]),
])
]
)
bundle["variables"] = OrderedDict([
("catalog", OrderedDict([
bundle["variables"] = OrderedDict(
[
(
"catalog",
OrderedDict(
[
("description", "Unity Catalog name for table references"),
("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
])),
("warehouse_id", OrderedDict([
]
),
),
(
"warehouse_id",
OrderedDict(
[
("description", "SQL warehouse ID for query execution"),
])),
])
]
),
),
]
)
bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")])
@@ -276,17 +344,21 @@ def emit(cfg_data: dict) -> dict[str, str]:
target["mode"] = tcfg["mode"]
if tcfg.get("default"):
target["default"] = True
target["variables"] = OrderedDict([
target["variables"] = OrderedDict(
[
("catalog", tcfg.get("catalog", "aco")),
])
]
)
if tcfg.get("warehouse_id"):
target["variables"]["warehouse_id"] = tcfg["warehouse_id"]
if tcfg.get("mode") == "production":
run_as = dab_cfg.get("run_as", {})
if run_as.get("service_principal_name"):
target["run_as"] = OrderedDict([
target["run_as"] = OrderedDict(
[
("service_principal_name", run_as["service_principal_name"]),
])
]
)
targets[tname] = target
bundle["targets"] = targets

View File

@@ -242,7 +242,7 @@ steps:
UV_PROJECT_ENVIRONMENT: .venv
commands:
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg
- uv run python dev/scripts/coverage_badge.py < pytest.out > assets/icons/coverage.svg
depends_on:
- lint
@@ -273,7 +273,7 @@ steps:
commands:
- git config user.name woodpecker-ci
- git config user.email ci@${{CI_REPO_OWNER}}.io
- git add coverage.svg
- git add assets/icons/coverage.svg
- git diff --cached --quiet && echo "no change" && exit 0
- git commit -m "update coverage badge [skip ci]"
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
@@ -293,7 +293,7 @@ steps:
from_secret: s3_access_key
secret_key:
from_secret: s3_secret_key
source: "coverage.svg"
source: "assets/icons/coverage.svg"
target: /badges/${{CI_REPO}}/
path_style: true
overwrite: true

View File

@@ -15,7 +15,6 @@ import argparse
import os
import shutil
import subprocess
import sys
from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2]
@@ -33,7 +32,7 @@ CA_SUBJECT = "/CN=Homelab CA/O=fhirworx"
CA_DAYS = 3650
TLS_DAYS = 3650
from conf import cfg
from conf import cfg # noqa: E402
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)

View File

@@ -8,7 +8,7 @@ Usage:
# CI_COMMIT_SHA from environment):
uv run python dev/scripts/coverage_badge.py --post pytest.out
# Update styles/coverage.svg in the repo via Gitea API:
# Update assets/icons/coverage.svg in the repo via Gitea API:
uv run python dev/scripts/coverage_badge.py --upload coverage.svg
"""
@@ -44,7 +44,7 @@ TEMPLATE = """\
</g>
</svg>"""
FILE_PATH = "styles/coverage.svg"
FILE_PATH = "assets/icons/coverage.svg"
def color_for(pct: int) -> str:
@@ -73,11 +73,13 @@ def post_status(pct: int) -> None:
token, url = _gitea_env()
sha = os.environ["CI_COMMIT_SHA"]
endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}"
body = json.dumps({
body = json.dumps(
{
"state": "success",
"description": f"{pct}% coverage",
"context": "coverage",
}).encode()
}
).encode()
req = urllib.request.Request(
endpoint,
data=body,
@@ -92,7 +94,7 @@ def post_status(pct: int) -> None:
def upload_badge(svg_path: str) -> None:
"""Update styles/coverage.svg in the repo via Gitea contents API."""
"""Update assets/icons/coverage.svg in the repo via Gitea contents API."""
token, url = _gitea_env()
endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}"
svg_data = open(svg_path, "rb").read()

View File

@@ -40,7 +40,6 @@ def main() -> int:
for col in COLUMNS:
for op in ("INSERT", "UPDATE"):
trigger = f"fix_{col}_{op.lower()}"
of_clause = f"OF {col} " if op == "UPDATE" else ""
db.execute(f"""
CREATE TRIGGER IF NOT EXISTS {trigger}
AFTER {op} ON items

View File

@@ -219,7 +219,11 @@ def generate_module() -> str:
def main():
"""Generate src/aco/table/alr_filenames.py."""
output_path = (
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "alr_filenames.py"
Path(__file__).resolve().parents[2]
/ "src"
/ "aco"
/ "table"
/ "alr_filenames.py"
)
code = generate_module()
@@ -228,9 +232,9 @@ def main():
output_path.write_text(code)
print(f"Generated {output_path}")
print(f" 18 ALR table patterns (9 annual + 9 quarterly)")
print(f" 2 ASR patterns (annual + quarterly)")
print(f" 4 package patterns")
print(" 18 ALR table patterns (9 annual + 9 quarterly)")
print(" 2 ASR patterns (annual + quarterly)")
print(" 4 package patterns")
return 0

View File

@@ -25,7 +25,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
Returns dict mapping table_id -> {title, fields: list[dict]}
"""
pdf = pdfplumber.open(pdf_path)
pdfplumber.open(pdf_path)
# Manual field specifications based on PDF extraction
# Table 1-1 is on pages 8-15, others follow
@@ -163,7 +163,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
{
"name": f"HCC_Position_{i:02d}",
"type": "int",
"desc": f"HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
"desc": "HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
}
)
@@ -530,7 +530,7 @@ def generate_module(tables: dict) -> str:
lines.append(" Source: ALRASRGuide.pdf")
lines.append(' """')
lines.append("")
lines.append(f' __schema__ = "alr"')
lines.append(' __schema__ = "alr"')
lines.append(f' __tablename__ = "{tbl}"')
for field in info["fields"]:

View File

@@ -370,7 +370,7 @@ def generate_module(tables: dict) -> str:
lines.append(" Source: ALRASRGuide.pdf")
lines.append(' """')
lines.append("")
lines.append(f' __schema__ = "asr"')
lines.append(' __schema__ = "asr"')
lines.append(f' __tablename__ = "{tbl}"')
for field in info["fields"]:
@@ -398,7 +398,9 @@ def generate_module(tables: dict) -> str:
def main():
"""Generate src/aco/table/asr.py from ALRASRGuide.pdf."""
output_path = Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
output_path = (
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
)
tables = extract_asr_table_specs()
code = generate_module(tables)

View File

@@ -14,7 +14,6 @@ Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
from __future__ import annotations
import re
from datetime import date
from pathlib import Path
import pdfplumber
@@ -206,7 +205,7 @@ def generate_module(tables: dict) -> str:
lines.append(f" {len(info['fields'])} fields, fixed-width layout.")
lines.append(' """')
lines.append("")
lines.append(f' __schema__ = "cclf"')
lines.append(' __schema__ = "cclf"')
lines.append(f' __tablename__ = "{tbl_name}"')
seen_names: set[str] = set()

View File

@@ -17,8 +17,7 @@ from __future__ import annotations
import argparse
import re
import textwrap
from dataclasses import dataclass, field
from dataclasses import dataclass
from pathlib import Path
import pdfplumber
@@ -146,7 +145,7 @@ def parse_entry(raw: str) -> Variable | None:
# Parse the remaining structured fields
# Rejoin everything from LABEL: onward
body = "\n".join(lines[label_idx:])
"\n".join(lines[label_idx:])
# Extract each field by finding the label and collecting text
# until the next label
@@ -211,9 +210,9 @@ def variable_to_py(var: Variable) -> str:
primary_name = var.name.split("\n")[0].strip()
lines = [
f'"""',
'"""',
f"Variable: {primary_name}",
f"",
"",
]
if var.label and var.label != "":
@@ -289,7 +288,9 @@ def main() -> None:
"--pdf",
default=str(
Path(__file__).resolve().parents[2]
/ "dev" / "seeds" / "codebook-ffs-claims.pdf"
/ "dev"
/ "seeds"
/ "codebook-ffs-claims.pdf"
),
help="Path to the CCW codebook PDF",
)

View File

@@ -16,11 +16,9 @@ Defaults:
from __future__ import annotations
import argparse
import re
from pathlib import Path
import fastexcel
import polars as pl
# ── Sheets and their slug names ──────────────────────────────────────────────
@@ -169,7 +167,7 @@ def generate_model(
f"class {cls}(SQLTable):",
f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""',
"",
f' __schema__ = "ccw"',
' __schema__ = "ccw"',
f' __tablename__ = "{claim_slug}_{table}"',
"",
]
@@ -214,7 +212,9 @@ def main() -> None:
"--xlsx",
default=str(
Path(__file__).resolve().parents[2]
/ "dev" / "seeds" / "record-layout-ffs-claims.xlsx"
/ "dev"
/ "seeds"
/ "record-layout-ffs-claims.xlsx"
),
help="Path to the CCW record layout Excel file",
)

View File

@@ -224,9 +224,9 @@ def generate_function(table: dict) -> str:
"@nw.narwhalify",
f"def {tablename}({param}: FrameT) -> FrameT:",
f' """Clean and type-cast cms.{tablename}.',
f"",
"",
f" {field_count} fields. Numeric columns auto-cast to Float64.",
f' """',
' """',
f" return auto_cast({param})",
]
return "\n".join(lines)
@@ -236,18 +236,18 @@ def generate_module(domain: str, tables: list[dict]) -> str:
"""Generate a full express module for a domain group."""
lines = [
f'"""CMS express — {domain} domain.',
f"",
"",
f"Auto-generated narwhals functions for {len(tables)} CMS tables.",
f"Each function takes the raw table and returns a typed frame",
f"with numeric columns cast to Float64.",
f'"""',
f"",
f"from __future__ import annotations",
f"",
f"import narwhals as nw",
f"from narwhals.typing import FrameT",
f"",
f"from cms.express._helpers import auto_cast",
"Each function takes the raw table and returns a typed frame",
"with numeric columns cast to Float64.",
'"""',
"",
"from __future__ import annotations",
"",
"import narwhals as nw",
"from narwhals.typing import FrameT",
"",
"from cms.express._helpers import auto_cast",
]
for tbl in sorted(tables, key=lambda t: t["tablename"]):

View File

@@ -14,7 +14,6 @@ import argparse
import re
from collections import defaultdict
from pathlib import Path
from textwrap import dedent, indent
import duckdb
@@ -268,7 +267,7 @@ def col_expr_to_nw(expr: str) -> str:
# date_diff('unit', a, b)
dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE)
if dd_m:
unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
_unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
if alias:
return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")'
return f"{_col_ref(b)} - {_col_ref(a)}"
@@ -712,13 +711,13 @@ def generate_function(
doc_ops = ", ".join(ops)
lines: list[str] = []
lines.append(f"@nw.narwhalify")
lines.append("@nw.narwhalify")
lines.append(f"def {func_name}({param_str}) -> FrameT:")
lines.append(f' """Build {schema}.{view_name}')
lines.append(f"")
lines.append("")
lines.append(f" Operations: {doc_ops}")
lines.append(f" Dependencies: {doc_deps}")
lines.append(f' """')
lines.append(' """')
if "passthrough" in ops:
p = params[0].split(":")[0]
@@ -880,7 +879,7 @@ def _gen_select_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
if nw_cols:
lines.append(f" return {p}.select(")
_emit_exprs(lines, nw_cols)
lines.append(f" )")
lines.append(" )")
else:
lines.append(f" return {p}")
@@ -902,13 +901,13 @@ def _gen_filter_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
lines.append(f" # {comment_part}")
lines.append(f" return {p}.filter(")
lines.append(f" {code_part}")
lines.append(f" )")
lines.append(" )")
else:
safe = clause.replace('"', "'")[:80]
lines.append(f" # TODO: complex filter: {safe}")
lines.append(f" return {p}")
else:
lines.append(f" # WHERE clause could not be parsed")
lines.append(" # WHERE clause could not be parsed")
lines.append(f" return {p}")
@@ -994,9 +993,9 @@ def _gen_join_body(
keys_str = ", ".join(f'"{k}"' for k in keys)
lines.append(f" result = result.group_by({keys_str}).agg()")
else:
lines.append(f" # TODO: complex GROUP BY")
lines.append(" # TODO: complex GROUP BY")
lines.append(f" return result")
lines.append(" return result")
def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
@@ -1023,7 +1022,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
keys.append(c_clean)
break
if not keys:
lines.append(f" # TODO: complex GROUP BY could not be parsed")
lines.append(" # TODO: complex GROUP BY could not be parsed")
lines.append(f" return {p}")
return
keys_str = ", ".join(f'"{k}"' for k in keys)
@@ -1045,7 +1044,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
if agg_cols:
lines.append(f" return {p}.group_by({keys_str}).agg(")
_emit_exprs(lines, agg_cols)
lines.append(f" )")
lines.append(" )")
else:
lines.append(f" return {p}.group_by({keys_str}).agg()")
@@ -1061,20 +1060,20 @@ def _gen_window_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
lines.append(f" return {p}.with_columns(")
_emit_exprs(lines, nw_cols)
lines.append(f" )")
lines.append(" )")
def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
params = [ref_to_param(s, t) for s, t in deps]
if len(params) >= 2:
lines.append(f" return nw.concat([")
lines.append(" return nw.concat([")
for p in params:
lines.append(f" {p},")
lines.append(f" ])")
lines.append(" ])")
elif params:
lines.append(f" return {params[0]}")
else:
lines.append(f" return df")
lines.append(" return df")
# ── Module generation ───────────────────────────────────────────────────────
@@ -1121,9 +1120,7 @@ def main() -> None:
default=str(_conf_path("db.aco")),
help="Path to DuckDB file",
)
parser.add_argument(
"--out", default="../src/aco/express", help="Output directory"
)
parser.add_argument("--out", default="../src/aco/express", help="Output directory")
args = parser.parse_args()
db_path = Path(args.db)

View File

@@ -147,13 +147,12 @@ def generate_model(
escaped = comment.replace('"', '\\"')
if nullable:
field_lines.append(
f' {fname}: {py} | None = Field('
f" {fname}: {py} | None = Field("
f'default=None, description="{escaped}")'
)
else:
field_lines.append(
f' {fname}: {py} = Field('
f'description="{escaped}")'
f' {fname}: {py} = Field(description="{escaped}")'
)
else:
if nullable:
@@ -202,14 +201,13 @@ def generate_schema_module(
return f"{header}\n\n\n{body}\n"
# ── Main ────────────────────────────────────────────────────────────────────
def main() -> None:
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
from conf import cfg, path as _conf_path
from conf import cfg
from conf import path as _conf_path
parser.add_argument(
"--db",

View File

@@ -58,7 +58,6 @@ Design notes
from __future__ import annotations
import argparse
import re
import textwrap
import zipfile
@@ -68,7 +67,6 @@ from typing import NamedTuple
import pdfplumber
# ── Path constants ────────────────────────────────────────────────────────────
from conf import ROOT as _ROOT
from conf import path as _conf_path
@@ -176,8 +174,9 @@ def _extract_from_zip(
return None
def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
zotero_root: Path, work_dir: Path) -> Path | None:
def _resolve_pdf(
pdf_basename: str, zip_basename: str | None, zotero_root: Path, work_dir: Path
) -> Path | None:
"""Return path to a MIF PDF, extracting from zip if necessary."""
direct = _find_in_zotero(pdf_basename, zotero_root)
if direct:
@@ -187,8 +186,12 @@ def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
return None
def _resolve_xlsx(xlsx_basename: str | None, zip_basename: str | None,
zotero_root: Path, work_dir: Path) -> Path | None:
def _resolve_xlsx(
xlsx_basename: str | None,
zip_basename: str | None,
zotero_root: Path,
work_dir: Path,
) -> Path | None:
"""Return path to a value set Excel file, extracting from zip if needed."""
if xlsx_basename is None:
return None
@@ -329,7 +332,9 @@ def _read_sheet_summaries(xlsx_path: Path) -> dict[str, SheetSummary]:
for row in rows[:3]:
non_none = [str(c).strip() for c in row if c is not None]
if len(non_none) >= 2:
header = [str(c).strip().replace("\n", " ") for c in row if c is not None]
header = [
str(c).strip().replace("\n", " ") for c in row if c is not None
]
break
summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header)
wb.close()
@@ -407,13 +412,13 @@ def build_uamcc_passages(
paa4_info = _vs_info("UAMCC PAA4")
cohort_info = _vs_info("UAMCC Cohort")
excl_info = _vs_info("UAMCC Exclusions")
ccs_cm_info = _vs_info("UAMCC CCS-ICD10CM")
ccs_pcs_info = _vs_info("UAMCC CCS-ICD10PCS")
_vs_info("UAMCC CCS-ICD10CM")
_vs_info("UAMCC CCS-ICD10PCS")
cite_pp = _cite(pdf_name, "§3.8", denom_page)
cite_num = _cite(pdf_name, "§3.63.7", num_page)
_cite(pdf_name, "§3.63.7", num_page)
cite_excl = _cite(pdf_name, "§3.10", denom_excl_page)
cite_risk = _cite(pdf_name, "§3.12", risk_page)
_cite(pdf_name, "§3.12", risk_page)
cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page)
passages: dict[str, str] = {}
@@ -423,13 +428,13 @@ def build_uamcc_passages(
f"Return the UAMCC performance period anchor row.\n\n"
f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n"
f"UAMCC §2.2 Measure Description:\n"
f" \"This outcome measure is calculated using 12 consecutive months\n"
f' "This outcome measure is calculated using 12 consecutive months\n'
f" of Medicare fee-for-service (FFS) claims data. The measure is a\n"
f" risk-standardized acute admission rate (RSAAR) that adjusts for\n"
f" age, clinical comorbidities, and other clinical and frailty risk\n"
f" factors present at the start of the 12-month measurement period,\n"
f" as well as social risk factors. Lower RSAARs indicate better\n"
f" performance.\"\n\n"
f' performance."\n\n'
f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n"
f"Measurement duration: 12 consecutive months (Jan 1 Dec 31).\n"
f"Performance Year: {py_version}.\n\n"
@@ -443,10 +448,10 @@ def build_uamcc_passages(
f"Identify each beneficiary's qualifying chronic condition groups.\n\n"
f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n"
f"UAMCC §3.9 Denominator Details:\n"
f" \"The cohort is Medicare FFS beneficiaries 66 years of age and\n"
f' "The cohort is Medicare FFS beneficiaries 66 years of age and\n'
f" older assigned to the REACH ACO during the measurement period\n"
f" with diagnoses that fall into two or more of nine chronic disease\n"
f" groups.\"\n\n"
f' groups."\n\n'
f"Nine chronic disease groups (MIF §3.9, pp.78):\n"
f" 1. Acute myocardial infarction (AMI)\n"
f" 2. Alzheimer's disease and related disorders or senile dementia\n"
@@ -471,7 +476,7 @@ def build_uamcc_passages(
f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n"
f"Source: {cite_pp}\n\n"
f"UAMCC §3.8 Denominator Statement:\n"
f" \"{denom_stmt_trimmed[:400]}\"\n\n"
f' "{denom_stmt_trimmed[:400]}"\n\n'
f"Inclusion criteria (MIF §3.9):\n"
f" 1. Age ≥66 at the first day of the measurement period\n"
f" 2. Two or more distinct MCC groups identified in the lookback year\n"
@@ -501,13 +506,15 @@ def build_uamcc_passages(
)
# ── Planned admission (PAA) ───────────────────────────────────────
paa_snippet = ""
if num_detail:
# Find the PAA description paragraph
m = re.search(r"planned admission algorithm.+?(?=\n\n|\Z)", num_detail,
re.DOTALL | re.IGNORECASE)
m = re.search(
r"planned admission algorithm.+?(?=\n\n|\Z)",
num_detail,
re.DOTALL | re.IGNORECASE,
)
if m:
paa_snippet = m.group(0)[:600]
m.group(0)[:600]
passages["uamcc_int_planned_admission"] = (
f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n"
f"Source: {cite_paa}\n\n"
@@ -516,7 +523,7 @@ def build_uamcc_passages(
f" Readmission Algorithm Version 4.0, which CMS originally created\n"
f" to identify planned readmissions for the hospital-wide readmission\n"
f" measure. In brief, the algorithm uses a flowchart and four tables\n"
f" of procedure and/or discharge diagnosis categories.\"\n\n"
f' of procedure and/or discharge diagnosis categories."\n\n'
f"PAA Rules (first match wins):\n"
f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n"
f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n"
@@ -562,14 +569,14 @@ def build_uamcc_passages(
f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n"
f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n"
f"UAMCC §3.11 Denominator Exclusion Details:\n"
f" \"Persons are considered at risk for admission if they are alive,\n"
f' "Persons are considered at risk for admission if they are alive,\n'
f" enrolled in Medicare FFS, and not admitted to an acute care\n"
f" hospital. In addition to time spent in the hospital, excluded\n"
f" from at-risk time are:\n"
f" (1) time spent in an SNF or acute rehabilitation facility;\n"
f" (2) time within 10 days following discharge from a hospital,\n"
f" SNF, or acute rehabilitation facility;\n"
f" (3) time after entering hospice care.\"\n\n"
f' (3) time after entering hospice care."\n\n'
f"Person-years = at_risk_days / 365.25\n\n"
f"Performance Year: {py_version}."
)

View File

@@ -19,7 +19,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
from __future__ import annotations
import re
import textwrap
from pathlib import Path
import pdfplumber
@@ -29,10 +28,7 @@ import pdfplumber
_ROOT = Path(__file__).resolve().parents[2]
_SEEDS = _ROOT / "dev" / "seeds"
PDF_PATH = (
_SEEDS
/ "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
)
PDF_PATH = _SEEDS / "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py"
FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py"
QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py"

View File

@@ -21,7 +21,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
from __future__ import annotations
import re
from pathlib import Path
# Define all table specifications based on extracted data dictionary

View File

@@ -336,7 +336,9 @@ def main():
"""Generate REACH participants table model and rex parser."""
project_root = Path(__file__).resolve().parents[2]
xlsx_path = (
project_root / "dev" / "seeds"
project_root
/ "dev"
/ "seeds"
/ "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx"
)

View File

@@ -79,15 +79,12 @@ def sanitize_field(name: str) -> str:
def get_tables(con: sqlite3.Connection) -> list[str]:
"""Return all user tables, sorted alphabetically."""
rows = con.execute(
"SELECT name FROM sqlite_master WHERE type='table' "
"ORDER BY name"
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
).fetchall()
return [r[0] for r in rows]
def get_columns(
con: sqlite3.Connection, table: str
) -> list[tuple[str, str, bool]]:
def get_columns(con: sqlite3.Connection, table: str) -> list[tuple[str, str, bool]]:
"""Return [(col_name, sqlite_type, nullable), ...].
SQLite PRAGMA table_info returns:

View File

@@ -63,9 +63,7 @@ def _get_item_tags(zcon: sqlite3.Connection, item_id: int) -> list[str]:
return [r["name"] for r in rows]
def _get_item_collections(
zcon: sqlite3.Connection, item_id: int
) -> list[str]:
def _get_item_collections(zcon: sqlite3.Connection, item_id: int) -> list[str]:
"""Read collection keys for a Zotero item."""
rows = zcon.execute(
"""SELECT c.key FROM collectionItems ci
@@ -76,9 +74,7 @@ def _get_item_collections(
return [r["key"] for r in rows]
def _get_item_creators(
zcon: sqlite3.Connection, item_id: int
) -> list[dict]:
def _get_item_creators(zcon: sqlite3.Connection, item_id: int) -> list[dict]:
"""Read creators for a Zotero item."""
rows = zcon.execute(
"""SELECT c.firstName, c.lastName, ct.creatorType,
@@ -101,9 +97,7 @@ def _get_item_creators(
]
def _classify_item(
zotero_type: str, fields: dict
) -> str:
def _classify_item(zotero_type: str, fields: dict) -> str:
"""Map Zotero itemType + fields to bib item_type."""
if zotero_type == "statute":
code = fields.get("code", "")
@@ -134,14 +128,16 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
rule_type = part[6:]
elif part.startswith("Effective: "):
eff_date = part[11:]
return json.dumps({
return json.dumps(
{
"fr_volume": fields.get("codeNumber", ""),
"fr_page": fields.get("pages", ""),
"document_number": doc_num,
"cms_id": fields.get("session", ""),
"rule_type": rule_type,
"effective_date": eff_date,
})
}
)
if item_type == "regulation":
history = fields.get("history", "")
@@ -152,13 +148,15 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
part = segment[5:]
elif segment.startswith("Authority: "):
authority = segment[11:]
return json.dumps({
return json.dumps(
{
"cfr_title": fields.get("codeNumber", ""),
"cfr_part": part,
"cfr_section": fields.get("section", ""),
"authority": authority,
"effective_date": fields.get("dateEnacted", ""),
})
}
)
if item_type == "manual":
extra = fields.get("extra", "")
@@ -169,33 +167,37 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
chapter = fields.get("seriesNumber", "")
if chapter.startswith("Chapter "):
chapter = chapter[8:]
return json.dumps({
return json.dumps(
{
"manual_name": fields.get("seriesTitle", ""),
"pub_number": fields.get("reportNumber", ""),
"chapter": chapter,
"transmittal": transmittal,
})
}
)
if item_type == "download":
extra = fields.get("extra", "")
file_urls: list[str] = []
for line in extra.split("\n"):
if line.startswith("Files: "):
file_urls = [
u.strip() for u in line[7:].split(";") if u.strip()
]
return json.dumps({
file_urls = [u.strip() for u in line[7:].split(";") if u.strip()]
return json.dumps(
{
"page_type": "",
"file_urls": file_urls,
"year": None,
"quarter": "",
"website_title": fields.get("websiteTitle", ""),
})
}
)
if item_type == "source":
return json.dumps({
return json.dumps(
{
"doc_type": fields.get("type", ""),
})
}
)
return "{}"
@@ -255,8 +257,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
parent_id = row["id"]
bcon.execute(
"INSERT OR IGNORE INTO collections (key, name, parent_id) "
"VALUES (?, ?, ?)",
"INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
(zc["key"], zc["collectionName"], parent_id),
)
counts["collections"] += 1
@@ -322,8 +323,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
for tag_name in ztags:
tag_id = store._ensure_tag(tag_name)
bcon.execute(
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) "
"VALUES (?, ?)",
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
(bib_id, tag_id),
)
counts["tags"] += 1
@@ -346,16 +346,14 @@ def migrate(src: str, dst: str) -> dict[str, int]:
for cr in creators:
# Find or create creator
crow = bcon.execute(
"SELECT id FROM creators "
"WHERE first_name = ? AND last_name = ?",
"SELECT id FROM creators WHERE first_name = ? AND last_name = ?",
(cr["first_name"], cr["last_name"]),
).fetchone()
if crow:
creator_id = crow["id"]
else:
cur = bcon.execute(
"INSERT INTO creators (first_name, last_name) "
"VALUES (?, ?)",
"INSERT INTO creators (first_name, last_name) VALUES (?, ?)",
(cr["first_name"], cr["last_name"]),
)
creator_id = cur.lastrowid
@@ -462,9 +460,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
def main() -> None:
parser = argparse.ArgumentParser(
description="Migrate Zotero SQLite to bib SQLite"
)
parser = argparse.ArgumentParser(description="Migrate Zotero SQLite to bib SQLite")
from conf import path as _conf_path
parser.add_argument(

View File

@@ -60,7 +60,6 @@ from typing import Any
import polars as pl
# ── Path constants ────────────────────────────────────────────────────────────
from conf import ROOT as _ROOT
from conf import path as _conf_path

View File

@@ -319,16 +319,12 @@ def _ensure_value(con: sqlite3.Connection, value: str) -> int:
).fetchone()
if row:
return row[0]
cur = con.execute(
"INSERT INTO itemDataValues (value) VALUES (?)", (value,)
)
cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
return cur.lastrowid
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
row = con.execute(
"SELECT tagID FROM tags WHERE name = ?", (name,)
).fetchone()
row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
if row:
return row[0]
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
@@ -342,8 +338,7 @@ def _set_zotero_field(
return
value_id = _ensure_value(con, value)
con.execute(
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) "
"VALUES (?, ?, ?)",
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(item_id, field_id, value_id),
)
@@ -392,8 +387,7 @@ def tag_existing_zotero(
for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name)
con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
"VALUES (?, ?, 0)",
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(parent_id, tag_id),
)
return True
@@ -416,9 +410,9 @@ def register_in_zotero(
return
now = _now_iso()
next_id = con.execute(
"SELECT COALESCE(MAX(itemID), 0) + 1 FROM items"
).fetchone()[0]
next_id = con.execute("SELECT COALESCE(MAX(itemID), 0) + 1 FROM items").fetchone()[
0
]
parent_key = _zotero_key()
# Create parent webpage item
@@ -434,9 +428,7 @@ def register_in_zotero(
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url)
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
_set_zotero_field(
con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal"
)
_set_zotero_field(con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal")
_set_zotero_field(
con,
next_id,
@@ -449,8 +441,7 @@ def register_in_zotero(
for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name)
con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
"VALUES (?, ?, 0)",
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
(next_id, tag_id),
)
@@ -464,13 +455,9 @@ def register_in_zotero(
# Copy to Zotero storage
storage_dir = ZOTERO_STORAGE / att_key
subprocess.run(
["sudo", "mkdir", "-p", str(storage_dir)], check=True
)
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
dest = storage_dir / seed_path.name
subprocess.run(
["sudo", "cp", str(seed_path), str(dest)], check=True
)
subprocess.run(["sudo", "cp", str(seed_path), str(dest)], check=True)
subprocess.run(
["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
check=True,
@@ -496,8 +483,7 @@ def register_in_zotero(
)
title_val = _ensure_value(con, seed_path.name)
con.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) "
"VALUES (?, ?, ?)",
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
(att_id, ZOTERO_FIELD_TITLE, title_val),
)

View File

@@ -97,11 +97,11 @@ def main():
print(f" ... and {len(tables) - 3} more tables")
print("\n" + "=" * 70)
print(f"\nTo create these in Unity Catalog, run without --show-schemas flag")
print("\nTo create these in Unity Catalog, run without --show-schemas flag")
return
# Create Unity client
print(f"Connecting to Unity Catalog...")
print("Connecting to Unity Catalog...")
print(f" Workspace ID: {args.workspace_id}")
print(f" Catalog: {args.catalog}")
print()
@@ -114,7 +114,7 @@ def main():
# Test connection
try:
catalogs = client.list_catalogs()
print(f"✓ Connected to Unity Catalog")
print("✓ Connected to Unity Catalog")
print(f" Found {len(catalogs)} existing catalogs:")
for cat in catalogs:
print(f" - {cat.name}")
@@ -130,7 +130,7 @@ def main():
print("=" * 70)
print()
print(f"Setting up catalog structure...")
print("Setting up catalog structure...")
print()
report = setup_catalog_from_schemas(

View File

@@ -20,11 +20,11 @@ fi
# Image definitions — matches stack.toml [images]
declare -A DOCKERFILES=(
[api]="api/Dockerfile"
[notebooks]="notebooks/Dockerfile"
[docs]="docs/Dockerfile"
[zotero]="zotero/Dockerfile"
[mc]="rustfs/Dockerfile.mc"
[api]="infra/images/api.Dockerfile"
[notebooks]="infra/images/notebooks.Dockerfile"
[docs]="infra/images/docs.Dockerfile"
[zotero]="infra/images/zotero.Dockerfile"
[mc]="infra/images/mc.Dockerfile"
)
declare -A CONTEXTS=(
@@ -32,7 +32,7 @@ declare -A CONTEXTS=(
[notebooks]="notebooks/"
[docs]="."
[zotero]="zotero/"
[mc]="rustfs/"
[mc]="infra/rustfs/"
)
# Select images to test

View File

@@ -23,7 +23,6 @@ import os
from aco.lake import IcebergContext, UnityClient
from aco.lake.catalog import Catalog
from aco.lake.engine import execute
def example_unity_client_basics():
@@ -185,7 +184,7 @@ def example_iceberg_context():
},
)
print(f"✓ IcebergContext configured")
print("✓ IcebergContext configured")
print(f" Catalog URI: {ctx.catalog_uri}")
print(f" Warehouse: {ctx.warehouse}")
print()
@@ -222,7 +221,7 @@ def example_pipeline_execution():
}
)
ctx = IcebergContext(
IcebergContext(
catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg",
warehouse="aco_dev",
catalog=catalog,

View File

@@ -1,4 +1,4 @@
/* LOCH — deep-navy design system — matches styles/inject.css */
/* LOCH — deep-navy design system — matches assets/css/inject.css */
:root {
--ifm-color-primary: #4488dd;

0
infra/polaris/.gitkeep Normal file
View File

View File

@@ -17,7 +17,7 @@ Usage::
CLI::
uv run python -m aco.dag
uv run python -m aco.dag -p readmissions --html dag.html
uv run python -m aco.dag -p readmissions --html assets/dag.html
uv run python -m aco.dag --dot dag.dot
uv run python -m aco.dag -p pharmacy --mermaid
"""