chore: clean sweep — lint, format, stale refs, generated artifacts
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled

- Fix all 72 ruff lint errors (unused imports, unused variables, E402)
- Format all 14 unformatted dev/scripts files
- Move generated artifacts to assets/ (dag.html, pfs.html)
- Remove duplicate root coverage.svg (already in assets/icons/)
- Update .dockerignore for infra/ tree layout
- Update .gitignore: add .env.bak, mirrors/, htmlcov/
- Fix stale path refs in coverage_badge.py, woodpecker backend,
  test_network_isolation.sh, docs custom.css
- Add .gitkeep to empty dirs (infra/polaris, cloud/*/terraform)
- Delete 12 stale local branches, 10 stale remote branches
This commit is contained in:
kert
2026-03-24 17:33:55 -04:00
parent 29b302cdf0
commit 85ce5e719d
38 changed files with 340 additions and 304 deletions

View File

@@ -12,13 +12,8 @@ logs/
tuva/ tuva/
dev/ dev/
tests/ tests/
grafana/ infra/
prometheus/ assets/
loki/ cloud/
trino/ mirrors/
traefik/
nginx/
gitea/
styles/
.woodpecker/
.claude/ .claude/

13
.gitignore vendored
View File

@@ -13,17 +13,28 @@ bundle/sql/
# Marimo cache # Marimo cache
notebooks/__marimo__/ notebooks/__marimo__/
# Secrets and keys
.deploy_key .deploy_key
.env
.deploy_key.pub .deploy_key.pub
.env
.env.bak
# Python build artifacts
__pycache__/ __pycache__/
*.pyc *.pyc
.coverage .coverage
dist/ dist/
*.egg-info/ *.egg-info/
htmlcov/
# IDE / tool settings
.claude/settings.local.json .claude/settings.local.json
notebooks/aco.duckdb notebooks/aco.duckdb
# Package mirrors (local cache, not tracked)
mirrors/
# Docs (auto-generated at build time) # Docs (auto-generated at build time)
docs/docs/api/ docs/docs/api/
docs/static/library.json docs/static/library.json

View File

View File

View File

View File

@@ -1,20 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" width="116" height="20">
<linearGradient id="b" x2="0" y2="100%">
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
<stop offset="1" stop-opacity=".1"/>
</linearGradient>
<clipPath id="a">
<rect width="116" height="20" rx="3" fill="#fff"/>
</clipPath>
<g clip-path="url(#a)">
<path fill="#555" d="M0 0h65v20H0z"/>
<path fill="#4c1" d="M65 0h51v20H65z"/>
<path fill="url(#b)" d="M0 0h116v20H0z"/>
</g>
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
<text x="32.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
<text x="32.5" y="14">coverage</text>
<text x="90" y="15" fill="#010101" fill-opacity=".3">99%</text>
<text x="90" y="14">99%</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 822 B

View File

@@ -7,11 +7,8 @@ Usage:
uv run python dev/scripts/add_carrier_to_zotero.py uv run python dev/scripts/add_carrier_to_zotero.py
""" """
import os
import random import random
import shutil
import sqlite3 import sqlite3
import string
import subprocess import subprocess
import zipfile import zipfile
from datetime import datetime, timezone from datetime import datetime, timezone

View File

@@ -9,7 +9,6 @@ Usage:
import random import random
import sqlite3 import sqlite3
import string
import subprocess import subprocess
import zipfile import zipfile
from datetime import datetime, timezone from datetime import datetime, timezone

View File

@@ -30,8 +30,11 @@ yaml.add_representer(
def _dump(data: dict) -> str: def _dump(data: dict) -> str:
return yaml.dump( return yaml.dump(
data, default_flow_style=False, sort_keys=False, data,
width=120, allow_unicode=True, default_flow_style=False,
sort_keys=False,
width=120,
allow_unicode=True,
) )
@@ -69,9 +72,7 @@ class SQLPlugin(DabPlugin):
output_mode="ctas", output_mode="ctas",
) )
except Exception as e: except Exception as e:
result[f"bundle/sql/{pipe_name}/_error.txt"] = ( result[f"bundle/sql/{pipe_name}/_error.txt"] = f"Transpile failed: {e}"
f"Transpile failed: {e}"
)
continue continue
for expr_name, sql in sql_map.items(): for expr_name, sql in sql_map.items():
@@ -80,8 +81,7 @@ class SQLPlugin(DabPlugin):
schema, table = expr_name.split(".", 1) schema, table = expr_name.split(".", 1)
path = f"bundle/sql/{pipe_name}/{table}.sql" path = f"bundle/sql/{pipe_name}/{table}.sql"
header = ( header = (
f"-- {expr_name}\n" f"-- {expr_name}\n-- Generated by gen_config.py — DO NOT EDIT\n\n"
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
) )
result[path] = header + sql + "\n" result[path] = header + sql + "\n"
@@ -102,27 +102,58 @@ class JobsPlugin(DabPlugin):
job_clusters = [] job_clusters = []
for profile_name in sorted(used_profiles): for profile_name in sorted(used_profiles):
cluster_spec = dict(clusters_cfg.get(profile_name, {})) cluster_spec = dict(clusters_cfg.get(profile_name, {}))
job_clusters.append(OrderedDict([ job_clusters.append(
OrderedDict(
[
("job_cluster_key", f"{profile_name}_cluster"), ("job_cluster_key", f"{profile_name}_cluster"),
("new_cluster", OrderedDict([ (
("spark_version", cluster_spec.get("spark_version", "15.4.x-scala2.12")), "new_cluster",
("node_type_id", cluster_spec.get("node_type_id", "i3.xlarge")), OrderedDict(
[
(
"spark_version",
cluster_spec.get(
"spark_version", "15.4.x-scala2.12"
),
),
(
"node_type_id",
cluster_spec.get("node_type_id", "i3.xlarge"),
),
("num_workers", cluster_spec.get("num_workers", 1)), ("num_workers", cluster_spec.get("num_workers", 1)),
])), ]
])) ),
),
]
)
)
# DDL tasks — one per schema to create tables before SQL runs # DDL tasks — one per schema to create tables before SQL runs
ddl_tasks = [] ddl_tasks = []
for schema_name in sorted(catalog_schemas): for schema_name in sorted(catalog_schemas):
ddl_tasks.append(OrderedDict([ ddl_tasks.append(
OrderedDict(
[
("task_key", f"ddl_{schema_name}"), ("task_key", f"ddl_{schema_name}"),
("job_cluster_key", "default_cluster"), ("job_cluster_key", "default_cluster"),
("sql_task", OrderedDict([ (
("file", OrderedDict([ "sql_task",
OrderedDict(
[
(
"file",
OrderedDict(
[
("path", f"bundle/ddl/{schema_name}/"), ("path", f"bundle/ddl/{schema_name}/"),
])), ]
])), ),
])) ),
]
),
),
]
)
)
# Pipeline tasks — each expression becomes a sql_task # Pipeline tasks — each expression becomes a sql_task
pipeline_tasks = [] pipeline_tasks = []
@@ -143,12 +174,19 @@ class JobsPlugin(DabPlugin):
# Each pipeline task runs all its SQL files in order # Each pipeline task runs all its SQL files in order
# We use a for_each or sequential notebook; simplest is # We use a for_each or sequential notebook; simplest is
# a single SQL file per pipeline that sources all expressions # a single SQL file per pipeline that sources all expressions
task["sql_task"] = OrderedDict([ task["sql_task"] = OrderedDict(
("file", OrderedDict([ [
(
"file",
OrderedDict(
[
("path", f"bundle/sql/{pipe_name}/"), ("path", f"bundle/sql/{pipe_name}/"),
])), ]
),
),
("warehouse_id", "${var.warehouse_id}"), ("warehouse_id", "${var.warehouse_id}"),
]) ]
)
pipeline_tasks.append(task) pipeline_tasks.append(task)
schedule = cfg.get("schedule", "0 0 6 * * ?") schedule = cfg.get("schedule", "0 0 6 * * ?")
@@ -156,11 +194,15 @@ class JobsPlugin(DabPlugin):
job = OrderedDict() job = OrderedDict()
job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines" job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines"
job["description"] = f"Run all {len(registry)} ACO pipelines in dependency order" job["description"] = (
job["schedule"] = OrderedDict([ f"Run all {len(registry)} ACO pipelines in dependency order"
)
job["schedule"] = OrderedDict(
[
("quartz_cron_expression", schedule), ("quartz_cron_expression", schedule),
("timezone_id", timezone), ("timezone_id", timezone),
]) ]
)
job["job_clusters"] = job_clusters job["job_clusters"] = job_clusters
job["tasks"] = pipeline_tasks job["tasks"] = pipeline_tasks
@@ -175,10 +217,12 @@ class SchemasPlugin(DabPlugin):
return {} return {}
schemas = OrderedDict() schemas = OrderedDict()
for schema_name in sorted(catalog_schemas): for schema_name in sorted(catalog_schemas):
schemas[schema_name] = OrderedDict([ schemas[schema_name] = OrderedDict(
[
("name", schema_name), ("name", schema_name),
("catalog_name", "${var.catalog}"), ("catalog_name", "${var.catalog}"),
]) ]
)
return {"schemas": schemas} return {"schemas": schemas}
@@ -186,14 +230,23 @@ class VolumesPlugin(DabPlugin):
"""Generate a staging volume for data upload.""" """Generate a staging volume for data upload."""
def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict: def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict:
return {"volumes": OrderedDict([ return {
("staging", OrderedDict([ "volumes": OrderedDict(
[
(
"staging",
OrderedDict(
[
("name", "staging"), ("name", "staging"),
("catalog_name", "${var.catalog}"), ("catalog_name", "${var.catalog}"),
("schema_name", "default"), ("schema_name", "default"),
("volume_type", "MANAGED"), ("volume_type", "MANAGED"),
])), ]
])} ),
),
]
)
}
# ── Core generator ─────────────────────────────────────────────── # ── Core generator ───────────────────────────────────────────────
@@ -214,6 +267,7 @@ _DEFAULT_PLUGINS: list[DabPlugin] = [
def _get_catalog_schemas() -> list[str]: def _get_catalog_schemas() -> list[str]:
try: try:
from aco.lake.catalog import Catalog from aco.lake.catalog import Catalog
return Catalog().schemas() return Catalog().schemas()
except Exception: except Exception:
return [] return []
@@ -252,19 +306,33 @@ def emit(cfg_data: dict) -> dict[str, str]:
bundle["bundle"] = OrderedDict([("name", bundle_name)]) bundle["bundle"] = OrderedDict([("name", bundle_name)])
# Sync SQL files to workspace # Sync SQL files to workspace
bundle["sync"] = OrderedDict([ bundle["sync"] = OrderedDict(
[
("include", ["bundle/**"]), ("include", ["bundle/**"]),
]) ]
)
bundle["variables"] = OrderedDict([ bundle["variables"] = OrderedDict(
("catalog", OrderedDict([ [
(
"catalog",
OrderedDict(
[
("description", "Unity Catalog name for table references"), ("description", "Unity Catalog name for table references"),
("default", targets_cfg.get("prod", {}).get("catalog", "aco")), ("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
])), ]
("warehouse_id", OrderedDict([ ),
),
(
"warehouse_id",
OrderedDict(
[
("description", "SQL warehouse ID for query execution"), ("description", "SQL warehouse ID for query execution"),
])), ]
]) ),
),
]
)
bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")]) bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")])
@@ -276,17 +344,21 @@ def emit(cfg_data: dict) -> dict[str, str]:
target["mode"] = tcfg["mode"] target["mode"] = tcfg["mode"]
if tcfg.get("default"): if tcfg.get("default"):
target["default"] = True target["default"] = True
target["variables"] = OrderedDict([ target["variables"] = OrderedDict(
[
("catalog", tcfg.get("catalog", "aco")), ("catalog", tcfg.get("catalog", "aco")),
]) ]
)
if tcfg.get("warehouse_id"): if tcfg.get("warehouse_id"):
target["variables"]["warehouse_id"] = tcfg["warehouse_id"] target["variables"]["warehouse_id"] = tcfg["warehouse_id"]
if tcfg.get("mode") == "production": if tcfg.get("mode") == "production":
run_as = dab_cfg.get("run_as", {}) run_as = dab_cfg.get("run_as", {})
if run_as.get("service_principal_name"): if run_as.get("service_principal_name"):
target["run_as"] = OrderedDict([ target["run_as"] = OrderedDict(
[
("service_principal_name", run_as["service_principal_name"]), ("service_principal_name", run_as["service_principal_name"]),
]) ]
)
targets[tname] = target targets[tname] = target
bundle["targets"] = targets bundle["targets"] = targets

View File

@@ -242,7 +242,7 @@ steps:
UV_PROJECT_ENVIRONMENT: .venv UV_PROJECT_ENVIRONMENT: .venv
commands: commands:
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out - uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg - uv run python dev/scripts/coverage_badge.py < pytest.out > assets/icons/coverage.svg
depends_on: depends_on:
- lint - lint
@@ -273,7 +273,7 @@ steps:
commands: commands:
- git config user.name woodpecker-ci - git config user.name woodpecker-ci
- git config user.email ci@${{CI_REPO_OWNER}}.io - git config user.email ci@${{CI_REPO_OWNER}}.io
- git add coverage.svg - git add assets/icons/coverage.svg
- git diff --cached --quiet && echo "no change" && exit 0 - git diff --cached --quiet && echo "no change" && exit 0
- git commit -m "update coverage badge [skip ci]" - git commit -m "update coverage badge [skip ci]"
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main - git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
@@ -293,7 +293,7 @@ steps:
from_secret: s3_access_key from_secret: s3_access_key
secret_key: secret_key:
from_secret: s3_secret_key from_secret: s3_secret_key
source: "coverage.svg" source: "assets/icons/coverage.svg"
target: /badges/${{CI_REPO}}/ target: /badges/${{CI_REPO}}/
path_style: true path_style: true
overwrite: true overwrite: true

View File

@@ -15,7 +15,6 @@ import argparse
import os import os
import shutil import shutil
import subprocess import subprocess
import sys
from pathlib import Path from pathlib import Path
REPO_ROOT = Path(__file__).resolve().parents[2] REPO_ROOT = Path(__file__).resolve().parents[2]
@@ -33,7 +32,7 @@ CA_SUBJECT = "/CN=Homelab CA/O=fhirworx"
CA_DAYS = 3650 CA_DAYS = 3650
TLS_DAYS = 3650 TLS_DAYS = 3650
from conf import cfg from conf import cfg # noqa: E402
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain) DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip) HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)

View File

@@ -8,7 +8,7 @@ Usage:
# CI_COMMIT_SHA from environment): # CI_COMMIT_SHA from environment):
uv run python dev/scripts/coverage_badge.py --post pytest.out uv run python dev/scripts/coverage_badge.py --post pytest.out
# Update styles/coverage.svg in the repo via Gitea API: # Update assets/icons/coverage.svg in the repo via Gitea API:
uv run python dev/scripts/coverage_badge.py --upload coverage.svg uv run python dev/scripts/coverage_badge.py --upload coverage.svg
""" """
@@ -44,7 +44,7 @@ TEMPLATE = """\
</g> </g>
</svg>""" </svg>"""
FILE_PATH = "styles/coverage.svg" FILE_PATH = "assets/icons/coverage.svg"
def color_for(pct: int) -> str: def color_for(pct: int) -> str:
@@ -73,11 +73,13 @@ def post_status(pct: int) -> None:
token, url = _gitea_env() token, url = _gitea_env()
sha = os.environ["CI_COMMIT_SHA"] sha = os.environ["CI_COMMIT_SHA"]
endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}" endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}"
body = json.dumps({ body = json.dumps(
{
"state": "success", "state": "success",
"description": f"{pct}% coverage", "description": f"{pct}% coverage",
"context": "coverage", "context": "coverage",
}).encode() }
).encode()
req = urllib.request.Request( req = urllib.request.Request(
endpoint, endpoint,
data=body, data=body,
@@ -92,7 +94,7 @@ def post_status(pct: int) -> None:
def upload_badge(svg_path: str) -> None: def upload_badge(svg_path: str) -> None:
"""Update styles/coverage.svg in the repo via Gitea contents API.""" """Update assets/icons/coverage.svg in the repo via Gitea contents API."""
token, url = _gitea_env() token, url = _gitea_env()
endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}" endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}"
svg_data = open(svg_path, "rb").read() svg_data = open(svg_path, "rb").read()

View File

@@ -40,7 +40,6 @@ def main() -> int:
for col in COLUMNS: for col in COLUMNS:
for op in ("INSERT", "UPDATE"): for op in ("INSERT", "UPDATE"):
trigger = f"fix_{col}_{op.lower()}" trigger = f"fix_{col}_{op.lower()}"
of_clause = f"OF {col} " if op == "UPDATE" else ""
db.execute(f""" db.execute(f"""
CREATE TRIGGER IF NOT EXISTS {trigger} CREATE TRIGGER IF NOT EXISTS {trigger}
AFTER {op} ON items AFTER {op} ON items

View File

@@ -219,7 +219,11 @@ def generate_module() -> str:
def main(): def main():
"""Generate src/aco/table/alr_filenames.py.""" """Generate src/aco/table/alr_filenames.py."""
output_path = ( output_path = (
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "alr_filenames.py" Path(__file__).resolve().parents[2]
/ "src"
/ "aco"
/ "table"
/ "alr_filenames.py"
) )
code = generate_module() code = generate_module()
@@ -228,9 +232,9 @@ def main():
output_path.write_text(code) output_path.write_text(code)
print(f"Generated {output_path}") print(f"Generated {output_path}")
print(f" 18 ALR table patterns (9 annual + 9 quarterly)") print(" 18 ALR table patterns (9 annual + 9 quarterly)")
print(f" 2 ASR patterns (annual + quarterly)") print(" 2 ASR patterns (annual + quarterly)")
print(f" 4 package patterns") print(" 4 package patterns")
return 0 return 0

View File

@@ -25,7 +25,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
Returns dict mapping table_id -> {title, fields: list[dict]} Returns dict mapping table_id -> {title, fields: list[dict]}
""" """
pdf = pdfplumber.open(pdf_path) pdfplumber.open(pdf_path)
# Manual field specifications based on PDF extraction # Manual field specifications based on PDF extraction
# Table 1-1 is on pages 8-15, others follow # Table 1-1 is on pages 8-15, others follow
@@ -163,7 +163,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
{ {
"name": f"HCC_Position_{i:02d}", "name": f"HCC_Position_{i:02d}",
"type": "int", "type": "int",
"desc": f"HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary", "desc": "HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
} }
) )
@@ -530,7 +530,7 @@ def generate_module(tables: dict) -> str:
lines.append(" Source: ALRASRGuide.pdf") lines.append(" Source: ALRASRGuide.pdf")
lines.append(' """') lines.append(' """')
lines.append("") lines.append("")
lines.append(f' __schema__ = "alr"') lines.append(' __schema__ = "alr"')
lines.append(f' __tablename__ = "{tbl}"') lines.append(f' __tablename__ = "{tbl}"')
for field in info["fields"]: for field in info["fields"]:

View File

@@ -370,7 +370,7 @@ def generate_module(tables: dict) -> str:
lines.append(" Source: ALRASRGuide.pdf") lines.append(" Source: ALRASRGuide.pdf")
lines.append(' """') lines.append(' """')
lines.append("") lines.append("")
lines.append(f' __schema__ = "asr"') lines.append(' __schema__ = "asr"')
lines.append(f' __tablename__ = "{tbl}"') lines.append(f' __tablename__ = "{tbl}"')
for field in info["fields"]: for field in info["fields"]:
@@ -398,7 +398,9 @@ def generate_module(tables: dict) -> str:
def main(): def main():
"""Generate src/aco/table/asr.py from ALRASRGuide.pdf.""" """Generate src/aco/table/asr.py from ALRASRGuide.pdf."""
output_path = Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py" output_path = (
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
)
tables = extract_asr_table_specs() tables = extract_asr_table_specs()
code = generate_module(tables) code = generate_module(tables)

View File

@@ -14,7 +14,6 @@ Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
from __future__ import annotations from __future__ import annotations
import re import re
from datetime import date
from pathlib import Path from pathlib import Path
import pdfplumber import pdfplumber
@@ -206,7 +205,7 @@ def generate_module(tables: dict) -> str:
lines.append(f" {len(info['fields'])} fields, fixed-width layout.") lines.append(f" {len(info['fields'])} fields, fixed-width layout.")
lines.append(' """') lines.append(' """')
lines.append("") lines.append("")
lines.append(f' __schema__ = "cclf"') lines.append(' __schema__ = "cclf"')
lines.append(f' __tablename__ = "{tbl_name}"') lines.append(f' __tablename__ = "{tbl_name}"')
seen_names: set[str] = set() seen_names: set[str] = set()

View File

@@ -17,8 +17,7 @@ from __future__ import annotations
import argparse import argparse
import re import re
import textwrap from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
import pdfplumber import pdfplumber
@@ -146,7 +145,7 @@ def parse_entry(raw: str) -> Variable | None:
# Parse the remaining structured fields # Parse the remaining structured fields
# Rejoin everything from LABEL: onward # Rejoin everything from LABEL: onward
body = "\n".join(lines[label_idx:]) "\n".join(lines[label_idx:])
# Extract each field by finding the label and collecting text # Extract each field by finding the label and collecting text
# until the next label # until the next label
@@ -211,9 +210,9 @@ def variable_to_py(var: Variable) -> str:
primary_name = var.name.split("\n")[0].strip() primary_name = var.name.split("\n")[0].strip()
lines = [ lines = [
f'"""', '"""',
f"Variable: {primary_name}", f"Variable: {primary_name}",
f"", "",
] ]
if var.label and var.label != "": if var.label and var.label != "":
@@ -289,7 +288,9 @@ def main() -> None:
"--pdf", "--pdf",
default=str( default=str(
Path(__file__).resolve().parents[2] Path(__file__).resolve().parents[2]
/ "dev" / "seeds" / "codebook-ffs-claims.pdf" / "dev"
/ "seeds"
/ "codebook-ffs-claims.pdf"
), ),
help="Path to the CCW codebook PDF", help="Path to the CCW codebook PDF",
) )

View File

@@ -16,11 +16,9 @@ Defaults:
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import re
from pathlib import Path from pathlib import Path
import fastexcel import fastexcel
import polars as pl
# ── Sheets and their slug names ────────────────────────────────────────────── # ── Sheets and their slug names ──────────────────────────────────────────────
@@ -169,7 +167,7 @@ def generate_model(
f"class {cls}(SQLTable):", f"class {cls}(SQLTable):",
f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""', f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""',
"", "",
f' __schema__ = "ccw"', ' __schema__ = "ccw"',
f' __tablename__ = "{claim_slug}_{table}"', f' __tablename__ = "{claim_slug}_{table}"',
"", "",
] ]
@@ -214,7 +212,9 @@ def main() -> None:
"--xlsx", "--xlsx",
default=str( default=str(
Path(__file__).resolve().parents[2] Path(__file__).resolve().parents[2]
/ "dev" / "seeds" / "record-layout-ffs-claims.xlsx" / "dev"
/ "seeds"
/ "record-layout-ffs-claims.xlsx"
), ),
help="Path to the CCW record layout Excel file", help="Path to the CCW record layout Excel file",
) )

View File

@@ -224,9 +224,9 @@ def generate_function(table: dict) -> str:
"@nw.narwhalify", "@nw.narwhalify",
f"def {tablename}({param}: FrameT) -> FrameT:", f"def {tablename}({param}: FrameT) -> FrameT:",
f' """Clean and type-cast cms.{tablename}.', f' """Clean and type-cast cms.{tablename}.',
f"", "",
f" {field_count} fields. Numeric columns auto-cast to Float64.", f" {field_count} fields. Numeric columns auto-cast to Float64.",
f' """', ' """',
f" return auto_cast({param})", f" return auto_cast({param})",
] ]
return "\n".join(lines) return "\n".join(lines)
@@ -236,18 +236,18 @@ def generate_module(domain: str, tables: list[dict]) -> str:
"""Generate a full express module for a domain group.""" """Generate a full express module for a domain group."""
lines = [ lines = [
f'"""CMS express — {domain} domain.', f'"""CMS express — {domain} domain.',
f"", "",
f"Auto-generated narwhals functions for {len(tables)} CMS tables.", f"Auto-generated narwhals functions for {len(tables)} CMS tables.",
f"Each function takes the raw table and returns a typed frame", "Each function takes the raw table and returns a typed frame",
f"with numeric columns cast to Float64.", "with numeric columns cast to Float64.",
f'"""', '"""',
f"", "",
f"from __future__ import annotations", "from __future__ import annotations",
f"", "",
f"import narwhals as nw", "import narwhals as nw",
f"from narwhals.typing import FrameT", "from narwhals.typing import FrameT",
f"", "",
f"from cms.express._helpers import auto_cast", "from cms.express._helpers import auto_cast",
] ]
for tbl in sorted(tables, key=lambda t: t["tablename"]): for tbl in sorted(tables, key=lambda t: t["tablename"]):

View File

@@ -14,7 +14,6 @@ import argparse
import re import re
from collections import defaultdict from collections import defaultdict
from pathlib import Path from pathlib import Path
from textwrap import dedent, indent
import duckdb import duckdb
@@ -268,7 +267,7 @@ def col_expr_to_nw(expr: str) -> str:
# date_diff('unit', a, b) # date_diff('unit', a, b)
dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE) dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE)
if dd_m: if dd_m:
unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip() _unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
if alias: if alias:
return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")' return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")'
return f"{_col_ref(b)} - {_col_ref(a)}" return f"{_col_ref(b)} - {_col_ref(a)}"
@@ -712,13 +711,13 @@ def generate_function(
doc_ops = ", ".join(ops) doc_ops = ", ".join(ops)
lines: list[str] = [] lines: list[str] = []
lines.append(f"@nw.narwhalify") lines.append("@nw.narwhalify")
lines.append(f"def {func_name}({param_str}) -> FrameT:") lines.append(f"def {func_name}({param_str}) -> FrameT:")
lines.append(f' """Build {schema}.{view_name}') lines.append(f' """Build {schema}.{view_name}')
lines.append(f"") lines.append("")
lines.append(f" Operations: {doc_ops}") lines.append(f" Operations: {doc_ops}")
lines.append(f" Dependencies: {doc_deps}") lines.append(f" Dependencies: {doc_deps}")
lines.append(f' """') lines.append(' """')
if "passthrough" in ops: if "passthrough" in ops:
p = params[0].split(":")[0] p = params[0].split(":")[0]
@@ -880,7 +879,7 @@ def _gen_select_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
if nw_cols: if nw_cols:
lines.append(f" return {p}.select(") lines.append(f" return {p}.select(")
_emit_exprs(lines, nw_cols) _emit_exprs(lines, nw_cols)
lines.append(f" )") lines.append(" )")
else: else:
lines.append(f" return {p}") lines.append(f" return {p}")
@@ -902,13 +901,13 @@ def _gen_filter_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
lines.append(f" # {comment_part}") lines.append(f" # {comment_part}")
lines.append(f" return {p}.filter(") lines.append(f" return {p}.filter(")
lines.append(f" {code_part}") lines.append(f" {code_part}")
lines.append(f" )") lines.append(" )")
else: else:
safe = clause.replace('"', "'")[:80] safe = clause.replace('"', "'")[:80]
lines.append(f" # TODO: complex filter: {safe}") lines.append(f" # TODO: complex filter: {safe}")
lines.append(f" return {p}") lines.append(f" return {p}")
else: else:
lines.append(f" # WHERE clause could not be parsed") lines.append(" # WHERE clause could not be parsed")
lines.append(f" return {p}") lines.append(f" return {p}")
@@ -994,9 +993,9 @@ def _gen_join_body(
keys_str = ", ".join(f'"{k}"' for k in keys) keys_str = ", ".join(f'"{k}"' for k in keys)
lines.append(f" result = result.group_by({keys_str}).agg()") lines.append(f" result = result.group_by({keys_str}).agg()")
else: else:
lines.append(f" # TODO: complex GROUP BY") lines.append(" # TODO: complex GROUP BY")
lines.append(f" return result") lines.append(" return result")
def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None: def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
@@ -1023,7 +1022,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
keys.append(c_clean) keys.append(c_clean)
break break
if not keys: if not keys:
lines.append(f" # TODO: complex GROUP BY could not be parsed") lines.append(" # TODO: complex GROUP BY could not be parsed")
lines.append(f" return {p}") lines.append(f" return {p}")
return return
keys_str = ", ".join(f'"{k}"' for k in keys) keys_str = ", ".join(f'"{k}"' for k in keys)
@@ -1045,7 +1044,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
if agg_cols: if agg_cols:
lines.append(f" return {p}.group_by({keys_str}).agg(") lines.append(f" return {p}.group_by({keys_str}).agg(")
_emit_exprs(lines, agg_cols) _emit_exprs(lines, agg_cols)
lines.append(f" )") lines.append(" )")
else: else:
lines.append(f" return {p}.group_by({keys_str}).agg()") lines.append(f" return {p}.group_by({keys_str}).agg()")
@@ -1061,20 +1060,20 @@ def _gen_window_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
lines.append(f" return {p}.with_columns(") lines.append(f" return {p}.with_columns(")
_emit_exprs(lines, nw_cols) _emit_exprs(lines, nw_cols)
lines.append(f" )") lines.append(" )")
def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None: def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
params = [ref_to_param(s, t) for s, t in deps] params = [ref_to_param(s, t) for s, t in deps]
if len(params) >= 2: if len(params) >= 2:
lines.append(f" return nw.concat([") lines.append(" return nw.concat([")
for p in params: for p in params:
lines.append(f" {p},") lines.append(f" {p},")
lines.append(f" ])") lines.append(" ])")
elif params: elif params:
lines.append(f" return {params[0]}") lines.append(f" return {params[0]}")
else: else:
lines.append(f" return df") lines.append(" return df")
# ── Module generation ─────────────────────────────────────────────────────── # ── Module generation ───────────────────────────────────────────────────────
@@ -1121,9 +1120,7 @@ def main() -> None:
default=str(_conf_path("db.aco")), default=str(_conf_path("db.aco")),
help="Path to DuckDB file", help="Path to DuckDB file",
) )
parser.add_argument( parser.add_argument("--out", default="../src/aco/express", help="Output directory")
"--out", default="../src/aco/express", help="Output directory"
)
args = parser.parse_args() args = parser.parse_args()
db_path = Path(args.db) db_path = Path(args.db)

View File

@@ -147,13 +147,12 @@ def generate_model(
escaped = comment.replace('"', '\\"') escaped = comment.replace('"', '\\"')
if nullable: if nullable:
field_lines.append( field_lines.append(
f' {fname}: {py} | None = Field(' f" {fname}: {py} | None = Field("
f'default=None, description="{escaped}")' f'default=None, description="{escaped}")'
) )
else: else:
field_lines.append( field_lines.append(
f' {fname}: {py} = Field(' f' {fname}: {py} = Field(description="{escaped}")'
f'description="{escaped}")'
) )
else: else:
if nullable: if nullable:
@@ -202,14 +201,13 @@ def generate_schema_module(
return f"{header}\n\n\n{body}\n" return f"{header}\n\n\n{body}\n"
# ── Main ──────────────────────────────────────────────────────────────────── # ── Main ────────────────────────────────────────────────────────────────────
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB") parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
from conf import cfg, path as _conf_path from conf import cfg
from conf import path as _conf_path
parser.add_argument( parser.add_argument(
"--db", "--db",

View File

@@ -58,7 +58,6 @@ Design notes
from __future__ import annotations from __future__ import annotations
import argparse
import re import re
import textwrap import textwrap
import zipfile import zipfile
@@ -68,7 +67,6 @@ from typing import NamedTuple
import pdfplumber import pdfplumber
# ── Path constants ──────────────────────────────────────────────────────────── # ── Path constants ────────────────────────────────────────────────────────────
from conf import ROOT as _ROOT from conf import ROOT as _ROOT
from conf import path as _conf_path from conf import path as _conf_path
@@ -176,8 +174,9 @@ def _extract_from_zip(
return None return None
def _resolve_pdf(pdf_basename: str, zip_basename: str | None, def _resolve_pdf(
zotero_root: Path, work_dir: Path) -> Path | None: pdf_basename: str, zip_basename: str | None, zotero_root: Path, work_dir: Path
) -> Path | None:
"""Return path to a MIF PDF, extracting from zip if necessary.""" """Return path to a MIF PDF, extracting from zip if necessary."""
direct = _find_in_zotero(pdf_basename, zotero_root) direct = _find_in_zotero(pdf_basename, zotero_root)
if direct: if direct:
@@ -187,8 +186,12 @@ def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
return None return None
def _resolve_xlsx(xlsx_basename: str | None, zip_basename: str | None, def _resolve_xlsx(
zotero_root: Path, work_dir: Path) -> Path | None: xlsx_basename: str | None,
zip_basename: str | None,
zotero_root: Path,
work_dir: Path,
) -> Path | None:
"""Return path to a value set Excel file, extracting from zip if needed.""" """Return path to a value set Excel file, extracting from zip if needed."""
if xlsx_basename is None: if xlsx_basename is None:
return None return None
@@ -329,7 +332,9 @@ def _read_sheet_summaries(xlsx_path: Path) -> dict[str, SheetSummary]:
for row in rows[:3]: for row in rows[:3]:
non_none = [str(c).strip() for c in row if c is not None] non_none = [str(c).strip() for c in row if c is not None]
if len(non_none) >= 2: if len(non_none) >= 2:
header = [str(c).strip().replace("\n", " ") for c in row if c is not None] header = [
str(c).strip().replace("\n", " ") for c in row if c is not None
]
break break
summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header) summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header)
wb.close() wb.close()
@@ -407,13 +412,13 @@ def build_uamcc_passages(
paa4_info = _vs_info("UAMCC PAA4") paa4_info = _vs_info("UAMCC PAA4")
cohort_info = _vs_info("UAMCC Cohort") cohort_info = _vs_info("UAMCC Cohort")
excl_info = _vs_info("UAMCC Exclusions") excl_info = _vs_info("UAMCC Exclusions")
ccs_cm_info = _vs_info("UAMCC CCS-ICD10CM") _vs_info("UAMCC CCS-ICD10CM")
ccs_pcs_info = _vs_info("UAMCC CCS-ICD10PCS") _vs_info("UAMCC CCS-ICD10PCS")
cite_pp = _cite(pdf_name, "§3.8", denom_page) cite_pp = _cite(pdf_name, "§3.8", denom_page)
cite_num = _cite(pdf_name, "§3.63.7", num_page) _cite(pdf_name, "§3.63.7", num_page)
cite_excl = _cite(pdf_name, "§3.10", denom_excl_page) cite_excl = _cite(pdf_name, "§3.10", denom_excl_page)
cite_risk = _cite(pdf_name, "§3.12", risk_page) _cite(pdf_name, "§3.12", risk_page)
cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page) cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page)
passages: dict[str, str] = {} passages: dict[str, str] = {}
@@ -423,13 +428,13 @@ def build_uamcc_passages(
f"Return the UAMCC performance period anchor row.\n\n" f"Return the UAMCC performance period anchor row.\n\n"
f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n" f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n"
f"UAMCC §2.2 Measure Description:\n" f"UAMCC §2.2 Measure Description:\n"
f" \"This outcome measure is calculated using 12 consecutive months\n" f' "This outcome measure is calculated using 12 consecutive months\n'
f" of Medicare fee-for-service (FFS) claims data. The measure is a\n" f" of Medicare fee-for-service (FFS) claims data. The measure is a\n"
f" risk-standardized acute admission rate (RSAAR) that adjusts for\n" f" risk-standardized acute admission rate (RSAAR) that adjusts for\n"
f" age, clinical comorbidities, and other clinical and frailty risk\n" f" age, clinical comorbidities, and other clinical and frailty risk\n"
f" factors present at the start of the 12-month measurement period,\n" f" factors present at the start of the 12-month measurement period,\n"
f" as well as social risk factors. Lower RSAARs indicate better\n" f" as well as social risk factors. Lower RSAARs indicate better\n"
f" performance.\"\n\n" f' performance."\n\n'
f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n" f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n"
f"Measurement duration: 12 consecutive months (Jan 1 Dec 31).\n" f"Measurement duration: 12 consecutive months (Jan 1 Dec 31).\n"
f"Performance Year: {py_version}.\n\n" f"Performance Year: {py_version}.\n\n"
@@ -443,10 +448,10 @@ def build_uamcc_passages(
f"Identify each beneficiary's qualifying chronic condition groups.\n\n" f"Identify each beneficiary's qualifying chronic condition groups.\n\n"
f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n" f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n"
f"UAMCC §3.9 Denominator Details:\n" f"UAMCC §3.9 Denominator Details:\n"
f" \"The cohort is Medicare FFS beneficiaries 66 years of age and\n" f' "The cohort is Medicare FFS beneficiaries 66 years of age and\n'
f" older assigned to the REACH ACO during the measurement period\n" f" older assigned to the REACH ACO during the measurement period\n"
f" with diagnoses that fall into two or more of nine chronic disease\n" f" with diagnoses that fall into two or more of nine chronic disease\n"
f" groups.\"\n\n" f' groups."\n\n'
f"Nine chronic disease groups (MIF §3.9, pp.78):\n" f"Nine chronic disease groups (MIF §3.9, pp.78):\n"
f" 1. Acute myocardial infarction (AMI)\n" f" 1. Acute myocardial infarction (AMI)\n"
f" 2. Alzheimer's disease and related disorders or senile dementia\n" f" 2. Alzheimer's disease and related disorders or senile dementia\n"
@@ -471,7 +476,7 @@ def build_uamcc_passages(
f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n" f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n"
f"Source: {cite_pp}\n\n" f"Source: {cite_pp}\n\n"
f"UAMCC §3.8 Denominator Statement:\n" f"UAMCC §3.8 Denominator Statement:\n"
f" \"{denom_stmt_trimmed[:400]}\"\n\n" f' "{denom_stmt_trimmed[:400]}"\n\n'
f"Inclusion criteria (MIF §3.9):\n" f"Inclusion criteria (MIF §3.9):\n"
f" 1. Age ≥66 at the first day of the measurement period\n" f" 1. Age ≥66 at the first day of the measurement period\n"
f" 2. Two or more distinct MCC groups identified in the lookback year\n" f" 2. Two or more distinct MCC groups identified in the lookback year\n"
@@ -501,13 +506,15 @@ def build_uamcc_passages(
) )
# ── Planned admission (PAA) ─────────────────────────────────────── # ── Planned admission (PAA) ───────────────────────────────────────
paa_snippet = ""
if num_detail: if num_detail:
# Find the PAA description paragraph # Find the PAA description paragraph
m = re.search(r"planned admission algorithm.+?(?=\n\n|\Z)", num_detail, m = re.search(
re.DOTALL | re.IGNORECASE) r"planned admission algorithm.+?(?=\n\n|\Z)",
num_detail,
re.DOTALL | re.IGNORECASE,
)
if m: if m:
paa_snippet = m.group(0)[:600] m.group(0)[:600]
passages["uamcc_int_planned_admission"] = ( passages["uamcc_int_planned_admission"] = (
f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n" f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n"
f"Source: {cite_paa}\n\n" f"Source: {cite_paa}\n\n"
@@ -516,7 +523,7 @@ def build_uamcc_passages(
f" Readmission Algorithm Version 4.0, which CMS originally created\n" f" Readmission Algorithm Version 4.0, which CMS originally created\n"
f" to identify planned readmissions for the hospital-wide readmission\n" f" to identify planned readmissions for the hospital-wide readmission\n"
f" measure. In brief, the algorithm uses a flowchart and four tables\n" f" measure. In brief, the algorithm uses a flowchart and four tables\n"
f" of procedure and/or discharge diagnosis categories.\"\n\n" f' of procedure and/or discharge diagnosis categories."\n\n'
f"PAA Rules (first match wins):\n" f"PAA Rules (first match wins):\n"
f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n" f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n"
f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n" f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n"
@@ -562,14 +569,14 @@ def build_uamcc_passages(
f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n" f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n"
f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n" f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n"
f"UAMCC §3.11 Denominator Exclusion Details:\n" f"UAMCC §3.11 Denominator Exclusion Details:\n"
f" \"Persons are considered at risk for admission if they are alive,\n" f' "Persons are considered at risk for admission if they are alive,\n'
f" enrolled in Medicare FFS, and not admitted to an acute care\n" f" enrolled in Medicare FFS, and not admitted to an acute care\n"
f" hospital. In addition to time spent in the hospital, excluded\n" f" hospital. In addition to time spent in the hospital, excluded\n"
f" from at-risk time are:\n" f" from at-risk time are:\n"
f" (1) time spent in an SNF or acute rehabilitation facility;\n" f" (1) time spent in an SNF or acute rehabilitation facility;\n"
f" (2) time within 10 days following discharge from a hospital,\n" f" (2) time within 10 days following discharge from a hospital,\n"
f" SNF, or acute rehabilitation facility;\n" f" SNF, or acute rehabilitation facility;\n"
f" (3) time after entering hospice care.\"\n\n" f' (3) time after entering hospice care."\n\n'
f"Person-years = at_risk_days / 365.25\n\n" f"Person-years = at_risk_days / 365.25\n\n"
f"Performance Year: {py_version}." f"Performance Year: {py_version}."
) )

View File

@@ -19,7 +19,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
from __future__ import annotations from __future__ import annotations
import re import re
import textwrap
from pathlib import Path from pathlib import Path
import pdfplumber import pdfplumber
@@ -29,10 +28,7 @@ import pdfplumber
_ROOT = Path(__file__).resolve().parents[2] _ROOT = Path(__file__).resolve().parents[2]
_SEEDS = _ROOT / "dev" / "seeds" _SEEDS = _ROOT / "dev" / "seeds"
PDF_PATH = ( PDF_PATH = _SEEDS / "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
_SEEDS
/ "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
)
ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py" ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py"
FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py" FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py"
QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py" QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py"

View File

@@ -21,7 +21,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
from __future__ import annotations from __future__ import annotations
import re
from pathlib import Path from pathlib import Path
# Define all table specifications based on extracted data dictionary # Define all table specifications based on extracted data dictionary

View File

@@ -336,7 +336,9 @@ def main():
"""Generate REACH participants table model and rex parser.""" """Generate REACH participants table model and rex parser."""
project_root = Path(__file__).resolve().parents[2] project_root = Path(__file__).resolve().parents[2]
xlsx_path = ( xlsx_path = (
project_root / "dev" / "seeds" project_root
/ "dev"
/ "seeds"
/ "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx" / "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx"
) )

View File

@@ -79,15 +79,12 @@ def sanitize_field(name: str) -> str:
def get_tables(con: sqlite3.Connection) -> list[str]: def get_tables(con: sqlite3.Connection) -> list[str]:
"""Return all user tables, sorted alphabetically.""" """Return all user tables, sorted alphabetically."""
rows = con.execute( rows = con.execute(
"SELECT name FROM sqlite_master WHERE type='table' " "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
"ORDER BY name"
).fetchall() ).fetchall()
return [r[0] for r in rows] return [r[0] for r in rows]
def get_columns( def get_columns(con: sqlite3.Connection, table: str) -> list[tuple[str, str, bool]]:
con: sqlite3.Connection, table: str
) -> list[tuple[str, str, bool]]:
"""Return [(col_name, sqlite_type, nullable), ...]. """Return [(col_name, sqlite_type, nullable), ...].
SQLite PRAGMA table_info returns: SQLite PRAGMA table_info returns:

View File

@@ -63,9 +63,7 @@ def _get_item_tags(zcon: sqlite3.Connection, item_id: int) -> list[str]:
return [r["name"] for r in rows] return [r["name"] for r in rows]
def _get_item_collections( def _get_item_collections(zcon: sqlite3.Connection, item_id: int) -> list[str]:
zcon: sqlite3.Connection, item_id: int
) -> list[str]:
"""Read collection keys for a Zotero item.""" """Read collection keys for a Zotero item."""
rows = zcon.execute( rows = zcon.execute(
"""SELECT c.key FROM collectionItems ci """SELECT c.key FROM collectionItems ci
@@ -76,9 +74,7 @@ def _get_item_collections(
return [r["key"] for r in rows] return [r["key"] for r in rows]
def _get_item_creators( def _get_item_creators(zcon: sqlite3.Connection, item_id: int) -> list[dict]:
zcon: sqlite3.Connection, item_id: int
) -> list[dict]:
"""Read creators for a Zotero item.""" """Read creators for a Zotero item."""
rows = zcon.execute( rows = zcon.execute(
"""SELECT c.firstName, c.lastName, ct.creatorType, """SELECT c.firstName, c.lastName, ct.creatorType,
@@ -101,9 +97,7 @@ def _get_item_creators(
] ]
def _classify_item( def _classify_item(zotero_type: str, fields: dict) -> str:
zotero_type: str, fields: dict
) -> str:
"""Map Zotero itemType + fields to bib item_type.""" """Map Zotero itemType + fields to bib item_type."""
if zotero_type == "statute": if zotero_type == "statute":
code = fields.get("code", "") code = fields.get("code", "")
@@ -134,14 +128,16 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
rule_type = part[6:] rule_type = part[6:]
elif part.startswith("Effective: "): elif part.startswith("Effective: "):
eff_date = part[11:] eff_date = part[11:]
return json.dumps({ return json.dumps(
{
"fr_volume": fields.get("codeNumber", ""), "fr_volume": fields.get("codeNumber", ""),
"fr_page": fields.get("pages", ""), "fr_page": fields.get("pages", ""),
"document_number": doc_num, "document_number": doc_num,
"cms_id": fields.get("session", ""), "cms_id": fields.get("session", ""),
"rule_type": rule_type, "rule_type": rule_type,
"effective_date": eff_date, "effective_date": eff_date,
}) }
)
if item_type == "regulation": if item_type == "regulation":
history = fields.get("history", "") history = fields.get("history", "")
@@ -152,13 +148,15 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
part = segment[5:] part = segment[5:]
elif segment.startswith("Authority: "): elif segment.startswith("Authority: "):
authority = segment[11:] authority = segment[11:]
return json.dumps({ return json.dumps(
{
"cfr_title": fields.get("codeNumber", ""), "cfr_title": fields.get("codeNumber", ""),
"cfr_part": part, "cfr_part": part,
"cfr_section": fields.get("section", ""), "cfr_section": fields.get("section", ""),
"authority": authority, "authority": authority,
"effective_date": fields.get("dateEnacted", ""), "effective_date": fields.get("dateEnacted", ""),
}) }
)
if item_type == "manual": if item_type == "manual":
extra = fields.get("extra", "") extra = fields.get("extra", "")
@@ -169,33 +167,37 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
chapter = fields.get("seriesNumber", "") chapter = fields.get("seriesNumber", "")
if chapter.startswith("Chapter "): if chapter.startswith("Chapter "):
chapter = chapter[8:] chapter = chapter[8:]
return json.dumps({ return json.dumps(
{
"manual_name": fields.get("seriesTitle", ""), "manual_name": fields.get("seriesTitle", ""),
"pub_number": fields.get("reportNumber", ""), "pub_number": fields.get("reportNumber", ""),
"chapter": chapter, "chapter": chapter,
"transmittal": transmittal, "transmittal": transmittal,
}) }
)
if item_type == "download": if item_type == "download":
extra = fields.get("extra", "") extra = fields.get("extra", "")
file_urls: list[str] = [] file_urls: list[str] = []
for line in extra.split("\n"): for line in extra.split("\n"):
if line.startswith("Files: "): if line.startswith("Files: "):
file_urls = [ file_urls = [u.strip() for u in line[7:].split(";") if u.strip()]
u.strip() for u in line[7:].split(";") if u.strip() return json.dumps(
] {
return json.dumps({
"page_type": "", "page_type": "",
"file_urls": file_urls, "file_urls": file_urls,
"year": None, "year": None,
"quarter": "", "quarter": "",
"website_title": fields.get("websiteTitle", ""), "website_title": fields.get("websiteTitle", ""),
}) }
)
if item_type == "source": if item_type == "source":
return json.dumps({ return json.dumps(
{
"doc_type": fields.get("type", ""), "doc_type": fields.get("type", ""),
}) }
)
return "{}" return "{}"
@@ -255,8 +257,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
parent_id = row["id"] parent_id = row["id"]
bcon.execute( bcon.execute(
"INSERT OR IGNORE INTO collections (key, name, parent_id) " "INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
"VALUES (?, ?, ?)",
(zc["key"], zc["collectionName"], parent_id), (zc["key"], zc["collectionName"], parent_id),
) )
counts["collections"] += 1 counts["collections"] += 1
@@ -322,8 +323,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
for tag_name in ztags: for tag_name in ztags:
tag_id = store._ensure_tag(tag_name) tag_id = store._ensure_tag(tag_name)
bcon.execute( bcon.execute(
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) " "INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
"VALUES (?, ?)",
(bib_id, tag_id), (bib_id, tag_id),
) )
counts["tags"] += 1 counts["tags"] += 1
@@ -346,16 +346,14 @@ def migrate(src: str, dst: str) -> dict[str, int]:
for cr in creators: for cr in creators:
# Find or create creator # Find or create creator
crow = bcon.execute( crow = bcon.execute(
"SELECT id FROM creators " "SELECT id FROM creators WHERE first_name = ? AND last_name = ?",
"WHERE first_name = ? AND last_name = ?",
(cr["first_name"], cr["last_name"]), (cr["first_name"], cr["last_name"]),
).fetchone() ).fetchone()
if crow: if crow:
creator_id = crow["id"] creator_id = crow["id"]
else: else:
cur = bcon.execute( cur = bcon.execute(
"INSERT INTO creators (first_name, last_name) " "INSERT INTO creators (first_name, last_name) VALUES (?, ?)",
"VALUES (?, ?)",
(cr["first_name"], cr["last_name"]), (cr["first_name"], cr["last_name"]),
) )
creator_id = cur.lastrowid creator_id = cur.lastrowid
@@ -462,9 +460,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
def main() -> None: def main() -> None:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(description="Migrate Zotero SQLite to bib SQLite")
description="Migrate Zotero SQLite to bib SQLite"
)
from conf import path as _conf_path from conf import path as _conf_path
parser.add_argument( parser.add_argument(

View File

@@ -60,7 +60,6 @@ from typing import Any
import polars as pl import polars as pl
# ── Path constants ──────────────────────────────────────────────────────────── # ── Path constants ────────────────────────────────────────────────────────────
from conf import ROOT as _ROOT from conf import ROOT as _ROOT
from conf import path as _conf_path from conf import path as _conf_path

View File

@@ -319,16 +319,12 @@ def _ensure_value(con: sqlite3.Connection, value: str) -> int:
).fetchone() ).fetchone()
if row: if row:
return row[0] return row[0]
cur = con.execute( cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
"INSERT INTO itemDataValues (value) VALUES (?)", (value,)
)
return cur.lastrowid return cur.lastrowid
def _ensure_tag(con: sqlite3.Connection, name: str) -> int: def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
row = con.execute( row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
"SELECT tagID FROM tags WHERE name = ?", (name,)
).fetchone()
if row: if row:
return row[0] return row[0]
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,)) cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
@@ -342,8 +338,7 @@ def _set_zotero_field(
return return
value_id = _ensure_value(con, value) value_id = _ensure_value(con, value)
con.execute( con.execute(
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) " "INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
"VALUES (?, ?, ?)",
(item_id, field_id, value_id), (item_id, field_id, value_id),
) )
@@ -392,8 +387,7 @@ def tag_existing_zotero(
for tag_name in all_tags: for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name) tag_id = _ensure_tag(con, tag_name)
con.execute( con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) " "INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
"VALUES (?, ?, 0)",
(parent_id, tag_id), (parent_id, tag_id),
) )
return True return True
@@ -416,9 +410,9 @@ def register_in_zotero(
return return
now = _now_iso() now = _now_iso()
next_id = con.execute( next_id = con.execute("SELECT COALESCE(MAX(itemID), 0) + 1 FROM items").fetchone()[
"SELECT COALESCE(MAX(itemID), 0) + 1 FROM items" 0
).fetchone()[0] ]
parent_key = _zotero_key() parent_key = _zotero_key()
# Create parent webpage item # Create parent webpage item
@@ -434,9 +428,7 @@ def register_in_zotero(
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url) _set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url)
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10]) _set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now) _set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
_set_zotero_field( _set_zotero_field(con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal")
con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal"
)
_set_zotero_field( _set_zotero_field(
con, con,
next_id, next_id,
@@ -449,8 +441,7 @@ def register_in_zotero(
for tag_name in all_tags: for tag_name in all_tags:
tag_id = _ensure_tag(con, tag_name) tag_id = _ensure_tag(con, tag_name)
con.execute( con.execute(
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) " "INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
"VALUES (?, ?, 0)",
(next_id, tag_id), (next_id, tag_id),
) )
@@ -464,13 +455,9 @@ def register_in_zotero(
# Copy to Zotero storage # Copy to Zotero storage
storage_dir = ZOTERO_STORAGE / att_key storage_dir = ZOTERO_STORAGE / att_key
subprocess.run( subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
["sudo", "mkdir", "-p", str(storage_dir)], check=True
)
dest = storage_dir / seed_path.name dest = storage_dir / seed_path.name
subprocess.run( subprocess.run(["sudo", "cp", str(seed_path), str(dest)], check=True)
["sudo", "cp", str(seed_path), str(dest)], check=True
)
subprocess.run( subprocess.run(
["sudo", "chown", "-R", "100999:100999", str(storage_dir)], ["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
check=True, check=True,
@@ -496,8 +483,7 @@ def register_in_zotero(
) )
title_val = _ensure_value(con, seed_path.name) title_val = _ensure_value(con, seed_path.name)
con.execute( con.execute(
"INSERT INTO itemData (itemID, fieldID, valueID) " "INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
"VALUES (?, ?, ?)",
(att_id, ZOTERO_FIELD_TITLE, title_val), (att_id, ZOTERO_FIELD_TITLE, title_val),
) )
@@ -567,7 +553,7 @@ def pull_from_zotero(*, dry_run: bool = False) -> int:
zpath = row[0] # "storage:filename" zpath = row[0] # "storage:filename"
if not zpath.startswith("storage:"): if not zpath.startswith("storage:"):
continue continue
filename = zpath[len("storage:"):] filename = zpath[len("storage:") :]
dest = SEEDS_DIR / filename dest = SEEDS_DIR / filename
if dest.exists(): if dest.exists():

View File

@@ -97,11 +97,11 @@ def main():
print(f" ... and {len(tables) - 3} more tables") print(f" ... and {len(tables) - 3} more tables")
print("\n" + "=" * 70) print("\n" + "=" * 70)
print(f"\nTo create these in Unity Catalog, run without --show-schemas flag") print("\nTo create these in Unity Catalog, run without --show-schemas flag")
return return
# Create Unity client # Create Unity client
print(f"Connecting to Unity Catalog...") print("Connecting to Unity Catalog...")
print(f" Workspace ID: {args.workspace_id}") print(f" Workspace ID: {args.workspace_id}")
print(f" Catalog: {args.catalog}") print(f" Catalog: {args.catalog}")
print() print()
@@ -114,7 +114,7 @@ def main():
# Test connection # Test connection
try: try:
catalogs = client.list_catalogs() catalogs = client.list_catalogs()
print(f"✓ Connected to Unity Catalog") print("✓ Connected to Unity Catalog")
print(f" Found {len(catalogs)} existing catalogs:") print(f" Found {len(catalogs)} existing catalogs:")
for cat in catalogs: for cat in catalogs:
print(f" - {cat.name}") print(f" - {cat.name}")
@@ -130,7 +130,7 @@ def main():
print("=" * 70) print("=" * 70)
print() print()
print(f"Setting up catalog structure...") print("Setting up catalog structure...")
print() print()
report = setup_catalog_from_schemas( report = setup_catalog_from_schemas(

View File

@@ -20,11 +20,11 @@ fi
# Image definitions — matches stack.toml [images] # Image definitions — matches stack.toml [images]
declare -A DOCKERFILES=( declare -A DOCKERFILES=(
[api]="api/Dockerfile" [api]="infra/images/api.Dockerfile"
[notebooks]="notebooks/Dockerfile" [notebooks]="infra/images/notebooks.Dockerfile"
[docs]="docs/Dockerfile" [docs]="infra/images/docs.Dockerfile"
[zotero]="zotero/Dockerfile" [zotero]="infra/images/zotero.Dockerfile"
[mc]="rustfs/Dockerfile.mc" [mc]="infra/images/mc.Dockerfile"
) )
declare -A CONTEXTS=( declare -A CONTEXTS=(
@@ -32,7 +32,7 @@ declare -A CONTEXTS=(
[notebooks]="notebooks/" [notebooks]="notebooks/"
[docs]="." [docs]="."
[zotero]="zotero/" [zotero]="zotero/"
[mc]="rustfs/" [mc]="infra/rustfs/"
) )
# Select images to test # Select images to test

View File

@@ -23,7 +23,6 @@ import os
from aco.lake import IcebergContext, UnityClient from aco.lake import IcebergContext, UnityClient
from aco.lake.catalog import Catalog from aco.lake.catalog import Catalog
from aco.lake.engine import execute
def example_unity_client_basics(): def example_unity_client_basics():
@@ -185,7 +184,7 @@ def example_iceberg_context():
}, },
) )
print(f"✓ IcebergContext configured") print("✓ IcebergContext configured")
print(f" Catalog URI: {ctx.catalog_uri}") print(f" Catalog URI: {ctx.catalog_uri}")
print(f" Warehouse: {ctx.warehouse}") print(f" Warehouse: {ctx.warehouse}")
print() print()
@@ -222,7 +221,7 @@ def example_pipeline_execution():
} }
) )
ctx = IcebergContext( IcebergContext(
catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg", catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg",
warehouse="aco_dev", warehouse="aco_dev",
catalog=catalog, catalog=catalog,

View File

@@ -1,4 +1,4 @@
/* LOCH — deep-navy design system — matches styles/inject.css */ /* LOCH — deep-navy design system — matches assets/css/inject.css */
:root { :root {
--ifm-color-primary: #4488dd; --ifm-color-primary: #4488dd;

0
infra/polaris/.gitkeep Normal file
View File

View File

@@ -17,7 +17,7 @@ Usage::
CLI:: CLI::
uv run python -m aco.dag uv run python -m aco.dag
uv run python -m aco.dag -p readmissions --html dag.html uv run python -m aco.dag -p readmissions --html assets/dag.html
uv run python -m aco.dag --dot dag.dot uv run python -m aco.dag --dot dag.dot
uv run python -m aco.dag -p pharmacy --mermaid uv run python -m aco.dag -p pharmacy --mermaid
""" """