chore: clean sweep — lint, format, stale refs, generated artifacts
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled
- Fix all 72 ruff lint errors (unused imports, unused variables, E402) - Format all 14 unformatted dev/scripts files - Move generated artifacts to assets/ (dag.html, pfs.html) - Remove duplicate root coverage.svg (already in assets/icons/) - Update .dockerignore for infra/ tree layout - Update .gitignore: add .env.bak, mirrors/, htmlcov/ - Fix stale path refs in coverage_badge.py, woodpecker backend, test_network_isolation.sh, docs custom.css - Add .gitkeep to empty dirs (infra/polaris, cloud/*/terraform) - Delete 12 stale local branches, 10 stale remote branches
This commit is contained in:
@@ -12,13 +12,8 @@ logs/
|
||||
tuva/
|
||||
dev/
|
||||
tests/
|
||||
grafana/
|
||||
prometheus/
|
||||
loki/
|
||||
trino/
|
||||
traefik/
|
||||
nginx/
|
||||
gitea/
|
||||
styles/
|
||||
.woodpecker/
|
||||
infra/
|
||||
assets/
|
||||
cloud/
|
||||
mirrors/
|
||||
.claude/
|
||||
|
||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -13,17 +13,28 @@ bundle/sql/
|
||||
|
||||
# Marimo cache
|
||||
notebooks/__marimo__/
|
||||
|
||||
# Secrets and keys
|
||||
.deploy_key
|
||||
.env
|
||||
.deploy_key.pub
|
||||
.env
|
||||
.env.bak
|
||||
|
||||
# Python build artifacts
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.coverage
|
||||
dist/
|
||||
*.egg-info/
|
||||
htmlcov/
|
||||
|
||||
# IDE / tool settings
|
||||
.claude/settings.local.json
|
||||
notebooks/aco.duckdb
|
||||
|
||||
# Package mirrors (local cache, not tracked)
|
||||
mirrors/
|
||||
|
||||
# Docs (auto-generated at build time)
|
||||
docs/docs/api/
|
||||
docs/static/library.json
|
||||
|
||||
0
cloud/aws/terraform/.gitkeep
Normal file
0
cloud/aws/terraform/.gitkeep
Normal file
0
cloud/azure/terraform/.gitkeep
Normal file
0
cloud/azure/terraform/.gitkeep
Normal file
0
cloud/gcp/terraform/.gitkeep
Normal file
0
cloud/gcp/terraform/.gitkeep
Normal file
20
coverage.svg
20
coverage.svg
@@ -1,20 +0,0 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="116" height="20">
|
||||
<linearGradient id="b" x2="0" y2="100%">
|
||||
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
|
||||
<stop offset="1" stop-opacity=".1"/>
|
||||
</linearGradient>
|
||||
<clipPath id="a">
|
||||
<rect width="116" height="20" rx="3" fill="#fff"/>
|
||||
</clipPath>
|
||||
<g clip-path="url(#a)">
|
||||
<path fill="#555" d="M0 0h65v20H0z"/>
|
||||
<path fill="#4c1" d="M65 0h51v20H65z"/>
|
||||
<path fill="url(#b)" d="M0 0h116v20H0z"/>
|
||||
</g>
|
||||
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
|
||||
<text x="32.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
|
||||
<text x="32.5" y="14">coverage</text>
|
||||
<text x="90" y="15" fill="#010101" fill-opacity=".3">99%</text>
|
||||
<text x="90" y="14">99%</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
Before Width: | Height: | Size: 822 B |
@@ -7,11 +7,8 @@ Usage:
|
||||
uv run python dev/scripts/add_carrier_to_zotero.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import sqlite3
|
||||
import string
|
||||
import subprocess
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -9,7 +9,6 @@ Usage:
|
||||
|
||||
import random
|
||||
import sqlite3
|
||||
import string
|
||||
import subprocess
|
||||
import zipfile
|
||||
from datetime import datetime, timezone
|
||||
|
||||
@@ -30,8 +30,11 @@ yaml.add_representer(
|
||||
|
||||
def _dump(data: dict) -> str:
|
||||
return yaml.dump(
|
||||
data, default_flow_style=False, sort_keys=False,
|
||||
width=120, allow_unicode=True,
|
||||
data,
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
width=120,
|
||||
allow_unicode=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -69,9 +72,7 @@ class SQLPlugin(DabPlugin):
|
||||
output_mode="ctas",
|
||||
)
|
||||
except Exception as e:
|
||||
result[f"bundle/sql/{pipe_name}/_error.txt"] = (
|
||||
f"Transpile failed: {e}"
|
||||
)
|
||||
result[f"bundle/sql/{pipe_name}/_error.txt"] = f"Transpile failed: {e}"
|
||||
continue
|
||||
|
||||
for expr_name, sql in sql_map.items():
|
||||
@@ -80,8 +81,7 @@ class SQLPlugin(DabPlugin):
|
||||
schema, table = expr_name.split(".", 1)
|
||||
path = f"bundle/sql/{pipe_name}/{table}.sql"
|
||||
header = (
|
||||
f"-- {expr_name}\n"
|
||||
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
||||
f"-- {expr_name}\n-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
||||
)
|
||||
result[path] = header + sql + "\n"
|
||||
|
||||
@@ -102,27 +102,58 @@ class JobsPlugin(DabPlugin):
|
||||
job_clusters = []
|
||||
for profile_name in sorted(used_profiles):
|
||||
cluster_spec = dict(clusters_cfg.get(profile_name, {}))
|
||||
job_clusters.append(OrderedDict([
|
||||
("job_cluster_key", f"{profile_name}_cluster"),
|
||||
("new_cluster", OrderedDict([
|
||||
("spark_version", cluster_spec.get("spark_version", "15.4.x-scala2.12")),
|
||||
("node_type_id", cluster_spec.get("node_type_id", "i3.xlarge")),
|
||||
("num_workers", cluster_spec.get("num_workers", 1)),
|
||||
])),
|
||||
]))
|
||||
job_clusters.append(
|
||||
OrderedDict(
|
||||
[
|
||||
("job_cluster_key", f"{profile_name}_cluster"),
|
||||
(
|
||||
"new_cluster",
|
||||
OrderedDict(
|
||||
[
|
||||
(
|
||||
"spark_version",
|
||||
cluster_spec.get(
|
||||
"spark_version", "15.4.x-scala2.12"
|
||||
),
|
||||
),
|
||||
(
|
||||
"node_type_id",
|
||||
cluster_spec.get("node_type_id", "i3.xlarge"),
|
||||
),
|
||||
("num_workers", cluster_spec.get("num_workers", 1)),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# DDL tasks — one per schema to create tables before SQL runs
|
||||
ddl_tasks = []
|
||||
for schema_name in sorted(catalog_schemas):
|
||||
ddl_tasks.append(OrderedDict([
|
||||
("task_key", f"ddl_{schema_name}"),
|
||||
("job_cluster_key", "default_cluster"),
|
||||
("sql_task", OrderedDict([
|
||||
("file", OrderedDict([
|
||||
("path", f"bundle/ddl/{schema_name}/"),
|
||||
])),
|
||||
])),
|
||||
]))
|
||||
ddl_tasks.append(
|
||||
OrderedDict(
|
||||
[
|
||||
("task_key", f"ddl_{schema_name}"),
|
||||
("job_cluster_key", "default_cluster"),
|
||||
(
|
||||
"sql_task",
|
||||
OrderedDict(
|
||||
[
|
||||
(
|
||||
"file",
|
||||
OrderedDict(
|
||||
[
|
||||
("path", f"bundle/ddl/{schema_name}/"),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# Pipeline tasks — each expression becomes a sql_task
|
||||
pipeline_tasks = []
|
||||
@@ -143,12 +174,19 @@ class JobsPlugin(DabPlugin):
|
||||
# Each pipeline task runs all its SQL files in order
|
||||
# We use a for_each or sequential notebook; simplest is
|
||||
# a single SQL file per pipeline that sources all expressions
|
||||
task["sql_task"] = OrderedDict([
|
||||
("file", OrderedDict([
|
||||
("path", f"bundle/sql/{pipe_name}/"),
|
||||
])),
|
||||
("warehouse_id", "${var.warehouse_id}"),
|
||||
])
|
||||
task["sql_task"] = OrderedDict(
|
||||
[
|
||||
(
|
||||
"file",
|
||||
OrderedDict(
|
||||
[
|
||||
("path", f"bundle/sql/{pipe_name}/"),
|
||||
]
|
||||
),
|
||||
),
|
||||
("warehouse_id", "${var.warehouse_id}"),
|
||||
]
|
||||
)
|
||||
pipeline_tasks.append(task)
|
||||
|
||||
schedule = cfg.get("schedule", "0 0 6 * * ?")
|
||||
@@ -156,11 +194,15 @@ class JobsPlugin(DabPlugin):
|
||||
|
||||
job = OrderedDict()
|
||||
job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines"
|
||||
job["description"] = f"Run all {len(registry)} ACO pipelines in dependency order"
|
||||
job["schedule"] = OrderedDict([
|
||||
("quartz_cron_expression", schedule),
|
||||
("timezone_id", timezone),
|
||||
])
|
||||
job["description"] = (
|
||||
f"Run all {len(registry)} ACO pipelines in dependency order"
|
||||
)
|
||||
job["schedule"] = OrderedDict(
|
||||
[
|
||||
("quartz_cron_expression", schedule),
|
||||
("timezone_id", timezone),
|
||||
]
|
||||
)
|
||||
job["job_clusters"] = job_clusters
|
||||
job["tasks"] = pipeline_tasks
|
||||
|
||||
@@ -175,10 +217,12 @@ class SchemasPlugin(DabPlugin):
|
||||
return {}
|
||||
schemas = OrderedDict()
|
||||
for schema_name in sorted(catalog_schemas):
|
||||
schemas[schema_name] = OrderedDict([
|
||||
("name", schema_name),
|
||||
("catalog_name", "${var.catalog}"),
|
||||
])
|
||||
schemas[schema_name] = OrderedDict(
|
||||
[
|
||||
("name", schema_name),
|
||||
("catalog_name", "${var.catalog}"),
|
||||
]
|
||||
)
|
||||
return {"schemas": schemas}
|
||||
|
||||
|
||||
@@ -186,14 +230,23 @@ class VolumesPlugin(DabPlugin):
|
||||
"""Generate a staging volume for data upload."""
|
||||
|
||||
def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict:
|
||||
return {"volumes": OrderedDict([
|
||||
("staging", OrderedDict([
|
||||
("name", "staging"),
|
||||
("catalog_name", "${var.catalog}"),
|
||||
("schema_name", "default"),
|
||||
("volume_type", "MANAGED"),
|
||||
])),
|
||||
])}
|
||||
return {
|
||||
"volumes": OrderedDict(
|
||||
[
|
||||
(
|
||||
"staging",
|
||||
OrderedDict(
|
||||
[
|
||||
("name", "staging"),
|
||||
("catalog_name", "${var.catalog}"),
|
||||
("schema_name", "default"),
|
||||
("volume_type", "MANAGED"),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
# ── Core generator ───────────────────────────────────────────────
|
||||
@@ -214,6 +267,7 @@ _DEFAULT_PLUGINS: list[DabPlugin] = [
|
||||
def _get_catalog_schemas() -> list[str]:
|
||||
try:
|
||||
from aco.lake.catalog import Catalog
|
||||
|
||||
return Catalog().schemas()
|
||||
except Exception:
|
||||
return []
|
||||
@@ -252,19 +306,33 @@ def emit(cfg_data: dict) -> dict[str, str]:
|
||||
bundle["bundle"] = OrderedDict([("name", bundle_name)])
|
||||
|
||||
# Sync SQL files to workspace
|
||||
bundle["sync"] = OrderedDict([
|
||||
("include", ["bundle/**"]),
|
||||
])
|
||||
bundle["sync"] = OrderedDict(
|
||||
[
|
||||
("include", ["bundle/**"]),
|
||||
]
|
||||
)
|
||||
|
||||
bundle["variables"] = OrderedDict([
|
||||
("catalog", OrderedDict([
|
||||
("description", "Unity Catalog name for table references"),
|
||||
("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
|
||||
])),
|
||||
("warehouse_id", OrderedDict([
|
||||
("description", "SQL warehouse ID for query execution"),
|
||||
])),
|
||||
])
|
||||
bundle["variables"] = OrderedDict(
|
||||
[
|
||||
(
|
||||
"catalog",
|
||||
OrderedDict(
|
||||
[
|
||||
("description", "Unity Catalog name for table references"),
|
||||
("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
|
||||
]
|
||||
),
|
||||
),
|
||||
(
|
||||
"warehouse_id",
|
||||
OrderedDict(
|
||||
[
|
||||
("description", "SQL warehouse ID for query execution"),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")])
|
||||
|
||||
@@ -276,17 +344,21 @@ def emit(cfg_data: dict) -> dict[str, str]:
|
||||
target["mode"] = tcfg["mode"]
|
||||
if tcfg.get("default"):
|
||||
target["default"] = True
|
||||
target["variables"] = OrderedDict([
|
||||
("catalog", tcfg.get("catalog", "aco")),
|
||||
])
|
||||
target["variables"] = OrderedDict(
|
||||
[
|
||||
("catalog", tcfg.get("catalog", "aco")),
|
||||
]
|
||||
)
|
||||
if tcfg.get("warehouse_id"):
|
||||
target["variables"]["warehouse_id"] = tcfg["warehouse_id"]
|
||||
if tcfg.get("mode") == "production":
|
||||
run_as = dab_cfg.get("run_as", {})
|
||||
if run_as.get("service_principal_name"):
|
||||
target["run_as"] = OrderedDict([
|
||||
("service_principal_name", run_as["service_principal_name"]),
|
||||
])
|
||||
target["run_as"] = OrderedDict(
|
||||
[
|
||||
("service_principal_name", run_as["service_principal_name"]),
|
||||
]
|
||||
)
|
||||
targets[tname] = target
|
||||
bundle["targets"] = targets
|
||||
|
||||
|
||||
@@ -242,7 +242,7 @@ steps:
|
||||
UV_PROJECT_ENVIRONMENT: .venv
|
||||
commands:
|
||||
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
|
||||
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg
|
||||
- uv run python dev/scripts/coverage_badge.py < pytest.out > assets/icons/coverage.svg
|
||||
depends_on:
|
||||
- lint
|
||||
|
||||
@@ -273,7 +273,7 @@ steps:
|
||||
commands:
|
||||
- git config user.name woodpecker-ci
|
||||
- git config user.email ci@${{CI_REPO_OWNER}}.io
|
||||
- git add coverage.svg
|
||||
- git add assets/icons/coverage.svg
|
||||
- git diff --cached --quiet && echo "no change" && exit 0
|
||||
- git commit -m "update coverage badge [skip ci]"
|
||||
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
|
||||
@@ -293,7 +293,7 @@ steps:
|
||||
from_secret: s3_access_key
|
||||
secret_key:
|
||||
from_secret: s3_secret_key
|
||||
source: "coverage.svg"
|
||||
source: "assets/icons/coverage.svg"
|
||||
target: /badges/${{CI_REPO}}/
|
||||
path_style: true
|
||||
overwrite: true
|
||||
|
||||
@@ -15,7 +15,6 @@ import argparse
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -33,7 +32,7 @@ CA_SUBJECT = "/CN=Homelab CA/O=fhirworx"
|
||||
CA_DAYS = 3650
|
||||
TLS_DAYS = 3650
|
||||
|
||||
from conf import cfg
|
||||
from conf import cfg # noqa: E402
|
||||
|
||||
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
|
||||
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)
|
||||
|
||||
@@ -8,7 +8,7 @@ Usage:
|
||||
# CI_COMMIT_SHA from environment):
|
||||
uv run python dev/scripts/coverage_badge.py --post pytest.out
|
||||
|
||||
# Update styles/coverage.svg in the repo via Gitea API:
|
||||
# Update assets/icons/coverage.svg in the repo via Gitea API:
|
||||
uv run python dev/scripts/coverage_badge.py --upload coverage.svg
|
||||
"""
|
||||
|
||||
@@ -44,7 +44,7 @@ TEMPLATE = """\
|
||||
</g>
|
||||
</svg>"""
|
||||
|
||||
FILE_PATH = "styles/coverage.svg"
|
||||
FILE_PATH = "assets/icons/coverage.svg"
|
||||
|
||||
|
||||
def color_for(pct: int) -> str:
|
||||
@@ -73,11 +73,13 @@ def post_status(pct: int) -> None:
|
||||
token, url = _gitea_env()
|
||||
sha = os.environ["CI_COMMIT_SHA"]
|
||||
endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}"
|
||||
body = json.dumps({
|
||||
"state": "success",
|
||||
"description": f"{pct}% coverage",
|
||||
"context": "coverage",
|
||||
}).encode()
|
||||
body = json.dumps(
|
||||
{
|
||||
"state": "success",
|
||||
"description": f"{pct}% coverage",
|
||||
"context": "coverage",
|
||||
}
|
||||
).encode()
|
||||
req = urllib.request.Request(
|
||||
endpoint,
|
||||
data=body,
|
||||
@@ -92,7 +94,7 @@ def post_status(pct: int) -> None:
|
||||
|
||||
|
||||
def upload_badge(svg_path: str) -> None:
|
||||
"""Update styles/coverage.svg in the repo via Gitea contents API."""
|
||||
"""Update assets/icons/coverage.svg in the repo via Gitea contents API."""
|
||||
token, url = _gitea_env()
|
||||
endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}"
|
||||
svg_data = open(svg_path, "rb").read()
|
||||
|
||||
@@ -40,7 +40,6 @@ def main() -> int:
|
||||
for col in COLUMNS:
|
||||
for op in ("INSERT", "UPDATE"):
|
||||
trigger = f"fix_{col}_{op.lower()}"
|
||||
of_clause = f"OF {col} " if op == "UPDATE" else ""
|
||||
db.execute(f"""
|
||||
CREATE TRIGGER IF NOT EXISTS {trigger}
|
||||
AFTER {op} ON items
|
||||
|
||||
@@ -219,7 +219,11 @@ def generate_module() -> str:
|
||||
def main():
|
||||
"""Generate src/aco/table/alr_filenames.py."""
|
||||
output_path = (
|
||||
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "alr_filenames.py"
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "src"
|
||||
/ "aco"
|
||||
/ "table"
|
||||
/ "alr_filenames.py"
|
||||
)
|
||||
|
||||
code = generate_module()
|
||||
@@ -228,9 +232,9 @@ def main():
|
||||
output_path.write_text(code)
|
||||
|
||||
print(f"Generated {output_path}")
|
||||
print(f" 18 ALR table patterns (9 annual + 9 quarterly)")
|
||||
print(f" 2 ASR patterns (annual + quarterly)")
|
||||
print(f" 4 package patterns")
|
||||
print(" 18 ALR table patterns (9 annual + 9 quarterly)")
|
||||
print(" 2 ASR patterns (annual + quarterly)")
|
||||
print(" 4 package patterns")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
|
||||
|
||||
Returns dict mapping table_id -> {title, fields: list[dict]}
|
||||
"""
|
||||
pdf = pdfplumber.open(pdf_path)
|
||||
pdfplumber.open(pdf_path)
|
||||
|
||||
# Manual field specifications based on PDF extraction
|
||||
# Table 1-1 is on pages 8-15, others follow
|
||||
@@ -163,7 +163,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
|
||||
{
|
||||
"name": f"HCC_Position_{i:02d}",
|
||||
"type": "int",
|
||||
"desc": f"HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
|
||||
"desc": "HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -530,7 +530,7 @@ def generate_module(tables: dict) -> str:
|
||||
lines.append(" Source: ALRASRGuide.pdf")
|
||||
lines.append(' """')
|
||||
lines.append("")
|
||||
lines.append(f' __schema__ = "alr"')
|
||||
lines.append(' __schema__ = "alr"')
|
||||
lines.append(f' __tablename__ = "{tbl}"')
|
||||
|
||||
for field in info["fields"]:
|
||||
|
||||
@@ -370,7 +370,7 @@ def generate_module(tables: dict) -> str:
|
||||
lines.append(" Source: ALRASRGuide.pdf")
|
||||
lines.append(' """')
|
||||
lines.append("")
|
||||
lines.append(f' __schema__ = "asr"')
|
||||
lines.append(' __schema__ = "asr"')
|
||||
lines.append(f' __tablename__ = "{tbl}"')
|
||||
|
||||
for field in info["fields"]:
|
||||
@@ -398,7 +398,9 @@ def generate_module(tables: dict) -> str:
|
||||
|
||||
def main():
|
||||
"""Generate src/aco/table/asr.py from ALRASRGuide.pdf."""
|
||||
output_path = Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
|
||||
output_path = (
|
||||
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
|
||||
)
|
||||
|
||||
tables = extract_asr_table_specs()
|
||||
code = generate_module(tables)
|
||||
|
||||
@@ -14,7 +14,6 @@ Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
@@ -206,7 +205,7 @@ def generate_module(tables: dict) -> str:
|
||||
lines.append(f" {len(info['fields'])} fields, fixed-width layout.")
|
||||
lines.append(' """')
|
||||
lines.append("")
|
||||
lines.append(f' __schema__ = "cclf"')
|
||||
lines.append(' __schema__ = "cclf"')
|
||||
lines.append(f' __tablename__ = "{tbl_name}"')
|
||||
|
||||
seen_names: set[str] = set()
|
||||
|
||||
@@ -17,8 +17,7 @@ from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import textwrap
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
@@ -146,7 +145,7 @@ def parse_entry(raw: str) -> Variable | None:
|
||||
|
||||
# Parse the remaining structured fields
|
||||
# Rejoin everything from LABEL: onward
|
||||
body = "\n".join(lines[label_idx:])
|
||||
"\n".join(lines[label_idx:])
|
||||
|
||||
# Extract each field by finding the label and collecting text
|
||||
# until the next label
|
||||
@@ -211,9 +210,9 @@ def variable_to_py(var: Variable) -> str:
|
||||
primary_name = var.name.split("\n")[0].strip()
|
||||
|
||||
lines = [
|
||||
f'"""',
|
||||
'"""',
|
||||
f"Variable: {primary_name}",
|
||||
f"",
|
||||
"",
|
||||
]
|
||||
|
||||
if var.label and var.label != "—":
|
||||
@@ -289,7 +288,9 @@ def main() -> None:
|
||||
"--pdf",
|
||||
default=str(
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "dev" / "seeds" / "codebook-ffs-claims.pdf"
|
||||
/ "dev"
|
||||
/ "seeds"
|
||||
/ "codebook-ffs-claims.pdf"
|
||||
),
|
||||
help="Path to the CCW codebook PDF",
|
||||
)
|
||||
|
||||
@@ -16,11 +16,9 @@ Defaults:
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import fastexcel
|
||||
import polars as pl
|
||||
|
||||
# ── Sheets and their slug names ──────────────────────────────────────────────
|
||||
|
||||
@@ -169,7 +167,7 @@ def generate_model(
|
||||
f"class {cls}(SQLTable):",
|
||||
f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""',
|
||||
"",
|
||||
f' __schema__ = "ccw"',
|
||||
' __schema__ = "ccw"',
|
||||
f' __tablename__ = "{claim_slug}_{table}"',
|
||||
"",
|
||||
]
|
||||
@@ -214,7 +212,9 @@ def main() -> None:
|
||||
"--xlsx",
|
||||
default=str(
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "dev" / "seeds" / "record-layout-ffs-claims.xlsx"
|
||||
/ "dev"
|
||||
/ "seeds"
|
||||
/ "record-layout-ffs-claims.xlsx"
|
||||
),
|
||||
help="Path to the CCW record layout Excel file",
|
||||
)
|
||||
|
||||
@@ -224,9 +224,9 @@ def generate_function(table: dict) -> str:
|
||||
"@nw.narwhalify",
|
||||
f"def {tablename}({param}: FrameT) -> FrameT:",
|
||||
f' """Clean and type-cast cms.{tablename}.',
|
||||
f"",
|
||||
"",
|
||||
f" {field_count} fields. Numeric columns auto-cast to Float64.",
|
||||
f' """',
|
||||
' """',
|
||||
f" return auto_cast({param})",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
@@ -236,18 +236,18 @@ def generate_module(domain: str, tables: list[dict]) -> str:
|
||||
"""Generate a full express module for a domain group."""
|
||||
lines = [
|
||||
f'"""CMS express — {domain} domain.',
|
||||
f"",
|
||||
"",
|
||||
f"Auto-generated narwhals functions for {len(tables)} CMS tables.",
|
||||
f"Each function takes the raw table and returns a typed frame",
|
||||
f"with numeric columns cast to Float64.",
|
||||
f'"""',
|
||||
f"",
|
||||
f"from __future__ import annotations",
|
||||
f"",
|
||||
f"import narwhals as nw",
|
||||
f"from narwhals.typing import FrameT",
|
||||
f"",
|
||||
f"from cms.express._helpers import auto_cast",
|
||||
"Each function takes the raw table and returns a typed frame",
|
||||
"with numeric columns cast to Float64.",
|
||||
'"""',
|
||||
"",
|
||||
"from __future__ import annotations",
|
||||
"",
|
||||
"import narwhals as nw",
|
||||
"from narwhals.typing import FrameT",
|
||||
"",
|
||||
"from cms.express._helpers import auto_cast",
|
||||
]
|
||||
|
||||
for tbl in sorted(tables, key=lambda t: t["tablename"]):
|
||||
|
||||
@@ -14,7 +14,6 @@ import argparse
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from textwrap import dedent, indent
|
||||
|
||||
import duckdb
|
||||
|
||||
@@ -268,7 +267,7 @@ def col_expr_to_nw(expr: str) -> str:
|
||||
# date_diff('unit', a, b)
|
||||
dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE)
|
||||
if dd_m:
|
||||
unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
|
||||
_unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
|
||||
if alias:
|
||||
return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")'
|
||||
return f"{_col_ref(b)} - {_col_ref(a)}"
|
||||
@@ -712,13 +711,13 @@ def generate_function(
|
||||
doc_ops = ", ".join(ops)
|
||||
|
||||
lines: list[str] = []
|
||||
lines.append(f"@nw.narwhalify")
|
||||
lines.append("@nw.narwhalify")
|
||||
lines.append(f"def {func_name}({param_str}) -> FrameT:")
|
||||
lines.append(f' """Build {schema}.{view_name}')
|
||||
lines.append(f"")
|
||||
lines.append("")
|
||||
lines.append(f" Operations: {doc_ops}")
|
||||
lines.append(f" Dependencies: {doc_deps}")
|
||||
lines.append(f' """')
|
||||
lines.append(' """')
|
||||
|
||||
if "passthrough" in ops:
|
||||
p = params[0].split(":")[0]
|
||||
@@ -880,7 +879,7 @@ def _gen_select_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
||||
if nw_cols:
|
||||
lines.append(f" return {p}.select(")
|
||||
_emit_exprs(lines, nw_cols)
|
||||
lines.append(f" )")
|
||||
lines.append(" )")
|
||||
else:
|
||||
lines.append(f" return {p}")
|
||||
|
||||
@@ -902,13 +901,13 @@ def _gen_filter_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
||||
lines.append(f" # {comment_part}")
|
||||
lines.append(f" return {p}.filter(")
|
||||
lines.append(f" {code_part}")
|
||||
lines.append(f" )")
|
||||
lines.append(" )")
|
||||
else:
|
||||
safe = clause.replace('"', "'")[:80]
|
||||
lines.append(f" # TODO: complex filter: {safe}")
|
||||
lines.append(f" return {p}")
|
||||
else:
|
||||
lines.append(f" # WHERE clause could not be parsed")
|
||||
lines.append(" # WHERE clause could not be parsed")
|
||||
lines.append(f" return {p}")
|
||||
|
||||
|
||||
@@ -994,9 +993,9 @@ def _gen_join_body(
|
||||
keys_str = ", ".join(f'"{k}"' for k in keys)
|
||||
lines.append(f" result = result.group_by({keys_str}).agg()")
|
||||
else:
|
||||
lines.append(f" # TODO: complex GROUP BY")
|
||||
lines.append(" # TODO: complex GROUP BY")
|
||||
|
||||
lines.append(f" return result")
|
||||
lines.append(" return result")
|
||||
|
||||
|
||||
def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
||||
@@ -1023,7 +1022,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
|
||||
keys.append(c_clean)
|
||||
break
|
||||
if not keys:
|
||||
lines.append(f" # TODO: complex GROUP BY could not be parsed")
|
||||
lines.append(" # TODO: complex GROUP BY could not be parsed")
|
||||
lines.append(f" return {p}")
|
||||
return
|
||||
keys_str = ", ".join(f'"{k}"' for k in keys)
|
||||
@@ -1045,7 +1044,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
|
||||
if agg_cols:
|
||||
lines.append(f" return {p}.group_by({keys_str}).agg(")
|
||||
_emit_exprs(lines, agg_cols)
|
||||
lines.append(f" )")
|
||||
lines.append(" )")
|
||||
else:
|
||||
lines.append(f" return {p}.group_by({keys_str}).agg()")
|
||||
|
||||
@@ -1061,20 +1060,20 @@ def _gen_window_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
||||
|
||||
lines.append(f" return {p}.with_columns(")
|
||||
_emit_exprs(lines, nw_cols)
|
||||
lines.append(f" )")
|
||||
lines.append(" )")
|
||||
|
||||
|
||||
def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
||||
params = [ref_to_param(s, t) for s, t in deps]
|
||||
if len(params) >= 2:
|
||||
lines.append(f" return nw.concat([")
|
||||
lines.append(" return nw.concat([")
|
||||
for p in params:
|
||||
lines.append(f" {p},")
|
||||
lines.append(f" ])")
|
||||
lines.append(" ])")
|
||||
elif params:
|
||||
lines.append(f" return {params[0]}")
|
||||
else:
|
||||
lines.append(f" return df")
|
||||
lines.append(" return df")
|
||||
|
||||
|
||||
# ── Module generation ───────────────────────────────────────────────────────
|
||||
@@ -1121,9 +1120,7 @@ def main() -> None:
|
||||
default=str(_conf_path("db.aco")),
|
||||
help="Path to DuckDB file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--out", default="../src/aco/express", help="Output directory"
|
||||
)
|
||||
parser.add_argument("--out", default="../src/aco/express", help="Output directory")
|
||||
args = parser.parse_args()
|
||||
|
||||
db_path = Path(args.db)
|
||||
|
||||
@@ -147,13 +147,12 @@ def generate_model(
|
||||
escaped = comment.replace('"', '\\"')
|
||||
if nullable:
|
||||
field_lines.append(
|
||||
f' {fname}: {py} | None = Field('
|
||||
f" {fname}: {py} | None = Field("
|
||||
f'default=None, description="{escaped}")'
|
||||
)
|
||||
else:
|
||||
field_lines.append(
|
||||
f' {fname}: {py} = Field('
|
||||
f'description="{escaped}")'
|
||||
f' {fname}: {py} = Field(description="{escaped}")'
|
||||
)
|
||||
else:
|
||||
if nullable:
|
||||
@@ -202,14 +201,13 @@ def generate_schema_module(
|
||||
return f"{header}\n\n\n{body}\n"
|
||||
|
||||
|
||||
|
||||
|
||||
# ── Main ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
|
||||
from conf import cfg, path as _conf_path
|
||||
from conf import cfg
|
||||
from conf import path as _conf_path
|
||||
|
||||
parser.add_argument(
|
||||
"--db",
|
||||
|
||||
@@ -58,7 +58,6 @@ Design notes
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import textwrap
|
||||
import zipfile
|
||||
@@ -68,7 +67,6 @@ from typing import NamedTuple
|
||||
import pdfplumber
|
||||
|
||||
# ── Path constants ────────────────────────────────────────────────────────────
|
||||
|
||||
from conf import ROOT as _ROOT
|
||||
from conf import path as _conf_path
|
||||
|
||||
@@ -176,8 +174,9 @@ def _extract_from_zip(
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
|
||||
zotero_root: Path, work_dir: Path) -> Path | None:
|
||||
def _resolve_pdf(
|
||||
pdf_basename: str, zip_basename: str | None, zotero_root: Path, work_dir: Path
|
||||
) -> Path | None:
|
||||
"""Return path to a MIF PDF, extracting from zip if necessary."""
|
||||
direct = _find_in_zotero(pdf_basename, zotero_root)
|
||||
if direct:
|
||||
@@ -187,8 +186,12 @@ def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_xlsx(xlsx_basename: str | None, zip_basename: str | None,
|
||||
zotero_root: Path, work_dir: Path) -> Path | None:
|
||||
def _resolve_xlsx(
|
||||
xlsx_basename: str | None,
|
||||
zip_basename: str | None,
|
||||
zotero_root: Path,
|
||||
work_dir: Path,
|
||||
) -> Path | None:
|
||||
"""Return path to a value set Excel file, extracting from zip if needed."""
|
||||
if xlsx_basename is None:
|
||||
return None
|
||||
@@ -329,7 +332,9 @@ def _read_sheet_summaries(xlsx_path: Path) -> dict[str, SheetSummary]:
|
||||
for row in rows[:3]:
|
||||
non_none = [str(c).strip() for c in row if c is not None]
|
||||
if len(non_none) >= 2:
|
||||
header = [str(c).strip().replace("\n", " ") for c in row if c is not None]
|
||||
header = [
|
||||
str(c).strip().replace("\n", " ") for c in row if c is not None
|
||||
]
|
||||
break
|
||||
summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header)
|
||||
wb.close()
|
||||
@@ -407,13 +412,13 @@ def build_uamcc_passages(
|
||||
paa4_info = _vs_info("UAMCC PAA4")
|
||||
cohort_info = _vs_info("UAMCC Cohort")
|
||||
excl_info = _vs_info("UAMCC Exclusions")
|
||||
ccs_cm_info = _vs_info("UAMCC CCS-ICD10CM")
|
||||
ccs_pcs_info = _vs_info("UAMCC CCS-ICD10PCS")
|
||||
_vs_info("UAMCC CCS-ICD10CM")
|
||||
_vs_info("UAMCC CCS-ICD10PCS")
|
||||
|
||||
cite_pp = _cite(pdf_name, "§3.8", denom_page)
|
||||
cite_num = _cite(pdf_name, "§3.6–3.7", num_page)
|
||||
_cite(pdf_name, "§3.6–3.7", num_page)
|
||||
cite_excl = _cite(pdf_name, "§3.10", denom_excl_page)
|
||||
cite_risk = _cite(pdf_name, "§3.12", risk_page)
|
||||
_cite(pdf_name, "§3.12", risk_page)
|
||||
cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page)
|
||||
|
||||
passages: dict[str, str] = {}
|
||||
@@ -423,13 +428,13 @@ def build_uamcc_passages(
|
||||
f"Return the UAMCC performance period anchor row.\n\n"
|
||||
f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n"
|
||||
f"UAMCC §2.2 Measure Description:\n"
|
||||
f" \"This outcome measure is calculated using 12 consecutive months\n"
|
||||
f' "This outcome measure is calculated using 12 consecutive months\n'
|
||||
f" of Medicare fee-for-service (FFS) claims data. The measure is a\n"
|
||||
f" risk-standardized acute admission rate (RSAAR) that adjusts for\n"
|
||||
f" age, clinical comorbidities, and other clinical and frailty risk\n"
|
||||
f" factors present at the start of the 12-month measurement period,\n"
|
||||
f" as well as social risk factors. Lower RSAARs indicate better\n"
|
||||
f" performance.\"\n\n"
|
||||
f' performance."\n\n'
|
||||
f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n"
|
||||
f"Measurement duration: 12 consecutive months (Jan 1 – Dec 31).\n"
|
||||
f"Performance Year: {py_version}.\n\n"
|
||||
@@ -443,10 +448,10 @@ def build_uamcc_passages(
|
||||
f"Identify each beneficiary's qualifying chronic condition groups.\n\n"
|
||||
f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n"
|
||||
f"UAMCC §3.9 Denominator Details:\n"
|
||||
f" \"The cohort is Medicare FFS beneficiaries 66 years of age and\n"
|
||||
f' "The cohort is Medicare FFS beneficiaries 66 years of age and\n'
|
||||
f" older assigned to the REACH ACO during the measurement period\n"
|
||||
f" with diagnoses that fall into two or more of nine chronic disease\n"
|
||||
f" groups.\"\n\n"
|
||||
f' groups."\n\n'
|
||||
f"Nine chronic disease groups (MIF §3.9, pp.7–8):\n"
|
||||
f" 1. Acute myocardial infarction (AMI)\n"
|
||||
f" 2. Alzheimer's disease and related disorders or senile dementia\n"
|
||||
@@ -471,7 +476,7 @@ def build_uamcc_passages(
|
||||
f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n"
|
||||
f"Source: {cite_pp}\n\n"
|
||||
f"UAMCC §3.8 Denominator Statement:\n"
|
||||
f" \"{denom_stmt_trimmed[:400]}\"\n\n"
|
||||
f' "{denom_stmt_trimmed[:400]}"\n\n'
|
||||
f"Inclusion criteria (MIF §3.9):\n"
|
||||
f" 1. Age ≥66 at the first day of the measurement period\n"
|
||||
f" 2. Two or more distinct MCC groups identified in the lookback year\n"
|
||||
@@ -501,13 +506,15 @@ def build_uamcc_passages(
|
||||
)
|
||||
|
||||
# ── Planned admission (PAA) ───────────────────────────────────────
|
||||
paa_snippet = ""
|
||||
if num_detail:
|
||||
# Find the PAA description paragraph
|
||||
m = re.search(r"planned admission algorithm.+?(?=\n\n|\Z)", num_detail,
|
||||
re.DOTALL | re.IGNORECASE)
|
||||
m = re.search(
|
||||
r"planned admission algorithm.+?(?=\n\n|\Z)",
|
||||
num_detail,
|
||||
re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
if m:
|
||||
paa_snippet = m.group(0)[:600]
|
||||
m.group(0)[:600]
|
||||
passages["uamcc_int_planned_admission"] = (
|
||||
f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n"
|
||||
f"Source: {cite_paa}\n\n"
|
||||
@@ -516,7 +523,7 @@ def build_uamcc_passages(
|
||||
f" Readmission Algorithm Version 4.0, which CMS originally created\n"
|
||||
f" to identify planned readmissions for the hospital-wide readmission\n"
|
||||
f" measure. In brief, the algorithm uses a flowchart and four tables\n"
|
||||
f" of procedure and/or discharge diagnosis categories.\"\n\n"
|
||||
f' of procedure and/or discharge diagnosis categories."\n\n'
|
||||
f"PAA Rules (first match wins):\n"
|
||||
f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n"
|
||||
f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n"
|
||||
@@ -562,14 +569,14 @@ def build_uamcc_passages(
|
||||
f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n"
|
||||
f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n"
|
||||
f"UAMCC §3.11 Denominator Exclusion Details:\n"
|
||||
f" \"Persons are considered at risk for admission if they are alive,\n"
|
||||
f' "Persons are considered at risk for admission if they are alive,\n'
|
||||
f" enrolled in Medicare FFS, and not admitted to an acute care\n"
|
||||
f" hospital. In addition to time spent in the hospital, excluded\n"
|
||||
f" from at-risk time are:\n"
|
||||
f" (1) time spent in an SNF or acute rehabilitation facility;\n"
|
||||
f" (2) time within 10 days following discharge from a hospital,\n"
|
||||
f" SNF, or acute rehabilitation facility;\n"
|
||||
f" (3) time after entering hospice care.\"\n\n"
|
||||
f' (3) time after entering hospice care."\n\n'
|
||||
f"Person-years = at_risk_days / 365.25\n\n"
|
||||
f"Performance Year: {py_version}."
|
||||
)
|
||||
|
||||
@@ -19,7 +19,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import textwrap
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
@@ -29,10 +28,7 @@ import pdfplumber
|
||||
_ROOT = Path(__file__).resolve().parents[2]
|
||||
_SEEDS = _ROOT / "dev" / "seeds"
|
||||
|
||||
PDF_PATH = (
|
||||
_SEEDS
|
||||
/ "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
|
||||
)
|
||||
PDF_PATH = _SEEDS / "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
|
||||
ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py"
|
||||
FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py"
|
||||
QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py"
|
||||
|
||||
@@ -21,7 +21,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
# Define all table specifications based on extracted data dictionary
|
||||
|
||||
@@ -336,7 +336,9 @@ def main():
|
||||
"""Generate REACH participants table model and rex parser."""
|
||||
project_root = Path(__file__).resolve().parents[2]
|
||||
xlsx_path = (
|
||||
project_root / "dev" / "seeds"
|
||||
project_root
|
||||
/ "dev"
|
||||
/ "seeds"
|
||||
/ "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx"
|
||||
)
|
||||
|
||||
|
||||
@@ -79,15 +79,12 @@ def sanitize_field(name: str) -> str:
|
||||
def get_tables(con: sqlite3.Connection) -> list[str]:
|
||||
"""Return all user tables, sorted alphabetically."""
|
||||
rows = con.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
||||
"ORDER BY name"
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
|
||||
|
||||
def get_columns(
|
||||
con: sqlite3.Connection, table: str
|
||||
) -> list[tuple[str, str, bool]]:
|
||||
def get_columns(con: sqlite3.Connection, table: str) -> list[tuple[str, str, bool]]:
|
||||
"""Return [(col_name, sqlite_type, nullable), ...].
|
||||
|
||||
SQLite PRAGMA table_info returns:
|
||||
|
||||
@@ -63,9 +63,7 @@ def _get_item_tags(zcon: sqlite3.Connection, item_id: int) -> list[str]:
|
||||
return [r["name"] for r in rows]
|
||||
|
||||
|
||||
def _get_item_collections(
|
||||
zcon: sqlite3.Connection, item_id: int
|
||||
) -> list[str]:
|
||||
def _get_item_collections(zcon: sqlite3.Connection, item_id: int) -> list[str]:
|
||||
"""Read collection keys for a Zotero item."""
|
||||
rows = zcon.execute(
|
||||
"""SELECT c.key FROM collectionItems ci
|
||||
@@ -76,9 +74,7 @@ def _get_item_collections(
|
||||
return [r["key"] for r in rows]
|
||||
|
||||
|
||||
def _get_item_creators(
|
||||
zcon: sqlite3.Connection, item_id: int
|
||||
) -> list[dict]:
|
||||
def _get_item_creators(zcon: sqlite3.Connection, item_id: int) -> list[dict]:
|
||||
"""Read creators for a Zotero item."""
|
||||
rows = zcon.execute(
|
||||
"""SELECT c.firstName, c.lastName, ct.creatorType,
|
||||
@@ -101,9 +97,7 @@ def _get_item_creators(
|
||||
]
|
||||
|
||||
|
||||
def _classify_item(
|
||||
zotero_type: str, fields: dict
|
||||
) -> str:
|
||||
def _classify_item(zotero_type: str, fields: dict) -> str:
|
||||
"""Map Zotero itemType + fields to bib item_type."""
|
||||
if zotero_type == "statute":
|
||||
code = fields.get("code", "")
|
||||
@@ -134,14 +128,16 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
||||
rule_type = part[6:]
|
||||
elif part.startswith("Effective: "):
|
||||
eff_date = part[11:]
|
||||
return json.dumps({
|
||||
"fr_volume": fields.get("codeNumber", ""),
|
||||
"fr_page": fields.get("pages", ""),
|
||||
"document_number": doc_num,
|
||||
"cms_id": fields.get("session", ""),
|
||||
"rule_type": rule_type,
|
||||
"effective_date": eff_date,
|
||||
})
|
||||
return json.dumps(
|
||||
{
|
||||
"fr_volume": fields.get("codeNumber", ""),
|
||||
"fr_page": fields.get("pages", ""),
|
||||
"document_number": doc_num,
|
||||
"cms_id": fields.get("session", ""),
|
||||
"rule_type": rule_type,
|
||||
"effective_date": eff_date,
|
||||
}
|
||||
)
|
||||
|
||||
if item_type == "regulation":
|
||||
history = fields.get("history", "")
|
||||
@@ -152,13 +148,15 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
||||
part = segment[5:]
|
||||
elif segment.startswith("Authority: "):
|
||||
authority = segment[11:]
|
||||
return json.dumps({
|
||||
"cfr_title": fields.get("codeNumber", ""),
|
||||
"cfr_part": part,
|
||||
"cfr_section": fields.get("section", ""),
|
||||
"authority": authority,
|
||||
"effective_date": fields.get("dateEnacted", ""),
|
||||
})
|
||||
return json.dumps(
|
||||
{
|
||||
"cfr_title": fields.get("codeNumber", ""),
|
||||
"cfr_part": part,
|
||||
"cfr_section": fields.get("section", ""),
|
||||
"authority": authority,
|
||||
"effective_date": fields.get("dateEnacted", ""),
|
||||
}
|
||||
)
|
||||
|
||||
if item_type == "manual":
|
||||
extra = fields.get("extra", "")
|
||||
@@ -169,33 +167,37 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
||||
chapter = fields.get("seriesNumber", "")
|
||||
if chapter.startswith("Chapter "):
|
||||
chapter = chapter[8:]
|
||||
return json.dumps({
|
||||
"manual_name": fields.get("seriesTitle", ""),
|
||||
"pub_number": fields.get("reportNumber", ""),
|
||||
"chapter": chapter,
|
||||
"transmittal": transmittal,
|
||||
})
|
||||
return json.dumps(
|
||||
{
|
||||
"manual_name": fields.get("seriesTitle", ""),
|
||||
"pub_number": fields.get("reportNumber", ""),
|
||||
"chapter": chapter,
|
||||
"transmittal": transmittal,
|
||||
}
|
||||
)
|
||||
|
||||
if item_type == "download":
|
||||
extra = fields.get("extra", "")
|
||||
file_urls: list[str] = []
|
||||
for line in extra.split("\n"):
|
||||
if line.startswith("Files: "):
|
||||
file_urls = [
|
||||
u.strip() for u in line[7:].split(";") if u.strip()
|
||||
]
|
||||
return json.dumps({
|
||||
"page_type": "",
|
||||
"file_urls": file_urls,
|
||||
"year": None,
|
||||
"quarter": "",
|
||||
"website_title": fields.get("websiteTitle", ""),
|
||||
})
|
||||
file_urls = [u.strip() for u in line[7:].split(";") if u.strip()]
|
||||
return json.dumps(
|
||||
{
|
||||
"page_type": "",
|
||||
"file_urls": file_urls,
|
||||
"year": None,
|
||||
"quarter": "",
|
||||
"website_title": fields.get("websiteTitle", ""),
|
||||
}
|
||||
)
|
||||
|
||||
if item_type == "source":
|
||||
return json.dumps({
|
||||
"doc_type": fields.get("type", ""),
|
||||
})
|
||||
return json.dumps(
|
||||
{
|
||||
"doc_type": fields.get("type", ""),
|
||||
}
|
||||
)
|
||||
|
||||
return "{}"
|
||||
|
||||
@@ -255,8 +257,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
||||
parent_id = row["id"]
|
||||
|
||||
bcon.execute(
|
||||
"INSERT OR IGNORE INTO collections (key, name, parent_id) "
|
||||
"VALUES (?, ?, ?)",
|
||||
"INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
|
||||
(zc["key"], zc["collectionName"], parent_id),
|
||||
)
|
||||
counts["collections"] += 1
|
||||
@@ -322,8 +323,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
||||
for tag_name in ztags:
|
||||
tag_id = store._ensure_tag(tag_name)
|
||||
bcon.execute(
|
||||
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) "
|
||||
"VALUES (?, ?)",
|
||||
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
|
||||
(bib_id, tag_id),
|
||||
)
|
||||
counts["tags"] += 1
|
||||
@@ -346,16 +346,14 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
||||
for cr in creators:
|
||||
# Find or create creator
|
||||
crow = bcon.execute(
|
||||
"SELECT id FROM creators "
|
||||
"WHERE first_name = ? AND last_name = ?",
|
||||
"SELECT id FROM creators WHERE first_name = ? AND last_name = ?",
|
||||
(cr["first_name"], cr["last_name"]),
|
||||
).fetchone()
|
||||
if crow:
|
||||
creator_id = crow["id"]
|
||||
else:
|
||||
cur = bcon.execute(
|
||||
"INSERT INTO creators (first_name, last_name) "
|
||||
"VALUES (?, ?)",
|
||||
"INSERT INTO creators (first_name, last_name) VALUES (?, ?)",
|
||||
(cr["first_name"], cr["last_name"]),
|
||||
)
|
||||
creator_id = cur.lastrowid
|
||||
@@ -462,9 +460,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Migrate Zotero SQLite to bib SQLite"
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Migrate Zotero SQLite to bib SQLite")
|
||||
from conf import path as _conf_path
|
||||
|
||||
parser.add_argument(
|
||||
|
||||
@@ -60,7 +60,6 @@ from typing import Any
|
||||
import polars as pl
|
||||
|
||||
# ── Path constants ────────────────────────────────────────────────────────────
|
||||
|
||||
from conf import ROOT as _ROOT
|
||||
from conf import path as _conf_path
|
||||
|
||||
|
||||
@@ -319,16 +319,12 @@ def _ensure_value(con: sqlite3.Connection, value: str) -> int:
|
||||
).fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
cur = con.execute(
|
||||
"INSERT INTO itemDataValues (value) VALUES (?)", (value,)
|
||||
)
|
||||
cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
|
||||
return cur.lastrowid
|
||||
|
||||
|
||||
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
|
||||
row = con.execute(
|
||||
"SELECT tagID FROM tags WHERE name = ?", (name,)
|
||||
).fetchone()
|
||||
row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
|
||||
@@ -342,8 +338,7 @@ def _set_zotero_field(
|
||||
return
|
||||
value_id = _ensure_value(con, value)
|
||||
con.execute(
|
||||
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) "
|
||||
"VALUES (?, ?, ?)",
|
||||
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
|
||||
(item_id, field_id, value_id),
|
||||
)
|
||||
|
||||
@@ -392,8 +387,7 @@ def tag_existing_zotero(
|
||||
for tag_name in all_tags:
|
||||
tag_id = _ensure_tag(con, tag_name)
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
|
||||
"VALUES (?, ?, 0)",
|
||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
|
||||
(parent_id, tag_id),
|
||||
)
|
||||
return True
|
||||
@@ -416,9 +410,9 @@ def register_in_zotero(
|
||||
return
|
||||
|
||||
now = _now_iso()
|
||||
next_id = con.execute(
|
||||
"SELECT COALESCE(MAX(itemID), 0) + 1 FROM items"
|
||||
).fetchone()[0]
|
||||
next_id = con.execute("SELECT COALESCE(MAX(itemID), 0) + 1 FROM items").fetchone()[
|
||||
0
|
||||
]
|
||||
parent_key = _zotero_key()
|
||||
|
||||
# Create parent webpage item
|
||||
@@ -434,9 +428,7 @@ def register_in_zotero(
|
||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url)
|
||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
|
||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
|
||||
_set_zotero_field(
|
||||
con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal"
|
||||
)
|
||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal")
|
||||
_set_zotero_field(
|
||||
con,
|
||||
next_id,
|
||||
@@ -449,8 +441,7 @@ def register_in_zotero(
|
||||
for tag_name in all_tags:
|
||||
tag_id = _ensure_tag(con, tag_name)
|
||||
con.execute(
|
||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
|
||||
"VALUES (?, ?, 0)",
|
||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
|
||||
(next_id, tag_id),
|
||||
)
|
||||
|
||||
@@ -464,13 +455,9 @@ def register_in_zotero(
|
||||
|
||||
# Copy to Zotero storage
|
||||
storage_dir = ZOTERO_STORAGE / att_key
|
||||
subprocess.run(
|
||||
["sudo", "mkdir", "-p", str(storage_dir)], check=True
|
||||
)
|
||||
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
|
||||
dest = storage_dir / seed_path.name
|
||||
subprocess.run(
|
||||
["sudo", "cp", str(seed_path), str(dest)], check=True
|
||||
)
|
||||
subprocess.run(["sudo", "cp", str(seed_path), str(dest)], check=True)
|
||||
subprocess.run(
|
||||
["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
|
||||
check=True,
|
||||
@@ -496,8 +483,7 @@ def register_in_zotero(
|
||||
)
|
||||
title_val = _ensure_value(con, seed_path.name)
|
||||
con.execute(
|
||||
"INSERT INTO itemData (itemID, fieldID, valueID) "
|
||||
"VALUES (?, ?, ?)",
|
||||
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
|
||||
(att_id, ZOTERO_FIELD_TITLE, title_val),
|
||||
)
|
||||
|
||||
@@ -567,7 +553,7 @@ def pull_from_zotero(*, dry_run: bool = False) -> int:
|
||||
zpath = row[0] # "storage:filename"
|
||||
if not zpath.startswith("storage:"):
|
||||
continue
|
||||
filename = zpath[len("storage:"):]
|
||||
filename = zpath[len("storage:") :]
|
||||
|
||||
dest = SEEDS_DIR / filename
|
||||
if dest.exists():
|
||||
|
||||
@@ -97,11 +97,11 @@ def main():
|
||||
print(f" ... and {len(tables) - 3} more tables")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(f"\nTo create these in Unity Catalog, run without --show-schemas flag")
|
||||
print("\nTo create these in Unity Catalog, run without --show-schemas flag")
|
||||
return
|
||||
|
||||
# Create Unity client
|
||||
print(f"Connecting to Unity Catalog...")
|
||||
print("Connecting to Unity Catalog...")
|
||||
print(f" Workspace ID: {args.workspace_id}")
|
||||
print(f" Catalog: {args.catalog}")
|
||||
print()
|
||||
@@ -114,7 +114,7 @@ def main():
|
||||
# Test connection
|
||||
try:
|
||||
catalogs = client.list_catalogs()
|
||||
print(f"✓ Connected to Unity Catalog")
|
||||
print("✓ Connected to Unity Catalog")
|
||||
print(f" Found {len(catalogs)} existing catalogs:")
|
||||
for cat in catalogs:
|
||||
print(f" - {cat.name}")
|
||||
@@ -130,7 +130,7 @@ def main():
|
||||
print("=" * 70)
|
||||
print()
|
||||
|
||||
print(f"Setting up catalog structure...")
|
||||
print("Setting up catalog structure...")
|
||||
print()
|
||||
|
||||
report = setup_catalog_from_schemas(
|
||||
|
||||
@@ -20,11 +20,11 @@ fi
|
||||
|
||||
# Image definitions — matches stack.toml [images]
|
||||
declare -A DOCKERFILES=(
|
||||
[api]="api/Dockerfile"
|
||||
[notebooks]="notebooks/Dockerfile"
|
||||
[docs]="docs/Dockerfile"
|
||||
[zotero]="zotero/Dockerfile"
|
||||
[mc]="rustfs/Dockerfile.mc"
|
||||
[api]="infra/images/api.Dockerfile"
|
||||
[notebooks]="infra/images/notebooks.Dockerfile"
|
||||
[docs]="infra/images/docs.Dockerfile"
|
||||
[zotero]="infra/images/zotero.Dockerfile"
|
||||
[mc]="infra/images/mc.Dockerfile"
|
||||
)
|
||||
|
||||
declare -A CONTEXTS=(
|
||||
@@ -32,7 +32,7 @@ declare -A CONTEXTS=(
|
||||
[notebooks]="notebooks/"
|
||||
[docs]="."
|
||||
[zotero]="zotero/"
|
||||
[mc]="rustfs/"
|
||||
[mc]="infra/rustfs/"
|
||||
)
|
||||
|
||||
# Select images to test
|
||||
|
||||
@@ -23,7 +23,6 @@ import os
|
||||
|
||||
from aco.lake import IcebergContext, UnityClient
|
||||
from aco.lake.catalog import Catalog
|
||||
from aco.lake.engine import execute
|
||||
|
||||
|
||||
def example_unity_client_basics():
|
||||
@@ -185,7 +184,7 @@ def example_iceberg_context():
|
||||
},
|
||||
)
|
||||
|
||||
print(f"✓ IcebergContext configured")
|
||||
print("✓ IcebergContext configured")
|
||||
print(f" Catalog URI: {ctx.catalog_uri}")
|
||||
print(f" Warehouse: {ctx.warehouse}")
|
||||
print()
|
||||
@@ -222,7 +221,7 @@ def example_pipeline_execution():
|
||||
}
|
||||
)
|
||||
|
||||
ctx = IcebergContext(
|
||||
IcebergContext(
|
||||
catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg",
|
||||
warehouse="aco_dev",
|
||||
catalog=catalog,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
/* LOCH — deep-navy design system — matches styles/inject.css */
|
||||
/* LOCH — deep-navy design system — matches assets/css/inject.css */
|
||||
|
||||
:root {
|
||||
--ifm-color-primary: #4488dd;
|
||||
|
||||
0
infra/polaris/.gitkeep
Normal file
0
infra/polaris/.gitkeep
Normal file
@@ -17,7 +17,7 @@ Usage::
|
||||
CLI::
|
||||
|
||||
uv run python -m aco.dag
|
||||
uv run python -m aco.dag -p readmissions --html dag.html
|
||||
uv run python -m aco.dag -p readmissions --html assets/dag.html
|
||||
uv run python -m aco.dag --dot dag.dot
|
||||
uv run python -m aco.dag -p pharmacy --mermaid
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user