chore: clean sweep — lint, format, stale refs, generated artifacts
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled
Some checks failed
CI / skinny-install (aco) (push) Successful in 45s
CI / skinny-install (api) (push) Successful in 29s
CI / skinny-install (bcda) (push) Successful in 25s
CI / skinny-install (bib) (push) Successful in 23s
CI / skinny-install (bls) (push) Successful in 20s
CI / skinny-install (ccw) (push) Successful in 36s
CI / skinny-install (cli) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (conf) (push) Successful in 27s
CI / skinny-install (pfs) (push) Successful in 25s
CI / skinny-install (rex) (push) Successful in 25s
CI / lint-test (push) Successful in 6m2s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Failing after 6s
Infra CI / docs (push) Successful in 33s
Infra CI / api (push) Successful in 6s
Infra CI / mc (push) Successful in 7s
Deploy / build-scan-report (push) Has been cancelled
- Fix all 72 ruff lint errors (unused imports, unused variables, E402) - Format all 14 unformatted dev/scripts files - Move generated artifacts to assets/ (dag.html, pfs.html) - Remove duplicate root coverage.svg (already in assets/icons/) - Update .dockerignore for infra/ tree layout - Update .gitignore: add .env.bak, mirrors/, htmlcov/ - Fix stale path refs in coverage_badge.py, woodpecker backend, test_network_isolation.sh, docs custom.css - Add .gitkeep to empty dirs (infra/polaris, cloud/*/terraform) - Delete 12 stale local branches, 10 stale remote branches
This commit is contained in:
@@ -12,13 +12,8 @@ logs/
|
|||||||
tuva/
|
tuva/
|
||||||
dev/
|
dev/
|
||||||
tests/
|
tests/
|
||||||
grafana/
|
infra/
|
||||||
prometheus/
|
assets/
|
||||||
loki/
|
cloud/
|
||||||
trino/
|
mirrors/
|
||||||
traefik/
|
|
||||||
nginx/
|
|
||||||
gitea/
|
|
||||||
styles/
|
|
||||||
.woodpecker/
|
|
||||||
.claude/
|
.claude/
|
||||||
|
|||||||
13
.gitignore
vendored
13
.gitignore
vendored
@@ -13,17 +13,28 @@ bundle/sql/
|
|||||||
|
|
||||||
# Marimo cache
|
# Marimo cache
|
||||||
notebooks/__marimo__/
|
notebooks/__marimo__/
|
||||||
|
|
||||||
|
# Secrets and keys
|
||||||
.deploy_key
|
.deploy_key
|
||||||
.env
|
|
||||||
.deploy_key.pub
|
.deploy_key.pub
|
||||||
|
.env
|
||||||
|
.env.bak
|
||||||
|
|
||||||
|
# Python build artifacts
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.pyc
|
*.pyc
|
||||||
.coverage
|
.coverage
|
||||||
dist/
|
dist/
|
||||||
*.egg-info/
|
*.egg-info/
|
||||||
|
htmlcov/
|
||||||
|
|
||||||
|
# IDE / tool settings
|
||||||
.claude/settings.local.json
|
.claude/settings.local.json
|
||||||
notebooks/aco.duckdb
|
notebooks/aco.duckdb
|
||||||
|
|
||||||
|
# Package mirrors (local cache, not tracked)
|
||||||
|
mirrors/
|
||||||
|
|
||||||
# Docs (auto-generated at build time)
|
# Docs (auto-generated at build time)
|
||||||
docs/docs/api/
|
docs/docs/api/
|
||||||
docs/static/library.json
|
docs/static/library.json
|
||||||
|
|||||||
0
cloud/aws/terraform/.gitkeep
Normal file
0
cloud/aws/terraform/.gitkeep
Normal file
0
cloud/azure/terraform/.gitkeep
Normal file
0
cloud/azure/terraform/.gitkeep
Normal file
0
cloud/gcp/terraform/.gitkeep
Normal file
0
cloud/gcp/terraform/.gitkeep
Normal file
20
coverage.svg
20
coverage.svg
@@ -1,20 +0,0 @@
|
|||||||
<svg xmlns="http://www.w3.org/2000/svg" width="116" height="20">
|
|
||||||
<linearGradient id="b" x2="0" y2="100%">
|
|
||||||
<stop offset="0" stop-color="#bbb" stop-opacity=".1"/>
|
|
||||||
<stop offset="1" stop-opacity=".1"/>
|
|
||||||
</linearGradient>
|
|
||||||
<clipPath id="a">
|
|
||||||
<rect width="116" height="20" rx="3" fill="#fff"/>
|
|
||||||
</clipPath>
|
|
||||||
<g clip-path="url(#a)">
|
|
||||||
<path fill="#555" d="M0 0h65v20H0z"/>
|
|
||||||
<path fill="#4c1" d="M65 0h51v20H65z"/>
|
|
||||||
<path fill="url(#b)" d="M0 0h116v20H0z"/>
|
|
||||||
</g>
|
|
||||||
<g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif" font-size="11">
|
|
||||||
<text x="32.5" y="15" fill="#010101" fill-opacity=".3">coverage</text>
|
|
||||||
<text x="32.5" y="14">coverage</text>
|
|
||||||
<text x="90" y="15" fill="#010101" fill-opacity=".3">99%</text>
|
|
||||||
<text x="90" y="14">99%</text>
|
|
||||||
</g>
|
|
||||||
</svg>
|
|
||||||
|
Before Width: | Height: | Size: 822 B |
@@ -7,11 +7,8 @@ Usage:
|
|||||||
uv run python dev/scripts/add_carrier_to_zotero.py
|
uv run python dev/scripts/add_carrier_to_zotero.py
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import os
|
|
||||||
import random
|
import random
|
||||||
import shutil
|
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import string
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|||||||
@@ -9,7 +9,6 @@ Usage:
|
|||||||
|
|
||||||
import random
|
import random
|
||||||
import sqlite3
|
import sqlite3
|
||||||
import string
|
|
||||||
import subprocess
|
import subprocess
|
||||||
import zipfile
|
import zipfile
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|||||||
@@ -30,8 +30,11 @@ yaml.add_representer(
|
|||||||
|
|
||||||
def _dump(data: dict) -> str:
|
def _dump(data: dict) -> str:
|
||||||
return yaml.dump(
|
return yaml.dump(
|
||||||
data, default_flow_style=False, sort_keys=False,
|
data,
|
||||||
width=120, allow_unicode=True,
|
default_flow_style=False,
|
||||||
|
sort_keys=False,
|
||||||
|
width=120,
|
||||||
|
allow_unicode=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -69,9 +72,7 @@ class SQLPlugin(DabPlugin):
|
|||||||
output_mode="ctas",
|
output_mode="ctas",
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
result[f"bundle/sql/{pipe_name}/_error.txt"] = (
|
result[f"bundle/sql/{pipe_name}/_error.txt"] = f"Transpile failed: {e}"
|
||||||
f"Transpile failed: {e}"
|
|
||||||
)
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
for expr_name, sql in sql_map.items():
|
for expr_name, sql in sql_map.items():
|
||||||
@@ -80,8 +81,7 @@ class SQLPlugin(DabPlugin):
|
|||||||
schema, table = expr_name.split(".", 1)
|
schema, table = expr_name.split(".", 1)
|
||||||
path = f"bundle/sql/{pipe_name}/{table}.sql"
|
path = f"bundle/sql/{pipe_name}/{table}.sql"
|
||||||
header = (
|
header = (
|
||||||
f"-- {expr_name}\n"
|
f"-- {expr_name}\n-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
||||||
f"-- Generated by gen_config.py — DO NOT EDIT\n\n"
|
|
||||||
)
|
)
|
||||||
result[path] = header + sql + "\n"
|
result[path] = header + sql + "\n"
|
||||||
|
|
||||||
@@ -102,27 +102,58 @@ class JobsPlugin(DabPlugin):
|
|||||||
job_clusters = []
|
job_clusters = []
|
||||||
for profile_name in sorted(used_profiles):
|
for profile_name in sorted(used_profiles):
|
||||||
cluster_spec = dict(clusters_cfg.get(profile_name, {}))
|
cluster_spec = dict(clusters_cfg.get(profile_name, {}))
|
||||||
job_clusters.append(OrderedDict([
|
job_clusters.append(
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("job_cluster_key", f"{profile_name}_cluster"),
|
("job_cluster_key", f"{profile_name}_cluster"),
|
||||||
("new_cluster", OrderedDict([
|
(
|
||||||
("spark_version", cluster_spec.get("spark_version", "15.4.x-scala2.12")),
|
"new_cluster",
|
||||||
("node_type_id", cluster_spec.get("node_type_id", "i3.xlarge")),
|
OrderedDict(
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"spark_version",
|
||||||
|
cluster_spec.get(
|
||||||
|
"spark_version", "15.4.x-scala2.12"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"node_type_id",
|
||||||
|
cluster_spec.get("node_type_id", "i3.xlarge"),
|
||||||
|
),
|
||||||
("num_workers", cluster_spec.get("num_workers", 1)),
|
("num_workers", cluster_spec.get("num_workers", 1)),
|
||||||
])),
|
]
|
||||||
]))
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# DDL tasks — one per schema to create tables before SQL runs
|
# DDL tasks — one per schema to create tables before SQL runs
|
||||||
ddl_tasks = []
|
ddl_tasks = []
|
||||||
for schema_name in sorted(catalog_schemas):
|
for schema_name in sorted(catalog_schemas):
|
||||||
ddl_tasks.append(OrderedDict([
|
ddl_tasks.append(
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("task_key", f"ddl_{schema_name}"),
|
("task_key", f"ddl_{schema_name}"),
|
||||||
("job_cluster_key", "default_cluster"),
|
("job_cluster_key", "default_cluster"),
|
||||||
("sql_task", OrderedDict([
|
(
|
||||||
("file", OrderedDict([
|
"sql_task",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"file",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("path", f"bundle/ddl/{schema_name}/"),
|
("path", f"bundle/ddl/{schema_name}/"),
|
||||||
])),
|
]
|
||||||
])),
|
),
|
||||||
]))
|
),
|
||||||
|
]
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
# Pipeline tasks — each expression becomes a sql_task
|
# Pipeline tasks — each expression becomes a sql_task
|
||||||
pipeline_tasks = []
|
pipeline_tasks = []
|
||||||
@@ -143,12 +174,19 @@ class JobsPlugin(DabPlugin):
|
|||||||
# Each pipeline task runs all its SQL files in order
|
# Each pipeline task runs all its SQL files in order
|
||||||
# We use a for_each or sequential notebook; simplest is
|
# We use a for_each or sequential notebook; simplest is
|
||||||
# a single SQL file per pipeline that sources all expressions
|
# a single SQL file per pipeline that sources all expressions
|
||||||
task["sql_task"] = OrderedDict([
|
task["sql_task"] = OrderedDict(
|
||||||
("file", OrderedDict([
|
[
|
||||||
|
(
|
||||||
|
"file",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("path", f"bundle/sql/{pipe_name}/"),
|
("path", f"bundle/sql/{pipe_name}/"),
|
||||||
])),
|
]
|
||||||
|
),
|
||||||
|
),
|
||||||
("warehouse_id", "${var.warehouse_id}"),
|
("warehouse_id", "${var.warehouse_id}"),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
pipeline_tasks.append(task)
|
pipeline_tasks.append(task)
|
||||||
|
|
||||||
schedule = cfg.get("schedule", "0 0 6 * * ?")
|
schedule = cfg.get("schedule", "0 0 6 * * ?")
|
||||||
@@ -156,11 +194,15 @@ class JobsPlugin(DabPlugin):
|
|||||||
|
|
||||||
job = OrderedDict()
|
job = OrderedDict()
|
||||||
job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines"
|
job["name"] = f"{cfg.get('bundle_name', 'stack')}-pipelines"
|
||||||
job["description"] = f"Run all {len(registry)} ACO pipelines in dependency order"
|
job["description"] = (
|
||||||
job["schedule"] = OrderedDict([
|
f"Run all {len(registry)} ACO pipelines in dependency order"
|
||||||
|
)
|
||||||
|
job["schedule"] = OrderedDict(
|
||||||
|
[
|
||||||
("quartz_cron_expression", schedule),
|
("quartz_cron_expression", schedule),
|
||||||
("timezone_id", timezone),
|
("timezone_id", timezone),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
job["job_clusters"] = job_clusters
|
job["job_clusters"] = job_clusters
|
||||||
job["tasks"] = pipeline_tasks
|
job["tasks"] = pipeline_tasks
|
||||||
|
|
||||||
@@ -175,10 +217,12 @@ class SchemasPlugin(DabPlugin):
|
|||||||
return {}
|
return {}
|
||||||
schemas = OrderedDict()
|
schemas = OrderedDict()
|
||||||
for schema_name in sorted(catalog_schemas):
|
for schema_name in sorted(catalog_schemas):
|
||||||
schemas[schema_name] = OrderedDict([
|
schemas[schema_name] = OrderedDict(
|
||||||
|
[
|
||||||
("name", schema_name),
|
("name", schema_name),
|
||||||
("catalog_name", "${var.catalog}"),
|
("catalog_name", "${var.catalog}"),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
return {"schemas": schemas}
|
return {"schemas": schemas}
|
||||||
|
|
||||||
|
|
||||||
@@ -186,14 +230,23 @@ class VolumesPlugin(DabPlugin):
|
|||||||
"""Generate a staging volume for data upload."""
|
"""Generate a staging volume for data upload."""
|
||||||
|
|
||||||
def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict:
|
def resources(self, cfg: dict, registry: dict, catalog_schemas: list) -> dict:
|
||||||
return {"volumes": OrderedDict([
|
return {
|
||||||
("staging", OrderedDict([
|
"volumes": OrderedDict(
|
||||||
|
[
|
||||||
|
(
|
||||||
|
"staging",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("name", "staging"),
|
("name", "staging"),
|
||||||
("catalog_name", "${var.catalog}"),
|
("catalog_name", "${var.catalog}"),
|
||||||
("schema_name", "default"),
|
("schema_name", "default"),
|
||||||
("volume_type", "MANAGED"),
|
("volume_type", "MANAGED"),
|
||||||
])),
|
]
|
||||||
])}
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# ── Core generator ───────────────────────────────────────────────
|
# ── Core generator ───────────────────────────────────────────────
|
||||||
@@ -214,6 +267,7 @@ _DEFAULT_PLUGINS: list[DabPlugin] = [
|
|||||||
def _get_catalog_schemas() -> list[str]:
|
def _get_catalog_schemas() -> list[str]:
|
||||||
try:
|
try:
|
||||||
from aco.lake.catalog import Catalog
|
from aco.lake.catalog import Catalog
|
||||||
|
|
||||||
return Catalog().schemas()
|
return Catalog().schemas()
|
||||||
except Exception:
|
except Exception:
|
||||||
return []
|
return []
|
||||||
@@ -252,19 +306,33 @@ def emit(cfg_data: dict) -> dict[str, str]:
|
|||||||
bundle["bundle"] = OrderedDict([("name", bundle_name)])
|
bundle["bundle"] = OrderedDict([("name", bundle_name)])
|
||||||
|
|
||||||
# Sync SQL files to workspace
|
# Sync SQL files to workspace
|
||||||
bundle["sync"] = OrderedDict([
|
bundle["sync"] = OrderedDict(
|
||||||
|
[
|
||||||
("include", ["bundle/**"]),
|
("include", ["bundle/**"]),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
|
|
||||||
bundle["variables"] = OrderedDict([
|
bundle["variables"] = OrderedDict(
|
||||||
("catalog", OrderedDict([
|
[
|
||||||
|
(
|
||||||
|
"catalog",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("description", "Unity Catalog name for table references"),
|
("description", "Unity Catalog name for table references"),
|
||||||
("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
|
("default", targets_cfg.get("prod", {}).get("catalog", "aco")),
|
||||||
])),
|
]
|
||||||
("warehouse_id", OrderedDict([
|
),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"warehouse_id",
|
||||||
|
OrderedDict(
|
||||||
|
[
|
||||||
("description", "SQL warehouse ID for query execution"),
|
("description", "SQL warehouse ID for query execution"),
|
||||||
])),
|
]
|
||||||
])
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")])
|
bundle["workspace"] = OrderedDict([("host", "${DATABRICKS_HOST}")])
|
||||||
|
|
||||||
@@ -276,17 +344,21 @@ def emit(cfg_data: dict) -> dict[str, str]:
|
|||||||
target["mode"] = tcfg["mode"]
|
target["mode"] = tcfg["mode"]
|
||||||
if tcfg.get("default"):
|
if tcfg.get("default"):
|
||||||
target["default"] = True
|
target["default"] = True
|
||||||
target["variables"] = OrderedDict([
|
target["variables"] = OrderedDict(
|
||||||
|
[
|
||||||
("catalog", tcfg.get("catalog", "aco")),
|
("catalog", tcfg.get("catalog", "aco")),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
if tcfg.get("warehouse_id"):
|
if tcfg.get("warehouse_id"):
|
||||||
target["variables"]["warehouse_id"] = tcfg["warehouse_id"]
|
target["variables"]["warehouse_id"] = tcfg["warehouse_id"]
|
||||||
if tcfg.get("mode") == "production":
|
if tcfg.get("mode") == "production":
|
||||||
run_as = dab_cfg.get("run_as", {})
|
run_as = dab_cfg.get("run_as", {})
|
||||||
if run_as.get("service_principal_name"):
|
if run_as.get("service_principal_name"):
|
||||||
target["run_as"] = OrderedDict([
|
target["run_as"] = OrderedDict(
|
||||||
|
[
|
||||||
("service_principal_name", run_as["service_principal_name"]),
|
("service_principal_name", run_as["service_principal_name"]),
|
||||||
])
|
]
|
||||||
|
)
|
||||||
targets[tname] = target
|
targets[tname] = target
|
||||||
bundle["targets"] = targets
|
bundle["targets"] = targets
|
||||||
|
|
||||||
|
|||||||
@@ -242,7 +242,7 @@ steps:
|
|||||||
UV_PROJECT_ENVIRONMENT: .venv
|
UV_PROJECT_ENVIRONMENT: .venv
|
||||||
commands:
|
commands:
|
||||||
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
|
- uv run pytest tests/ --cov=src --cov-report=term-missing --cov-fail-under=99 -q 2>&1 | tee pytest.out
|
||||||
- uv run python dev/scripts/coverage_badge.py < pytest.out > coverage.svg
|
- uv run python dev/scripts/coverage_badge.py < pytest.out > assets/icons/coverage.svg
|
||||||
depends_on:
|
depends_on:
|
||||||
- lint
|
- lint
|
||||||
|
|
||||||
@@ -273,7 +273,7 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- git config user.name woodpecker-ci
|
- git config user.name woodpecker-ci
|
||||||
- git config user.email ci@${{CI_REPO_OWNER}}.io
|
- git config user.email ci@${{CI_REPO_OWNER}}.io
|
||||||
- git add coverage.svg
|
- git add assets/icons/coverage.svg
|
||||||
- git diff --cached --quiet && echo "no change" && exit 0
|
- git diff --cached --quiet && echo "no change" && exit 0
|
||||||
- git commit -m "update coverage badge [skip ci]"
|
- git commit -m "update coverage badge [skip ci]"
|
||||||
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
|
- git push http://$REGISTRY_USER:$REGISTRY_PASS@gitea:3000/${{CI_REPO}}.git HEAD:main
|
||||||
@@ -293,7 +293,7 @@ steps:
|
|||||||
from_secret: s3_access_key
|
from_secret: s3_access_key
|
||||||
secret_key:
|
secret_key:
|
||||||
from_secret: s3_secret_key
|
from_secret: s3_secret_key
|
||||||
source: "coverage.svg"
|
source: "assets/icons/coverage.svg"
|
||||||
target: /badges/${{CI_REPO}}/
|
target: /badges/${{CI_REPO}}/
|
||||||
path_style: true
|
path_style: true
|
||||||
overwrite: true
|
overwrite: true
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ import argparse
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||||
@@ -33,7 +32,7 @@ CA_SUBJECT = "/CN=Homelab CA/O=fhirworx"
|
|||||||
CA_DAYS = 3650
|
CA_DAYS = 3650
|
||||||
TLS_DAYS = 3650
|
TLS_DAYS = 3650
|
||||||
|
|
||||||
from conf import cfg
|
from conf import cfg # noqa: E402
|
||||||
|
|
||||||
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
|
DOMAIN = os.environ.get("DOMAIN", cfg.platform.domain)
|
||||||
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)
|
HOST_IP = os.environ.get("HOST_IP", cfg.platform.host_ip)
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ Usage:
|
|||||||
# CI_COMMIT_SHA from environment):
|
# CI_COMMIT_SHA from environment):
|
||||||
uv run python dev/scripts/coverage_badge.py --post pytest.out
|
uv run python dev/scripts/coverage_badge.py --post pytest.out
|
||||||
|
|
||||||
# Update styles/coverage.svg in the repo via Gitea API:
|
# Update assets/icons/coverage.svg in the repo via Gitea API:
|
||||||
uv run python dev/scripts/coverage_badge.py --upload coverage.svg
|
uv run python dev/scripts/coverage_badge.py --upload coverage.svg
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -44,7 +44,7 @@ TEMPLATE = """\
|
|||||||
</g>
|
</g>
|
||||||
</svg>"""
|
</svg>"""
|
||||||
|
|
||||||
FILE_PATH = "styles/coverage.svg"
|
FILE_PATH = "assets/icons/coverage.svg"
|
||||||
|
|
||||||
|
|
||||||
def color_for(pct: int) -> str:
|
def color_for(pct: int) -> str:
|
||||||
@@ -73,11 +73,13 @@ def post_status(pct: int) -> None:
|
|||||||
token, url = _gitea_env()
|
token, url = _gitea_env()
|
||||||
sha = os.environ["CI_COMMIT_SHA"]
|
sha = os.environ["CI_COMMIT_SHA"]
|
||||||
endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}"
|
endpoint = f"{url}/api/v1/repos/homelab/stack/statuses/{sha}"
|
||||||
body = json.dumps({
|
body = json.dumps(
|
||||||
|
{
|
||||||
"state": "success",
|
"state": "success",
|
||||||
"description": f"{pct}% coverage",
|
"description": f"{pct}% coverage",
|
||||||
"context": "coverage",
|
"context": "coverage",
|
||||||
}).encode()
|
}
|
||||||
|
).encode()
|
||||||
req = urllib.request.Request(
|
req = urllib.request.Request(
|
||||||
endpoint,
|
endpoint,
|
||||||
data=body,
|
data=body,
|
||||||
@@ -92,7 +94,7 @@ def post_status(pct: int) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def upload_badge(svg_path: str) -> None:
|
def upload_badge(svg_path: str) -> None:
|
||||||
"""Update styles/coverage.svg in the repo via Gitea contents API."""
|
"""Update assets/icons/coverage.svg in the repo via Gitea contents API."""
|
||||||
token, url = _gitea_env()
|
token, url = _gitea_env()
|
||||||
endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}"
|
endpoint = f"{url}/api/v1/repos/homelab/stack/contents/{FILE_PATH}"
|
||||||
svg_data = open(svg_path, "rb").read()
|
svg_data = open(svg_path, "rb").read()
|
||||||
|
|||||||
@@ -40,7 +40,6 @@ def main() -> int:
|
|||||||
for col in COLUMNS:
|
for col in COLUMNS:
|
||||||
for op in ("INSERT", "UPDATE"):
|
for op in ("INSERT", "UPDATE"):
|
||||||
trigger = f"fix_{col}_{op.lower()}"
|
trigger = f"fix_{col}_{op.lower()}"
|
||||||
of_clause = f"OF {col} " if op == "UPDATE" else ""
|
|
||||||
db.execute(f"""
|
db.execute(f"""
|
||||||
CREATE TRIGGER IF NOT EXISTS {trigger}
|
CREATE TRIGGER IF NOT EXISTS {trigger}
|
||||||
AFTER {op} ON items
|
AFTER {op} ON items
|
||||||
|
|||||||
@@ -219,7 +219,11 @@ def generate_module() -> str:
|
|||||||
def main():
|
def main():
|
||||||
"""Generate src/aco/table/alr_filenames.py."""
|
"""Generate src/aco/table/alr_filenames.py."""
|
||||||
output_path = (
|
output_path = (
|
||||||
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "alr_filenames.py"
|
Path(__file__).resolve().parents[2]
|
||||||
|
/ "src"
|
||||||
|
/ "aco"
|
||||||
|
/ "table"
|
||||||
|
/ "alr_filenames.py"
|
||||||
)
|
)
|
||||||
|
|
||||||
code = generate_module()
|
code = generate_module()
|
||||||
@@ -228,9 +232,9 @@ def main():
|
|||||||
output_path.write_text(code)
|
output_path.write_text(code)
|
||||||
|
|
||||||
print(f"Generated {output_path}")
|
print(f"Generated {output_path}")
|
||||||
print(f" 18 ALR table patterns (9 annual + 9 quarterly)")
|
print(" 18 ALR table patterns (9 annual + 9 quarterly)")
|
||||||
print(f" 2 ASR patterns (annual + quarterly)")
|
print(" 2 ASR patterns (annual + quarterly)")
|
||||||
print(f" 4 package patterns")
|
print(" 4 package patterns")
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
|
|||||||
|
|
||||||
Returns dict mapping table_id -> {title, fields: list[dict]}
|
Returns dict mapping table_id -> {title, fields: list[dict]}
|
||||||
"""
|
"""
|
||||||
pdf = pdfplumber.open(pdf_path)
|
pdfplumber.open(pdf_path)
|
||||||
|
|
||||||
# Manual field specifications based on PDF extraction
|
# Manual field specifications based on PDF extraction
|
||||||
# Table 1-1 is on pages 8-15, others follow
|
# Table 1-1 is on pages 8-15, others follow
|
||||||
@@ -163,7 +163,7 @@ def extract_alr_table_specs(pdf_path: str) -> dict:
|
|||||||
{
|
{
|
||||||
"name": f"HCC_Position_{i:02d}",
|
"name": f"HCC_Position_{i:02d}",
|
||||||
"type": "int",
|
"type": "int",
|
||||||
"desc": f"HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
|
"desc": "HCC indicator (0=no, 1=yes) - maps to specific HCC via data dictionary",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -530,7 +530,7 @@ def generate_module(tables: dict) -> str:
|
|||||||
lines.append(" Source: ALRASRGuide.pdf")
|
lines.append(" Source: ALRASRGuide.pdf")
|
||||||
lines.append(' """')
|
lines.append(' """')
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(f' __schema__ = "alr"')
|
lines.append(' __schema__ = "alr"')
|
||||||
lines.append(f' __tablename__ = "{tbl}"')
|
lines.append(f' __tablename__ = "{tbl}"')
|
||||||
|
|
||||||
for field in info["fields"]:
|
for field in info["fields"]:
|
||||||
|
|||||||
@@ -370,7 +370,7 @@ def generate_module(tables: dict) -> str:
|
|||||||
lines.append(" Source: ALRASRGuide.pdf")
|
lines.append(" Source: ALRASRGuide.pdf")
|
||||||
lines.append(' """')
|
lines.append(' """')
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(f' __schema__ = "asr"')
|
lines.append(' __schema__ = "asr"')
|
||||||
lines.append(f' __tablename__ = "{tbl}"')
|
lines.append(f' __tablename__ = "{tbl}"')
|
||||||
|
|
||||||
for field in info["fields"]:
|
for field in info["fields"]:
|
||||||
@@ -398,7 +398,9 @@ def generate_module(tables: dict) -> str:
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
"""Generate src/aco/table/asr.py from ALRASRGuide.pdf."""
|
"""Generate src/aco/table/asr.py from ALRASRGuide.pdf."""
|
||||||
output_path = Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
|
output_path = (
|
||||||
|
Path(__file__).resolve().parents[2] / "src" / "aco" / "table" / "asr.py"
|
||||||
|
)
|
||||||
|
|
||||||
tables = extract_asr_table_specs()
|
tables = extract_asr_table_specs()
|
||||||
code = generate_module(tables)
|
code = generate_module(tables)
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ Source: https://www.cms.gov/files/document/cclf-information-packet.pdf
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
from datetime import date
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pdfplumber
|
import pdfplumber
|
||||||
@@ -206,7 +205,7 @@ def generate_module(tables: dict) -> str:
|
|||||||
lines.append(f" {len(info['fields'])} fields, fixed-width layout.")
|
lines.append(f" {len(info['fields'])} fields, fixed-width layout.")
|
||||||
lines.append(' """')
|
lines.append(' """')
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append(f' __schema__ = "cclf"')
|
lines.append(' __schema__ = "cclf"')
|
||||||
lines.append(f' __tablename__ = "{tbl_name}"')
|
lines.append(f' __tablename__ = "{tbl_name}"')
|
||||||
|
|
||||||
seen_names: set[str] = set()
|
seen_names: set[str] = set()
|
||||||
|
|||||||
@@ -17,8 +17,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import re
|
import re
|
||||||
import textwrap
|
from dataclasses import dataclass
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pdfplumber
|
import pdfplumber
|
||||||
@@ -146,7 +145,7 @@ def parse_entry(raw: str) -> Variable | None:
|
|||||||
|
|
||||||
# Parse the remaining structured fields
|
# Parse the remaining structured fields
|
||||||
# Rejoin everything from LABEL: onward
|
# Rejoin everything from LABEL: onward
|
||||||
body = "\n".join(lines[label_idx:])
|
"\n".join(lines[label_idx:])
|
||||||
|
|
||||||
# Extract each field by finding the label and collecting text
|
# Extract each field by finding the label and collecting text
|
||||||
# until the next label
|
# until the next label
|
||||||
@@ -211,9 +210,9 @@ def variable_to_py(var: Variable) -> str:
|
|||||||
primary_name = var.name.split("\n")[0].strip()
|
primary_name = var.name.split("\n")[0].strip()
|
||||||
|
|
||||||
lines = [
|
lines = [
|
||||||
f'"""',
|
'"""',
|
||||||
f"Variable: {primary_name}",
|
f"Variable: {primary_name}",
|
||||||
f"",
|
"",
|
||||||
]
|
]
|
||||||
|
|
||||||
if var.label and var.label != "—":
|
if var.label and var.label != "—":
|
||||||
@@ -289,7 +288,9 @@ def main() -> None:
|
|||||||
"--pdf",
|
"--pdf",
|
||||||
default=str(
|
default=str(
|
||||||
Path(__file__).resolve().parents[2]
|
Path(__file__).resolve().parents[2]
|
||||||
/ "dev" / "seeds" / "codebook-ffs-claims.pdf"
|
/ "dev"
|
||||||
|
/ "seeds"
|
||||||
|
/ "codebook-ffs-claims.pdf"
|
||||||
),
|
),
|
||||||
help="Path to the CCW codebook PDF",
|
help="Path to the CCW codebook PDF",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -16,11 +16,9 @@ Defaults:
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import re
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import fastexcel
|
import fastexcel
|
||||||
import polars as pl
|
|
||||||
|
|
||||||
# ── Sheets and their slug names ──────────────────────────────────────────────
|
# ── Sheets and their slug names ──────────────────────────────────────────────
|
||||||
|
|
||||||
@@ -169,7 +167,7 @@ def generate_model(
|
|||||||
f"class {cls}(SQLTable):",
|
f"class {cls}(SQLTable):",
|
||||||
f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""',
|
f' """CCW FFS Claims: {claim_slug.upper()} / {section}"""',
|
||||||
"",
|
"",
|
||||||
f' __schema__ = "ccw"',
|
' __schema__ = "ccw"',
|
||||||
f' __tablename__ = "{claim_slug}_{table}"',
|
f' __tablename__ = "{claim_slug}_{table}"',
|
||||||
"",
|
"",
|
||||||
]
|
]
|
||||||
@@ -214,7 +212,9 @@ def main() -> None:
|
|||||||
"--xlsx",
|
"--xlsx",
|
||||||
default=str(
|
default=str(
|
||||||
Path(__file__).resolve().parents[2]
|
Path(__file__).resolve().parents[2]
|
||||||
/ "dev" / "seeds" / "record-layout-ffs-claims.xlsx"
|
/ "dev"
|
||||||
|
/ "seeds"
|
||||||
|
/ "record-layout-ffs-claims.xlsx"
|
||||||
),
|
),
|
||||||
help="Path to the CCW record layout Excel file",
|
help="Path to the CCW record layout Excel file",
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -224,9 +224,9 @@ def generate_function(table: dict) -> str:
|
|||||||
"@nw.narwhalify",
|
"@nw.narwhalify",
|
||||||
f"def {tablename}({param}: FrameT) -> FrameT:",
|
f"def {tablename}({param}: FrameT) -> FrameT:",
|
||||||
f' """Clean and type-cast cms.{tablename}.',
|
f' """Clean and type-cast cms.{tablename}.',
|
||||||
f"",
|
"",
|
||||||
f" {field_count} fields. Numeric columns auto-cast to Float64.",
|
f" {field_count} fields. Numeric columns auto-cast to Float64.",
|
||||||
f' """',
|
' """',
|
||||||
f" return auto_cast({param})",
|
f" return auto_cast({param})",
|
||||||
]
|
]
|
||||||
return "\n".join(lines)
|
return "\n".join(lines)
|
||||||
@@ -236,18 +236,18 @@ def generate_module(domain: str, tables: list[dict]) -> str:
|
|||||||
"""Generate a full express module for a domain group."""
|
"""Generate a full express module for a domain group."""
|
||||||
lines = [
|
lines = [
|
||||||
f'"""CMS express — {domain} domain.',
|
f'"""CMS express — {domain} domain.',
|
||||||
f"",
|
"",
|
||||||
f"Auto-generated narwhals functions for {len(tables)} CMS tables.",
|
f"Auto-generated narwhals functions for {len(tables)} CMS tables.",
|
||||||
f"Each function takes the raw table and returns a typed frame",
|
"Each function takes the raw table and returns a typed frame",
|
||||||
f"with numeric columns cast to Float64.",
|
"with numeric columns cast to Float64.",
|
||||||
f'"""',
|
'"""',
|
||||||
f"",
|
"",
|
||||||
f"from __future__ import annotations",
|
"from __future__ import annotations",
|
||||||
f"",
|
"",
|
||||||
f"import narwhals as nw",
|
"import narwhals as nw",
|
||||||
f"from narwhals.typing import FrameT",
|
"from narwhals.typing import FrameT",
|
||||||
f"",
|
"",
|
||||||
f"from cms.express._helpers import auto_cast",
|
"from cms.express._helpers import auto_cast",
|
||||||
]
|
]
|
||||||
|
|
||||||
for tbl in sorted(tables, key=lambda t: t["tablename"]):
|
for tbl in sorted(tables, key=lambda t: t["tablename"]):
|
||||||
|
|||||||
@@ -14,7 +14,6 @@ import argparse
|
|||||||
import re
|
import re
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from textwrap import dedent, indent
|
|
||||||
|
|
||||||
import duckdb
|
import duckdb
|
||||||
|
|
||||||
@@ -268,7 +267,7 @@ def col_expr_to_nw(expr: str) -> str:
|
|||||||
# date_diff('unit', a, b)
|
# date_diff('unit', a, b)
|
||||||
dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE)
|
dd_m = re.match(r"date_diff\('(\w+)',\s*(.+?),\s*(.+)\)", expr, re.IGNORECASE)
|
||||||
if dd_m:
|
if dd_m:
|
||||||
unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
|
_unit, a, b = dd_m.group(1), dd_m.group(2).strip(), dd_m.group(3).strip()
|
||||||
if alias:
|
if alias:
|
||||||
return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")'
|
return f'({_col_ref(b)} - {_col_ref(a)}).alias("{alias}")'
|
||||||
return f"{_col_ref(b)} - {_col_ref(a)}"
|
return f"{_col_ref(b)} - {_col_ref(a)}"
|
||||||
@@ -712,13 +711,13 @@ def generate_function(
|
|||||||
doc_ops = ", ".join(ops)
|
doc_ops = ", ".join(ops)
|
||||||
|
|
||||||
lines: list[str] = []
|
lines: list[str] = []
|
||||||
lines.append(f"@nw.narwhalify")
|
lines.append("@nw.narwhalify")
|
||||||
lines.append(f"def {func_name}({param_str}) -> FrameT:")
|
lines.append(f"def {func_name}({param_str}) -> FrameT:")
|
||||||
lines.append(f' """Build {schema}.{view_name}')
|
lines.append(f' """Build {schema}.{view_name}')
|
||||||
lines.append(f"")
|
lines.append("")
|
||||||
lines.append(f" Operations: {doc_ops}")
|
lines.append(f" Operations: {doc_ops}")
|
||||||
lines.append(f" Dependencies: {doc_deps}")
|
lines.append(f" Dependencies: {doc_deps}")
|
||||||
lines.append(f' """')
|
lines.append(' """')
|
||||||
|
|
||||||
if "passthrough" in ops:
|
if "passthrough" in ops:
|
||||||
p = params[0].split(":")[0]
|
p = params[0].split(":")[0]
|
||||||
@@ -880,7 +879,7 @@ def _gen_select_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
|||||||
if nw_cols:
|
if nw_cols:
|
||||||
lines.append(f" return {p}.select(")
|
lines.append(f" return {p}.select(")
|
||||||
_emit_exprs(lines, nw_cols)
|
_emit_exprs(lines, nw_cols)
|
||||||
lines.append(f" )")
|
lines.append(" )")
|
||||||
else:
|
else:
|
||||||
lines.append(f" return {p}")
|
lines.append(f" return {p}")
|
||||||
|
|
||||||
@@ -902,13 +901,13 @@ def _gen_filter_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
|||||||
lines.append(f" # {comment_part}")
|
lines.append(f" # {comment_part}")
|
||||||
lines.append(f" return {p}.filter(")
|
lines.append(f" return {p}.filter(")
|
||||||
lines.append(f" {code_part}")
|
lines.append(f" {code_part}")
|
||||||
lines.append(f" )")
|
lines.append(" )")
|
||||||
else:
|
else:
|
||||||
safe = clause.replace('"', "'")[:80]
|
safe = clause.replace('"', "'")[:80]
|
||||||
lines.append(f" # TODO: complex filter: {safe}")
|
lines.append(f" # TODO: complex filter: {safe}")
|
||||||
lines.append(f" return {p}")
|
lines.append(f" return {p}")
|
||||||
else:
|
else:
|
||||||
lines.append(f" # WHERE clause could not be parsed")
|
lines.append(" # WHERE clause could not be parsed")
|
||||||
lines.append(f" return {p}")
|
lines.append(f" return {p}")
|
||||||
|
|
||||||
|
|
||||||
@@ -994,9 +993,9 @@ def _gen_join_body(
|
|||||||
keys_str = ", ".join(f'"{k}"' for k in keys)
|
keys_str = ", ".join(f'"{k}"' for k in keys)
|
||||||
lines.append(f" result = result.group_by({keys_str}).agg()")
|
lines.append(f" result = result.group_by({keys_str}).agg()")
|
||||||
else:
|
else:
|
||||||
lines.append(f" # TODO: complex GROUP BY")
|
lines.append(" # TODO: complex GROUP BY")
|
||||||
|
|
||||||
lines.append(f" return result")
|
lines.append(" return result")
|
||||||
|
|
||||||
|
|
||||||
def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
||||||
@@ -1023,7 +1022,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
|
|||||||
keys.append(c_clean)
|
keys.append(c_clean)
|
||||||
break
|
break
|
||||||
if not keys:
|
if not keys:
|
||||||
lines.append(f" # TODO: complex GROUP BY could not be parsed")
|
lines.append(" # TODO: complex GROUP BY could not be parsed")
|
||||||
lines.append(f" return {p}")
|
lines.append(f" return {p}")
|
||||||
return
|
return
|
||||||
keys_str = ", ".join(f'"{k}"' for k in keys)
|
keys_str = ", ".join(f'"{k}"' for k in keys)
|
||||||
@@ -1045,7 +1044,7 @@ def _gen_group_by_body(lines: list[str], deps: list[tuple[str, str]], sql: str)
|
|||||||
if agg_cols:
|
if agg_cols:
|
||||||
lines.append(f" return {p}.group_by({keys_str}).agg(")
|
lines.append(f" return {p}.group_by({keys_str}).agg(")
|
||||||
_emit_exprs(lines, agg_cols)
|
_emit_exprs(lines, agg_cols)
|
||||||
lines.append(f" )")
|
lines.append(" )")
|
||||||
else:
|
else:
|
||||||
lines.append(f" return {p}.group_by({keys_str}).agg()")
|
lines.append(f" return {p}.group_by({keys_str}).agg()")
|
||||||
|
|
||||||
@@ -1061,20 +1060,20 @@ def _gen_window_body(lines: list[str], deps: list[tuple[str, str]], sql: str) ->
|
|||||||
|
|
||||||
lines.append(f" return {p}.with_columns(")
|
lines.append(f" return {p}.with_columns(")
|
||||||
_emit_exprs(lines, nw_cols)
|
_emit_exprs(lines, nw_cols)
|
||||||
lines.append(f" )")
|
lines.append(" )")
|
||||||
|
|
||||||
|
|
||||||
def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
def _gen_union_body(lines: list[str], deps: list[tuple[str, str]], sql: str) -> None:
|
||||||
params = [ref_to_param(s, t) for s, t in deps]
|
params = [ref_to_param(s, t) for s, t in deps]
|
||||||
if len(params) >= 2:
|
if len(params) >= 2:
|
||||||
lines.append(f" return nw.concat([")
|
lines.append(" return nw.concat([")
|
||||||
for p in params:
|
for p in params:
|
||||||
lines.append(f" {p},")
|
lines.append(f" {p},")
|
||||||
lines.append(f" ])")
|
lines.append(" ])")
|
||||||
elif params:
|
elif params:
|
||||||
lines.append(f" return {params[0]}")
|
lines.append(f" return {params[0]}")
|
||||||
else:
|
else:
|
||||||
lines.append(f" return df")
|
lines.append(" return df")
|
||||||
|
|
||||||
|
|
||||||
# ── Module generation ───────────────────────────────────────────────────────
|
# ── Module generation ───────────────────────────────────────────────────────
|
||||||
@@ -1121,9 +1120,7 @@ def main() -> None:
|
|||||||
default=str(_conf_path("db.aco")),
|
default=str(_conf_path("db.aco")),
|
||||||
help="Path to DuckDB file",
|
help="Path to DuckDB file",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("--out", default="../src/aco/express", help="Output directory")
|
||||||
"--out", default="../src/aco/express", help="Output directory"
|
|
||||||
)
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
db_path = Path(args.db)
|
db_path = Path(args.db)
|
||||||
|
|||||||
@@ -147,13 +147,12 @@ def generate_model(
|
|||||||
escaped = comment.replace('"', '\\"')
|
escaped = comment.replace('"', '\\"')
|
||||||
if nullable:
|
if nullable:
|
||||||
field_lines.append(
|
field_lines.append(
|
||||||
f' {fname}: {py} | None = Field('
|
f" {fname}: {py} | None = Field("
|
||||||
f'default=None, description="{escaped}")'
|
f'default=None, description="{escaped}")'
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
field_lines.append(
|
field_lines.append(
|
||||||
f' {fname}: {py} = Field('
|
f' {fname}: {py} = Field(description="{escaped}")'
|
||||||
f'description="{escaped}")'
|
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
if nullable:
|
if nullable:
|
||||||
@@ -202,14 +201,13 @@ def generate_schema_module(
|
|||||||
return f"{header}\n\n\n{body}\n"
|
return f"{header}\n\n\n{body}\n"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ── Main ────────────────────────────────────────────────────────────────────
|
# ── Main ────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
|
parser = argparse.ArgumentParser(description="Generate Pydantic models from DuckDB")
|
||||||
from conf import cfg, path as _conf_path
|
from conf import cfg
|
||||||
|
from conf import path as _conf_path
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--db",
|
"--db",
|
||||||
|
|||||||
@@ -58,7 +58,6 @@ Design notes
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
|
||||||
import re
|
import re
|
||||||
import textwrap
|
import textwrap
|
||||||
import zipfile
|
import zipfile
|
||||||
@@ -68,7 +67,6 @@ from typing import NamedTuple
|
|||||||
import pdfplumber
|
import pdfplumber
|
||||||
|
|
||||||
# ── Path constants ────────────────────────────────────────────────────────────
|
# ── Path constants ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
from conf import ROOT as _ROOT
|
from conf import ROOT as _ROOT
|
||||||
from conf import path as _conf_path
|
from conf import path as _conf_path
|
||||||
|
|
||||||
@@ -176,8 +174,9 @@ def _extract_from_zip(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
|
def _resolve_pdf(
|
||||||
zotero_root: Path, work_dir: Path) -> Path | None:
|
pdf_basename: str, zip_basename: str | None, zotero_root: Path, work_dir: Path
|
||||||
|
) -> Path | None:
|
||||||
"""Return path to a MIF PDF, extracting from zip if necessary."""
|
"""Return path to a MIF PDF, extracting from zip if necessary."""
|
||||||
direct = _find_in_zotero(pdf_basename, zotero_root)
|
direct = _find_in_zotero(pdf_basename, zotero_root)
|
||||||
if direct:
|
if direct:
|
||||||
@@ -187,8 +186,12 @@ def _resolve_pdf(pdf_basename: str, zip_basename: str | None,
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _resolve_xlsx(xlsx_basename: str | None, zip_basename: str | None,
|
def _resolve_xlsx(
|
||||||
zotero_root: Path, work_dir: Path) -> Path | None:
|
xlsx_basename: str | None,
|
||||||
|
zip_basename: str | None,
|
||||||
|
zotero_root: Path,
|
||||||
|
work_dir: Path,
|
||||||
|
) -> Path | None:
|
||||||
"""Return path to a value set Excel file, extracting from zip if needed."""
|
"""Return path to a value set Excel file, extracting from zip if needed."""
|
||||||
if xlsx_basename is None:
|
if xlsx_basename is None:
|
||||||
return None
|
return None
|
||||||
@@ -329,7 +332,9 @@ def _read_sheet_summaries(xlsx_path: Path) -> dict[str, SheetSummary]:
|
|||||||
for row in rows[:3]:
|
for row in rows[:3]:
|
||||||
non_none = [str(c).strip() for c in row if c is not None]
|
non_none = [str(c).strip() for c in row if c is not None]
|
||||||
if len(non_none) >= 2:
|
if len(non_none) >= 2:
|
||||||
header = [str(c).strip().replace("\n", " ") for c in row if c is not None]
|
header = [
|
||||||
|
str(c).strip().replace("\n", " ") for c in row if c is not None
|
||||||
|
]
|
||||||
break
|
break
|
||||||
summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header)
|
summaries[sheet_name] = SheetSummary(sheet_name, max(0, row_count), header)
|
||||||
wb.close()
|
wb.close()
|
||||||
@@ -407,13 +412,13 @@ def build_uamcc_passages(
|
|||||||
paa4_info = _vs_info("UAMCC PAA4")
|
paa4_info = _vs_info("UAMCC PAA4")
|
||||||
cohort_info = _vs_info("UAMCC Cohort")
|
cohort_info = _vs_info("UAMCC Cohort")
|
||||||
excl_info = _vs_info("UAMCC Exclusions")
|
excl_info = _vs_info("UAMCC Exclusions")
|
||||||
ccs_cm_info = _vs_info("UAMCC CCS-ICD10CM")
|
_vs_info("UAMCC CCS-ICD10CM")
|
||||||
ccs_pcs_info = _vs_info("UAMCC CCS-ICD10PCS")
|
_vs_info("UAMCC CCS-ICD10PCS")
|
||||||
|
|
||||||
cite_pp = _cite(pdf_name, "§3.8", denom_page)
|
cite_pp = _cite(pdf_name, "§3.8", denom_page)
|
||||||
cite_num = _cite(pdf_name, "§3.6–3.7", num_page)
|
_cite(pdf_name, "§3.6–3.7", num_page)
|
||||||
cite_excl = _cite(pdf_name, "§3.10", denom_excl_page)
|
cite_excl = _cite(pdf_name, "§3.10", denom_excl_page)
|
||||||
cite_risk = _cite(pdf_name, "§3.12", risk_page)
|
_cite(pdf_name, "§3.12", risk_page)
|
||||||
cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page)
|
cite_paa = _cite(pdf_name, "§3.7 PAA v4.0", num_d_page)
|
||||||
|
|
||||||
passages: dict[str, str] = {}
|
passages: dict[str, str] = {}
|
||||||
@@ -423,13 +428,13 @@ def build_uamcc_passages(
|
|||||||
f"Return the UAMCC performance period anchor row.\n\n"
|
f"Return the UAMCC performance period anchor row.\n\n"
|
||||||
f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n"
|
f"Source: {_cite(pdf_name, '§1 Effective Date + §2.2', 1)}\n\n"
|
||||||
f"UAMCC §2.2 Measure Description:\n"
|
f"UAMCC §2.2 Measure Description:\n"
|
||||||
f" \"This outcome measure is calculated using 12 consecutive months\n"
|
f' "This outcome measure is calculated using 12 consecutive months\n'
|
||||||
f" of Medicare fee-for-service (FFS) claims data. The measure is a\n"
|
f" of Medicare fee-for-service (FFS) claims data. The measure is a\n"
|
||||||
f" risk-standardized acute admission rate (RSAAR) that adjusts for\n"
|
f" risk-standardized acute admission rate (RSAAR) that adjusts for\n"
|
||||||
f" age, clinical comorbidities, and other clinical and frailty risk\n"
|
f" age, clinical comorbidities, and other clinical and frailty risk\n"
|
||||||
f" factors present at the start of the 12-month measurement period,\n"
|
f" factors present at the start of the 12-month measurement period,\n"
|
||||||
f" as well as social risk factors. Lower RSAARs indicate better\n"
|
f" as well as social risk factors. Lower RSAARs indicate better\n"
|
||||||
f" performance.\"\n\n"
|
f' performance."\n\n'
|
||||||
f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n"
|
f"NQF ID: #2888 (ACO RSAAR Quality Measure)\n"
|
||||||
f"Measurement duration: 12 consecutive months (Jan 1 – Dec 31).\n"
|
f"Measurement duration: 12 consecutive months (Jan 1 – Dec 31).\n"
|
||||||
f"Performance Year: {py_version}.\n\n"
|
f"Performance Year: {py_version}.\n\n"
|
||||||
@@ -443,10 +448,10 @@ def build_uamcc_passages(
|
|||||||
f"Identify each beneficiary's qualifying chronic condition groups.\n\n"
|
f"Identify each beneficiary's qualifying chronic condition groups.\n\n"
|
||||||
f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n"
|
f"Source: {_cite(pdf_name, '§3.9', denom_d_page)}\n\n"
|
||||||
f"UAMCC §3.9 Denominator Details:\n"
|
f"UAMCC §3.9 Denominator Details:\n"
|
||||||
f" \"The cohort is Medicare FFS beneficiaries 66 years of age and\n"
|
f' "The cohort is Medicare FFS beneficiaries 66 years of age and\n'
|
||||||
f" older assigned to the REACH ACO during the measurement period\n"
|
f" older assigned to the REACH ACO during the measurement period\n"
|
||||||
f" with diagnoses that fall into two or more of nine chronic disease\n"
|
f" with diagnoses that fall into two or more of nine chronic disease\n"
|
||||||
f" groups.\"\n\n"
|
f' groups."\n\n'
|
||||||
f"Nine chronic disease groups (MIF §3.9, pp.7–8):\n"
|
f"Nine chronic disease groups (MIF §3.9, pp.7–8):\n"
|
||||||
f" 1. Acute myocardial infarction (AMI)\n"
|
f" 1. Acute myocardial infarction (AMI)\n"
|
||||||
f" 2. Alzheimer's disease and related disorders or senile dementia\n"
|
f" 2. Alzheimer's disease and related disorders or senile dementia\n"
|
||||||
@@ -471,7 +476,7 @@ def build_uamcc_passages(
|
|||||||
f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n"
|
f"Build the UAMCC denominator: MCC-eligible beneficiaries aged ≥66.\n\n"
|
||||||
f"Source: {cite_pp}\n\n"
|
f"Source: {cite_pp}\n\n"
|
||||||
f"UAMCC §3.8 Denominator Statement:\n"
|
f"UAMCC §3.8 Denominator Statement:\n"
|
||||||
f" \"{denom_stmt_trimmed[:400]}\"\n\n"
|
f' "{denom_stmt_trimmed[:400]}"\n\n'
|
||||||
f"Inclusion criteria (MIF §3.9):\n"
|
f"Inclusion criteria (MIF §3.9):\n"
|
||||||
f" 1. Age ≥66 at the first day of the measurement period\n"
|
f" 1. Age ≥66 at the first day of the measurement period\n"
|
||||||
f" 2. Two or more distinct MCC groups identified in the lookback year\n"
|
f" 2. Two or more distinct MCC groups identified in the lookback year\n"
|
||||||
@@ -501,13 +506,15 @@ def build_uamcc_passages(
|
|||||||
)
|
)
|
||||||
|
|
||||||
# ── Planned admission (PAA) ───────────────────────────────────────
|
# ── Planned admission (PAA) ───────────────────────────────────────
|
||||||
paa_snippet = ""
|
|
||||||
if num_detail:
|
if num_detail:
|
||||||
# Find the PAA description paragraph
|
# Find the PAA description paragraph
|
||||||
m = re.search(r"planned admission algorithm.+?(?=\n\n|\Z)", num_detail,
|
m = re.search(
|
||||||
re.DOTALL | re.IGNORECASE)
|
r"planned admission algorithm.+?(?=\n\n|\Z)",
|
||||||
|
num_detail,
|
||||||
|
re.DOTALL | re.IGNORECASE,
|
||||||
|
)
|
||||||
if m:
|
if m:
|
||||||
paa_snippet = m.group(0)[:600]
|
m.group(0)[:600]
|
||||||
passages["uamcc_int_planned_admission"] = (
|
passages["uamcc_int_planned_admission"] = (
|
||||||
f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n"
|
f"Apply PAA v4.0 {py_version} to classify inpatient admissions as planned.\n\n"
|
||||||
f"Source: {cite_paa}\n\n"
|
f"Source: {cite_paa}\n\n"
|
||||||
@@ -516,7 +523,7 @@ def build_uamcc_passages(
|
|||||||
f" Readmission Algorithm Version 4.0, which CMS originally created\n"
|
f" Readmission Algorithm Version 4.0, which CMS originally created\n"
|
||||||
f" to identify planned readmissions for the hospital-wide readmission\n"
|
f" to identify planned readmissions for the hospital-wide readmission\n"
|
||||||
f" measure. In brief, the algorithm uses a flowchart and four tables\n"
|
f" measure. In brief, the algorithm uses a flowchart and four tables\n"
|
||||||
f" of procedure and/or discharge diagnosis categories.\"\n\n"
|
f' of procedure and/or discharge diagnosis categories."\n\n'
|
||||||
f"PAA Rules (first match wins):\n"
|
f"PAA Rules (first match wins):\n"
|
||||||
f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n"
|
f" Rule 1 — Any procedure in an always-planned CCS category (PAA1).\n"
|
||||||
f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n"
|
f" Rule 2 — Principal diagnosis in an always-planned CCS diagnosis\n"
|
||||||
@@ -562,14 +569,14 @@ def build_uamcc_passages(
|
|||||||
f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n"
|
f"Calculate at-risk person-time for each UAMCC-eligible beneficiary.\n\n"
|
||||||
f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n"
|
f"Source: {_cite(pdf_name, '§3.11', denom_d_page)}\n\n"
|
||||||
f"UAMCC §3.11 Denominator Exclusion Details:\n"
|
f"UAMCC §3.11 Denominator Exclusion Details:\n"
|
||||||
f" \"Persons are considered at risk for admission if they are alive,\n"
|
f' "Persons are considered at risk for admission if they are alive,\n'
|
||||||
f" enrolled in Medicare FFS, and not admitted to an acute care\n"
|
f" enrolled in Medicare FFS, and not admitted to an acute care\n"
|
||||||
f" hospital. In addition to time spent in the hospital, excluded\n"
|
f" hospital. In addition to time spent in the hospital, excluded\n"
|
||||||
f" from at-risk time are:\n"
|
f" from at-risk time are:\n"
|
||||||
f" (1) time spent in an SNF or acute rehabilitation facility;\n"
|
f" (1) time spent in an SNF or acute rehabilitation facility;\n"
|
||||||
f" (2) time within 10 days following discharge from a hospital,\n"
|
f" (2) time within 10 days following discharge from a hospital,\n"
|
||||||
f" SNF, or acute rehabilitation facility;\n"
|
f" SNF, or acute rehabilitation facility;\n"
|
||||||
f" (3) time after entering hospice care.\"\n\n"
|
f' (3) time after entering hospice care."\n\n'
|
||||||
f"Person-years = at_risk_days / 365.25\n\n"
|
f"Person-years = at_risk_days / 365.25\n\n"
|
||||||
f"Performance Year: {py_version}."
|
f"Performance Year: {py_version}."
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import textwrap
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pdfplumber
|
import pdfplumber
|
||||||
@@ -29,10 +28,7 @@ import pdfplumber
|
|||||||
_ROOT = Path(__file__).resolve().parents[2]
|
_ROOT = Path(__file__).resolve().parents[2]
|
||||||
_SEEDS = _ROOT / "dev" / "seeds"
|
_SEEDS = _ROOT / "dev" / "seeds"
|
||||||
|
|
||||||
PDF_PATH = (
|
PDF_PATH = _SEEDS / "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
|
||||||
_SEEDS
|
|
||||||
/ "PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf"
|
|
||||||
)
|
|
||||||
ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py"
|
ALIGNMENT_PATH = _ROOT / "src" / "aco" / "table" / "reach_alignment.py"
|
||||||
FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py"
|
FINANCE_PATH = _ROOT / "src" / "aco" / "table" / "reach_finance.py"
|
||||||
QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py"
|
QUALITY_PATH = _ROOT / "src" / "aco" / "table" / "reach_quality.py"
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ Source: dev/PY2023 ACO REACH Reporting and Data Sharing Overview_v20232203.pdf
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
# Define all table specifications based on extracted data dictionary
|
# Define all table specifications based on extracted data dictionary
|
||||||
|
|||||||
@@ -336,7 +336,9 @@ def main():
|
|||||||
"""Generate REACH participants table model and rex parser."""
|
"""Generate REACH participants table model and rex parser."""
|
||||||
project_root = Path(__file__).resolve().parents[2]
|
project_root = Path(__file__).resolve().parents[2]
|
||||||
xlsx_path = (
|
xlsx_path = (
|
||||||
project_root / "dev" / "seeds"
|
project_root
|
||||||
|
/ "dev"
|
||||||
|
/ "seeds"
|
||||||
/ "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx"
|
/ "ACO_REACH_bulk_upload_participants 5-19 (1).xlsx"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -79,15 +79,12 @@ def sanitize_field(name: str) -> str:
|
|||||||
def get_tables(con: sqlite3.Connection) -> list[str]:
|
def get_tables(con: sqlite3.Connection) -> list[str]:
|
||||||
"""Return all user tables, sorted alphabetically."""
|
"""Return all user tables, sorted alphabetically."""
|
||||||
rows = con.execute(
|
rows = con.execute(
|
||||||
"SELECT name FROM sqlite_master WHERE type='table' "
|
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||||
"ORDER BY name"
|
|
||||||
).fetchall()
|
).fetchall()
|
||||||
return [r[0] for r in rows]
|
return [r[0] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
def get_columns(
|
def get_columns(con: sqlite3.Connection, table: str) -> list[tuple[str, str, bool]]:
|
||||||
con: sqlite3.Connection, table: str
|
|
||||||
) -> list[tuple[str, str, bool]]:
|
|
||||||
"""Return [(col_name, sqlite_type, nullable), ...].
|
"""Return [(col_name, sqlite_type, nullable), ...].
|
||||||
|
|
||||||
SQLite PRAGMA table_info returns:
|
SQLite PRAGMA table_info returns:
|
||||||
|
|||||||
@@ -63,9 +63,7 @@ def _get_item_tags(zcon: sqlite3.Connection, item_id: int) -> list[str]:
|
|||||||
return [r["name"] for r in rows]
|
return [r["name"] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
def _get_item_collections(
|
def _get_item_collections(zcon: sqlite3.Connection, item_id: int) -> list[str]:
|
||||||
zcon: sqlite3.Connection, item_id: int
|
|
||||||
) -> list[str]:
|
|
||||||
"""Read collection keys for a Zotero item."""
|
"""Read collection keys for a Zotero item."""
|
||||||
rows = zcon.execute(
|
rows = zcon.execute(
|
||||||
"""SELECT c.key FROM collectionItems ci
|
"""SELECT c.key FROM collectionItems ci
|
||||||
@@ -76,9 +74,7 @@ def _get_item_collections(
|
|||||||
return [r["key"] for r in rows]
|
return [r["key"] for r in rows]
|
||||||
|
|
||||||
|
|
||||||
def _get_item_creators(
|
def _get_item_creators(zcon: sqlite3.Connection, item_id: int) -> list[dict]:
|
||||||
zcon: sqlite3.Connection, item_id: int
|
|
||||||
) -> list[dict]:
|
|
||||||
"""Read creators for a Zotero item."""
|
"""Read creators for a Zotero item."""
|
||||||
rows = zcon.execute(
|
rows = zcon.execute(
|
||||||
"""SELECT c.firstName, c.lastName, ct.creatorType,
|
"""SELECT c.firstName, c.lastName, ct.creatorType,
|
||||||
@@ -101,9 +97,7 @@ def _get_item_creators(
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
def _classify_item(
|
def _classify_item(zotero_type: str, fields: dict) -> str:
|
||||||
zotero_type: str, fields: dict
|
|
||||||
) -> str:
|
|
||||||
"""Map Zotero itemType + fields to bib item_type."""
|
"""Map Zotero itemType + fields to bib item_type."""
|
||||||
if zotero_type == "statute":
|
if zotero_type == "statute":
|
||||||
code = fields.get("code", "")
|
code = fields.get("code", "")
|
||||||
@@ -134,14 +128,16 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
|||||||
rule_type = part[6:]
|
rule_type = part[6:]
|
||||||
elif part.startswith("Effective: "):
|
elif part.startswith("Effective: "):
|
||||||
eff_date = part[11:]
|
eff_date = part[11:]
|
||||||
return json.dumps({
|
return json.dumps(
|
||||||
|
{
|
||||||
"fr_volume": fields.get("codeNumber", ""),
|
"fr_volume": fields.get("codeNumber", ""),
|
||||||
"fr_page": fields.get("pages", ""),
|
"fr_page": fields.get("pages", ""),
|
||||||
"document_number": doc_num,
|
"document_number": doc_num,
|
||||||
"cms_id": fields.get("session", ""),
|
"cms_id": fields.get("session", ""),
|
||||||
"rule_type": rule_type,
|
"rule_type": rule_type,
|
||||||
"effective_date": eff_date,
|
"effective_date": eff_date,
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
if item_type == "regulation":
|
if item_type == "regulation":
|
||||||
history = fields.get("history", "")
|
history = fields.get("history", "")
|
||||||
@@ -152,13 +148,15 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
|||||||
part = segment[5:]
|
part = segment[5:]
|
||||||
elif segment.startswith("Authority: "):
|
elif segment.startswith("Authority: "):
|
||||||
authority = segment[11:]
|
authority = segment[11:]
|
||||||
return json.dumps({
|
return json.dumps(
|
||||||
|
{
|
||||||
"cfr_title": fields.get("codeNumber", ""),
|
"cfr_title": fields.get("codeNumber", ""),
|
||||||
"cfr_part": part,
|
"cfr_part": part,
|
||||||
"cfr_section": fields.get("section", ""),
|
"cfr_section": fields.get("section", ""),
|
||||||
"authority": authority,
|
"authority": authority,
|
||||||
"effective_date": fields.get("dateEnacted", ""),
|
"effective_date": fields.get("dateEnacted", ""),
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
if item_type == "manual":
|
if item_type == "manual":
|
||||||
extra = fields.get("extra", "")
|
extra = fields.get("extra", "")
|
||||||
@@ -169,33 +167,37 @@ def _build_extra_json(item_type: str, fields: dict) -> str:
|
|||||||
chapter = fields.get("seriesNumber", "")
|
chapter = fields.get("seriesNumber", "")
|
||||||
if chapter.startswith("Chapter "):
|
if chapter.startswith("Chapter "):
|
||||||
chapter = chapter[8:]
|
chapter = chapter[8:]
|
||||||
return json.dumps({
|
return json.dumps(
|
||||||
|
{
|
||||||
"manual_name": fields.get("seriesTitle", ""),
|
"manual_name": fields.get("seriesTitle", ""),
|
||||||
"pub_number": fields.get("reportNumber", ""),
|
"pub_number": fields.get("reportNumber", ""),
|
||||||
"chapter": chapter,
|
"chapter": chapter,
|
||||||
"transmittal": transmittal,
|
"transmittal": transmittal,
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
if item_type == "download":
|
if item_type == "download":
|
||||||
extra = fields.get("extra", "")
|
extra = fields.get("extra", "")
|
||||||
file_urls: list[str] = []
|
file_urls: list[str] = []
|
||||||
for line in extra.split("\n"):
|
for line in extra.split("\n"):
|
||||||
if line.startswith("Files: "):
|
if line.startswith("Files: "):
|
||||||
file_urls = [
|
file_urls = [u.strip() for u in line[7:].split(";") if u.strip()]
|
||||||
u.strip() for u in line[7:].split(";") if u.strip()
|
return json.dumps(
|
||||||
]
|
{
|
||||||
return json.dumps({
|
|
||||||
"page_type": "",
|
"page_type": "",
|
||||||
"file_urls": file_urls,
|
"file_urls": file_urls,
|
||||||
"year": None,
|
"year": None,
|
||||||
"quarter": "",
|
"quarter": "",
|
||||||
"website_title": fields.get("websiteTitle", ""),
|
"website_title": fields.get("websiteTitle", ""),
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
if item_type == "source":
|
if item_type == "source":
|
||||||
return json.dumps({
|
return json.dumps(
|
||||||
|
{
|
||||||
"doc_type": fields.get("type", ""),
|
"doc_type": fields.get("type", ""),
|
||||||
})
|
}
|
||||||
|
)
|
||||||
|
|
||||||
return "{}"
|
return "{}"
|
||||||
|
|
||||||
@@ -255,8 +257,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
|||||||
parent_id = row["id"]
|
parent_id = row["id"]
|
||||||
|
|
||||||
bcon.execute(
|
bcon.execute(
|
||||||
"INSERT OR IGNORE INTO collections (key, name, parent_id) "
|
"INSERT OR IGNORE INTO collections (key, name, parent_id) VALUES (?, ?, ?)",
|
||||||
"VALUES (?, ?, ?)",
|
|
||||||
(zc["key"], zc["collectionName"], parent_id),
|
(zc["key"], zc["collectionName"], parent_id),
|
||||||
)
|
)
|
||||||
counts["collections"] += 1
|
counts["collections"] += 1
|
||||||
@@ -322,8 +323,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
|||||||
for tag_name in ztags:
|
for tag_name in ztags:
|
||||||
tag_id = store._ensure_tag(tag_name)
|
tag_id = store._ensure_tag(tag_name)
|
||||||
bcon.execute(
|
bcon.execute(
|
||||||
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) "
|
"INSERT OR IGNORE INTO item_tags (item_id, tag_id) VALUES (?, ?)",
|
||||||
"VALUES (?, ?)",
|
|
||||||
(bib_id, tag_id),
|
(bib_id, tag_id),
|
||||||
)
|
)
|
||||||
counts["tags"] += 1
|
counts["tags"] += 1
|
||||||
@@ -346,16 +346,14 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
|||||||
for cr in creators:
|
for cr in creators:
|
||||||
# Find or create creator
|
# Find or create creator
|
||||||
crow = bcon.execute(
|
crow = bcon.execute(
|
||||||
"SELECT id FROM creators "
|
"SELECT id FROM creators WHERE first_name = ? AND last_name = ?",
|
||||||
"WHERE first_name = ? AND last_name = ?",
|
|
||||||
(cr["first_name"], cr["last_name"]),
|
(cr["first_name"], cr["last_name"]),
|
||||||
).fetchone()
|
).fetchone()
|
||||||
if crow:
|
if crow:
|
||||||
creator_id = crow["id"]
|
creator_id = crow["id"]
|
||||||
else:
|
else:
|
||||||
cur = bcon.execute(
|
cur = bcon.execute(
|
||||||
"INSERT INTO creators (first_name, last_name) "
|
"INSERT INTO creators (first_name, last_name) VALUES (?, ?)",
|
||||||
"VALUES (?, ?)",
|
|
||||||
(cr["first_name"], cr["last_name"]),
|
(cr["first_name"], cr["last_name"]),
|
||||||
)
|
)
|
||||||
creator_id = cur.lastrowid
|
creator_id = cur.lastrowid
|
||||||
@@ -462,9 +460,7 @@ def migrate(src: str, dst: str) -> dict[str, int]:
|
|||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(description="Migrate Zotero SQLite to bib SQLite")
|
||||||
description="Migrate Zotero SQLite to bib SQLite"
|
|
||||||
)
|
|
||||||
from conf import path as _conf_path
|
from conf import path as _conf_path
|
||||||
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|||||||
@@ -60,7 +60,6 @@ from typing import Any
|
|||||||
import polars as pl
|
import polars as pl
|
||||||
|
|
||||||
# ── Path constants ────────────────────────────────────────────────────────────
|
# ── Path constants ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
from conf import ROOT as _ROOT
|
from conf import ROOT as _ROOT
|
||||||
from conf import path as _conf_path
|
from conf import path as _conf_path
|
||||||
|
|
||||||
|
|||||||
@@ -319,16 +319,12 @@ def _ensure_value(con: sqlite3.Connection, value: str) -> int:
|
|||||||
).fetchone()
|
).fetchone()
|
||||||
if row:
|
if row:
|
||||||
return row[0]
|
return row[0]
|
||||||
cur = con.execute(
|
cur = con.execute("INSERT INTO itemDataValues (value) VALUES (?)", (value,))
|
||||||
"INSERT INTO itemDataValues (value) VALUES (?)", (value,)
|
|
||||||
)
|
|
||||||
return cur.lastrowid
|
return cur.lastrowid
|
||||||
|
|
||||||
|
|
||||||
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
|
def _ensure_tag(con: sqlite3.Connection, name: str) -> int:
|
||||||
row = con.execute(
|
row = con.execute("SELECT tagID FROM tags WHERE name = ?", (name,)).fetchone()
|
||||||
"SELECT tagID FROM tags WHERE name = ?", (name,)
|
|
||||||
).fetchone()
|
|
||||||
if row:
|
if row:
|
||||||
return row[0]
|
return row[0]
|
||||||
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
|
cur = con.execute("INSERT INTO tags (name) VALUES (?)", (name,))
|
||||||
@@ -342,8 +338,7 @@ def _set_zotero_field(
|
|||||||
return
|
return
|
||||||
value_id = _ensure_value(con, value)
|
value_id = _ensure_value(con, value)
|
||||||
con.execute(
|
con.execute(
|
||||||
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) "
|
"INSERT OR REPLACE INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
|
||||||
"VALUES (?, ?, ?)",
|
|
||||||
(item_id, field_id, value_id),
|
(item_id, field_id, value_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -392,8 +387,7 @@ def tag_existing_zotero(
|
|||||||
for tag_name in all_tags:
|
for tag_name in all_tags:
|
||||||
tag_id = _ensure_tag(con, tag_name)
|
tag_id = _ensure_tag(con, tag_name)
|
||||||
con.execute(
|
con.execute(
|
||||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
|
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
|
||||||
"VALUES (?, ?, 0)",
|
|
||||||
(parent_id, tag_id),
|
(parent_id, tag_id),
|
||||||
)
|
)
|
||||||
return True
|
return True
|
||||||
@@ -416,9 +410,9 @@ def register_in_zotero(
|
|||||||
return
|
return
|
||||||
|
|
||||||
now = _now_iso()
|
now = _now_iso()
|
||||||
next_id = con.execute(
|
next_id = con.execute("SELECT COALESCE(MAX(itemID), 0) + 1 FROM items").fetchone()[
|
||||||
"SELECT COALESCE(MAX(itemID), 0) + 1 FROM items"
|
0
|
||||||
).fetchone()[0]
|
]
|
||||||
parent_key = _zotero_key()
|
parent_key = _zotero_key()
|
||||||
|
|
||||||
# Create parent webpage item
|
# Create parent webpage item
|
||||||
@@ -434,9 +428,7 @@ def register_in_zotero(
|
|||||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url)
|
_set_zotero_field(con, next_id, ZOTERO_FIELD_URL, spec.url)
|
||||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
|
_set_zotero_field(con, next_id, ZOTERO_FIELD_DATE, now[:10])
|
||||||
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
|
_set_zotero_field(con, next_id, ZOTERO_FIELD_ACCESS_DATE, now)
|
||||||
_set_zotero_field(
|
_set_zotero_field(con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal")
|
||||||
con, next_id, ZOTERO_FIELD_WEBSITE_TYPE, "Government Data Portal"
|
|
||||||
)
|
|
||||||
_set_zotero_field(
|
_set_zotero_field(
|
||||||
con,
|
con,
|
||||||
next_id,
|
next_id,
|
||||||
@@ -449,8 +441,7 @@ def register_in_zotero(
|
|||||||
for tag_name in all_tags:
|
for tag_name in all_tags:
|
||||||
tag_id = _ensure_tag(con, tag_name)
|
tag_id = _ensure_tag(con, tag_name)
|
||||||
con.execute(
|
con.execute(
|
||||||
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) "
|
"INSERT OR IGNORE INTO itemTags (itemID, tagID, type) VALUES (?, ?, 0)",
|
||||||
"VALUES (?, ?, 0)",
|
|
||||||
(next_id, tag_id),
|
(next_id, tag_id),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -464,13 +455,9 @@ def register_in_zotero(
|
|||||||
|
|
||||||
# Copy to Zotero storage
|
# Copy to Zotero storage
|
||||||
storage_dir = ZOTERO_STORAGE / att_key
|
storage_dir = ZOTERO_STORAGE / att_key
|
||||||
subprocess.run(
|
subprocess.run(["sudo", "mkdir", "-p", str(storage_dir)], check=True)
|
||||||
["sudo", "mkdir", "-p", str(storage_dir)], check=True
|
|
||||||
)
|
|
||||||
dest = storage_dir / seed_path.name
|
dest = storage_dir / seed_path.name
|
||||||
subprocess.run(
|
subprocess.run(["sudo", "cp", str(seed_path), str(dest)], check=True)
|
||||||
["sudo", "cp", str(seed_path), str(dest)], check=True
|
|
||||||
)
|
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
|
["sudo", "chown", "-R", "100999:100999", str(storage_dir)],
|
||||||
check=True,
|
check=True,
|
||||||
@@ -496,8 +483,7 @@ def register_in_zotero(
|
|||||||
)
|
)
|
||||||
title_val = _ensure_value(con, seed_path.name)
|
title_val = _ensure_value(con, seed_path.name)
|
||||||
con.execute(
|
con.execute(
|
||||||
"INSERT INTO itemData (itemID, fieldID, valueID) "
|
"INSERT INTO itemData (itemID, fieldID, valueID) VALUES (?, ?, ?)",
|
||||||
"VALUES (?, ?, ?)",
|
|
||||||
(att_id, ZOTERO_FIELD_TITLE, title_val),
|
(att_id, ZOTERO_FIELD_TITLE, title_val),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -567,7 +553,7 @@ def pull_from_zotero(*, dry_run: bool = False) -> int:
|
|||||||
zpath = row[0] # "storage:filename"
|
zpath = row[0] # "storage:filename"
|
||||||
if not zpath.startswith("storage:"):
|
if not zpath.startswith("storage:"):
|
||||||
continue
|
continue
|
||||||
filename = zpath[len("storage:"):]
|
filename = zpath[len("storage:") :]
|
||||||
|
|
||||||
dest = SEEDS_DIR / filename
|
dest = SEEDS_DIR / filename
|
||||||
if dest.exists():
|
if dest.exists():
|
||||||
|
|||||||
@@ -97,11 +97,11 @@ def main():
|
|||||||
print(f" ... and {len(tables) - 3} more tables")
|
print(f" ... and {len(tables) - 3} more tables")
|
||||||
|
|
||||||
print("\n" + "=" * 70)
|
print("\n" + "=" * 70)
|
||||||
print(f"\nTo create these in Unity Catalog, run without --show-schemas flag")
|
print("\nTo create these in Unity Catalog, run without --show-schemas flag")
|
||||||
return
|
return
|
||||||
|
|
||||||
# Create Unity client
|
# Create Unity client
|
||||||
print(f"Connecting to Unity Catalog...")
|
print("Connecting to Unity Catalog...")
|
||||||
print(f" Workspace ID: {args.workspace_id}")
|
print(f" Workspace ID: {args.workspace_id}")
|
||||||
print(f" Catalog: {args.catalog}")
|
print(f" Catalog: {args.catalog}")
|
||||||
print()
|
print()
|
||||||
@@ -114,7 +114,7 @@ def main():
|
|||||||
# Test connection
|
# Test connection
|
||||||
try:
|
try:
|
||||||
catalogs = client.list_catalogs()
|
catalogs = client.list_catalogs()
|
||||||
print(f"✓ Connected to Unity Catalog")
|
print("✓ Connected to Unity Catalog")
|
||||||
print(f" Found {len(catalogs)} existing catalogs:")
|
print(f" Found {len(catalogs)} existing catalogs:")
|
||||||
for cat in catalogs:
|
for cat in catalogs:
|
||||||
print(f" - {cat.name}")
|
print(f" - {cat.name}")
|
||||||
@@ -130,7 +130,7 @@ def main():
|
|||||||
print("=" * 70)
|
print("=" * 70)
|
||||||
print()
|
print()
|
||||||
|
|
||||||
print(f"Setting up catalog structure...")
|
print("Setting up catalog structure...")
|
||||||
print()
|
print()
|
||||||
|
|
||||||
report = setup_catalog_from_schemas(
|
report = setup_catalog_from_schemas(
|
||||||
|
|||||||
@@ -20,11 +20,11 @@ fi
|
|||||||
|
|
||||||
# Image definitions — matches stack.toml [images]
|
# Image definitions — matches stack.toml [images]
|
||||||
declare -A DOCKERFILES=(
|
declare -A DOCKERFILES=(
|
||||||
[api]="api/Dockerfile"
|
[api]="infra/images/api.Dockerfile"
|
||||||
[notebooks]="notebooks/Dockerfile"
|
[notebooks]="infra/images/notebooks.Dockerfile"
|
||||||
[docs]="docs/Dockerfile"
|
[docs]="infra/images/docs.Dockerfile"
|
||||||
[zotero]="zotero/Dockerfile"
|
[zotero]="infra/images/zotero.Dockerfile"
|
||||||
[mc]="rustfs/Dockerfile.mc"
|
[mc]="infra/images/mc.Dockerfile"
|
||||||
)
|
)
|
||||||
|
|
||||||
declare -A CONTEXTS=(
|
declare -A CONTEXTS=(
|
||||||
@@ -32,7 +32,7 @@ declare -A CONTEXTS=(
|
|||||||
[notebooks]="notebooks/"
|
[notebooks]="notebooks/"
|
||||||
[docs]="."
|
[docs]="."
|
||||||
[zotero]="zotero/"
|
[zotero]="zotero/"
|
||||||
[mc]="rustfs/"
|
[mc]="infra/rustfs/"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Select images to test
|
# Select images to test
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ import os
|
|||||||
|
|
||||||
from aco.lake import IcebergContext, UnityClient
|
from aco.lake import IcebergContext, UnityClient
|
||||||
from aco.lake.catalog import Catalog
|
from aco.lake.catalog import Catalog
|
||||||
from aco.lake.engine import execute
|
|
||||||
|
|
||||||
|
|
||||||
def example_unity_client_basics():
|
def example_unity_client_basics():
|
||||||
@@ -185,7 +184,7 @@ def example_iceberg_context():
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
print(f"✓ IcebergContext configured")
|
print("✓ IcebergContext configured")
|
||||||
print(f" Catalog URI: {ctx.catalog_uri}")
|
print(f" Catalog URI: {ctx.catalog_uri}")
|
||||||
print(f" Warehouse: {ctx.warehouse}")
|
print(f" Warehouse: {ctx.warehouse}")
|
||||||
print()
|
print()
|
||||||
@@ -222,7 +221,7 @@ def example_pipeline_execution():
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
ctx = IcebergContext(
|
IcebergContext(
|
||||||
catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg",
|
catalog_uri=f"https://dbc-{workspace_id}.cloud.databricks.com/api/2.1/unity-catalog/iceberg",
|
||||||
warehouse="aco_dev",
|
warehouse="aco_dev",
|
||||||
catalog=catalog,
|
catalog=catalog,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
/* LOCH — deep-navy design system — matches styles/inject.css */
|
/* LOCH — deep-navy design system — matches assets/css/inject.css */
|
||||||
|
|
||||||
:root {
|
:root {
|
||||||
--ifm-color-primary: #4488dd;
|
--ifm-color-primary: #4488dd;
|
||||||
|
|||||||
0
infra/polaris/.gitkeep
Normal file
0
infra/polaris/.gitkeep
Normal file
@@ -17,7 +17,7 @@ Usage::
|
|||||||
CLI::
|
CLI::
|
||||||
|
|
||||||
uv run python -m aco.dag
|
uv run python -m aco.dag
|
||||||
uv run python -m aco.dag -p readmissions --html dag.html
|
uv run python -m aco.dag -p readmissions --html assets/dag.html
|
||||||
uv run python -m aco.dag --dot dag.dot
|
uv run python -m aco.dag --dot dag.dot
|
||||||
uv run python -m aco.dag -p pharmacy --mermaid
|
uv run python -m aco.dag -p pharmacy --mermaid
|
||||||
"""
|
"""
|
||||||
|
|||||||
Reference in New Issue
Block a user