Files
stack/src/aco/dag.py
kert c81992184a
Some checks failed
CI / skinny-install (aco) (push) Successful in 54s
CI / skinny-install (api) (push) Successful in 32s
CI / skinny-install (bcda) (push) Successful in 33s
CI / skinny-install (bib) (push) Successful in 30s
CI / skinny-install (bls) (push) Successful in 26s
CI / lint-test (push) Successful in 3m19s
CI / skinny-install (ccw) (push) Successful in 27s
CI / skinny-install (cms) (push) Successful in 32s
CI / skinny-install (cli) (push) Successful in 38s
CI / skinny-install (conf) (push) Successful in 33s
CI / skinny-install (opps) (push) Successful in 43s
Infra CI / notebooks (push) Successful in 7s
Infra CI / zotero (push) Successful in 7s
Infra CI / docs (push) Failing after 10s
Infra CI / api (push) Successful in 13s
Infra CI / mc (push) Successful in 7s
CI / skinny-install (perf) (push) Successful in 31s
CI / skinny-install (pfs) (push) Successful in 30s
CI / skinny-install (rex) (push) Successful in 31s
Deploy / build-scan-report (push) Failing after 1m47s
feat: port fhirworx theme — editorial redesign based on hti5.org
Rip and replace nature/loch with fhirworx — a Federal Register / data
journalism aesthetic inspired by hti5.org.

Typography: Playfair Display for display headings, Source Serif 4 for
reading body, JetBrains Mono reserved for IDs, labels, metadata (docket
codes, commit hashes, query editors, file paths, ports). Stripped
carpet-bombed font-family: mono !important across inject.css and
per-service sheets; kept mono only where load-bearing (th, pre/code,
query editors, span IDs, badge/chip/tag, bucket names, query IDs).

Palette: warm cream (#F7F5F0) background, near-black warm foreground,
deep federal navy (#1C2B3A) primary/sidebar, indigo chart scale +
teal (support) + amber (opposition) semantic accents. PALETTE extended
to 8 colours (+ plum, gray, sepia) for categorical notebook charts.

Rename (no backwards compat):
- assets/nature.py → assets/fhirworx.py
- assets/css/loch.css → assets/css/fhirworx.css
- theme-loch.css → theme-fhirworx.css (Gitea)
- Altair theme registered as "fhirworx" via new alt.theme.register
  decorator API, eliminating the altair 5.5 deprecation warning
- Traefik inject-loch middleware → inject-fhirworx
- Gitea DEFAULT_THEME=fhirworx
- Delete empty theme-throwback.css stub

Rewrote infra/nginx/index.html and infra/gitea/custom/templates/home.tmpl
with editorial masthead (rule-line + Playfair headline + mono identifier
+ serif tagline + rule-line-thin), SERVICES eyebrow + Infrastructure h2,
tiles with .tile-eyebrow / .tile-title / .tile-desc / .tile-port
structure. Stripped letter-spacing: 2-4px tricks and text-transform:
uppercase from non-metadata elements across all per-service sheets.

Propagated import renames: src/conf/connect.py, src/aco/dag.py,
notebooks/acodb_explorer.py, tests/conf/test_connect.py,
tests/test_notebook_layout.py, docs/src/css/custom.css, README.md,
README.md.j2, compose.yml, infra/traefik/dynamic/services.yml.
2026-04-10 14:37:20 -04:00

613 lines
19 KiB
Python

"""DAG visualization derived from Pipeline introspection.
Builds a dependency graph from ``Expr.inputs`` (parameter-derived ground
truth) and renders it as interactive HTML, static DOT, or Mermaid.
No modifications to express/ or pipe/ required — pure read-only
introspection of existing Expr objects.
Usage::
from aco.dag import build_graph, to_html, to_dot, to_mermaid
from aco.pipe import readmissions
g = build_graph(readmissions.pipeline)
html = to_html(g)
CLI::
uv run python -m aco.dag
uv run python -m aco.dag -p readmissions --html assets/dag.html
uv run python -m aco.dag --dot dag.dot
uv run python -m aco.dag -p pharmacy --mermaid
"""
from __future__ import annotations
import html as html_mod
import json
import sys as _sys
from dataclasses import dataclass, field
from pathlib import Path as _Path
from textwrap import dedent
_assets_dir = str(_Path(__file__).resolve().parents[2] / "assets")
if _assets_dir not in _sys.path:
_sys.path.insert(0, _assets_dir)
from fhirworx import EXTERNAL_COLOR as _EXTERNAL_COLOR # noqa: E402, I001
from fhirworx import SCHEMA_COLORS as _SCHEMA_COLORS # noqa: E402, F401
from fhirworx import schema_color as _color # noqa: E402
from aco.pipe.base import Pipeline # noqa: E402
# ---------------------------------------------------------------------------
# Data model
# ---------------------------------------------------------------------------
@dataclass
class Node:
name: str
schema: str
is_external: bool
description: str = ""
@dataclass
class Edge:
source: str
target: str
@dataclass
class Graph:
nodes: list[Node] = field(default_factory=list)
edges: list[Edge] = field(default_factory=list)
def schemas(self) -> set[str]:
return {n.schema for n in self.nodes}
def filter(self, schemas: set[str] | None = None) -> Graph:
if schemas is None:
return self
keep = {n.name for n in self.nodes if n.schema in schemas}
nodes = [n for n in self.nodes if n.name in keep]
edges = [e for e in self.edges if e.source in keep and e.target in keep]
return Graph(nodes=nodes, edges=edges)
# ---------------------------------------------------------------------------
# Builder
# ---------------------------------------------------------------------------
def build_graph(*pipelines: Pipeline) -> Graph:
"""Derive a dependency graph from one or more Pipelines."""
nodes: dict[str, Node] = {}
edges: list[Edge] = []
produced: set[str] = set()
for pipeline in pipelines:
for expr in pipeline.exprs:
produced.add(expr.name)
schema = expr.name.split(".")[0]
nodes[expr.name] = Node(
name=expr.name,
schema=schema,
is_external=False,
description=expr.doc,
)
for pipeline in pipelines:
for expr in pipeline.exprs:
for inp in expr.inputs:
edges.append(Edge(source=inp, target=expr.name))
if inp not in nodes:
schema = inp.split(".")[0] if "." in inp else inp
nodes[inp] = Node(
name=inp,
schema=schema,
is_external=True,
)
return Graph(nodes=list(nodes.values()), edges=edges)
# ---------------------------------------------------------------------------
# DOT renderer
# ---------------------------------------------------------------------------
def to_dot(graph: Graph, *, title: str = "ACO Pipeline DAG") -> str:
"""Render graph as DOT language string."""
lines = [
f'digraph "{title}" {{',
" rankdir=LR;",
' node [fontname="Helvetica" fontsize=10];',
' edge [color="#555555"];',
"",
]
by_schema: dict[str, list[Node]] = {}
for node in graph.nodes:
by_schema.setdefault(node.schema, []).append(node)
for schema in sorted(by_schema):
color = _color(schema)
lines.append(f" subgraph cluster_{schema} {{")
lines.append(f' label="{schema}";')
lines.append(f' color="{color}";')
lines.append(" style=rounded;")
for node in by_schema[schema]:
short = node.name.split(".", 1)[1] if "." in node.name else node.name
nid = node.name.replace(".", "__")
if node.is_external:
lines.append(
f' {nid} [label="{short}" '
f'style="dashed,filled" fillcolor="#2a2a2a" '
f'fontcolor="#aaaaaa" shape=rectangle];'
)
else:
lines.append(
f' {nid} [label="{short}" '
f'style="filled,rounded" fillcolor="{color}20" '
f'fontcolor="{color}" shape=rectangle];'
)
lines.append(" }")
lines.append("")
for edge in graph.edges:
src = edge.source.replace(".", "__")
tgt = edge.target.replace(".", "__")
lines.append(f" {src} -> {tgt};")
lines.append("}")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Mermaid renderer
# ---------------------------------------------------------------------------
def to_mermaid(graph: Graph) -> str:
"""Render graph as Mermaid flowchart syntax."""
lines = ["graph LR"]
for node in graph.nodes:
nid = node.name.replace(".", "_").replace("-", "_")
short = node.name.split(".", 1)[1] if "." in node.name else node.name
if node.is_external:
lines.append(f" {nid}[/{short}/]")
else:
lines.append(f" {nid}[{short}]")
for edge in graph.edges:
src = edge.source.replace(".", "_").replace("-", "_")
tgt = edge.target.replace(".", "_").replace("-", "_")
lines.append(f" {src} --> {tgt}")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Interactive HTML renderer (Cytoscape.js + Dagre from CDN)
# ---------------------------------------------------------------------------
def to_html(graph: Graph, *, title: str = "ACO Pipeline DAG") -> str:
"""Render graph as self-contained interactive HTML."""
cy_nodes = []
cy_edges = []
# Parent (compound) nodes for each schema
for schema in sorted(graph.schemas()):
cy_nodes.append(
{
"data": {
"id": f"grp_{schema}",
"label": schema,
"is_group": True,
},
"classes": "group",
}
)
for node in graph.nodes:
classes = "external" if node.is_external else "expr"
cy_nodes.append(
{
"data": {
"id": node.name,
"label": (
node.name.split(".", 1)[1] if "." in node.name else node.name
),
"parent": f"grp_{node.schema}",
"description": node.description,
"schema": node.schema,
"is_external": node.is_external,
},
"classes": classes,
}
)
for edge in graph.edges:
cy_edges.append(
{
"data": {
"source": edge.source,
"target": edge.target,
}
}
)
elements_json = json.dumps({"nodes": cy_nodes, "edges": cy_edges}, indent=2)
schema_colors_json = json.dumps({s: _color(s) for s in graph.schemas()})
esc_title = html_mod.escape(title)
return dedent(f"""\
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title>{esc_title}</title>
<script src="https://unpkg.com/cytoscape@3/dist/cytoscape.min.js"></script>
<script src="https://unpkg.com/dagre@0.8/dist/dagre.min.js"></script>
<script src="https://unpkg.com/cytoscape-dagre@2/cytoscape-dagre.js"></script>
<style>
* {{ margin: 0; padding: 0; box-sizing: border-box; }}
body {{ background: #1a1a2e; color: #e0e0e0; font-family: Helvetica, Arial, sans-serif; }}
#cy {{ width: 100vw; height: 100vh; }}
#controls {{
position: fixed; top: 12px; left: 12px; z-index: 10;
background: #16213e; border: 1px solid #333; border-radius: 8px;
padding: 12px; max-width: 260px; font-size: 13px;
}}
#controls h3 {{ margin-bottom: 8px; font-size: 14px; color: #fff; }}
#search {{
width: 100%; padding: 6px 8px; border-radius: 4px;
border: 1px solid #444; background: #0f3460; color: #e0e0e0;
margin-bottom: 8px; font-size: 13px;
}}
#legend {{ display: flex; flex-wrap: wrap; gap: 4px; }}
.legend-item {{
display: inline-flex; align-items: center; gap: 4px;
cursor: pointer; padding: 2px 6px; border-radius: 4px;
font-size: 11px; opacity: 1; transition: opacity 0.2s;
}}
.legend-item.hidden {{ opacity: 0.3; }}
.legend-dot {{
width: 10px; height: 10px; border-radius: 50%;
display: inline-block; flex-shrink: 0;
}}
#tooltip {{
position: fixed; display: none; background: #16213e;
border: 1px solid #444; border-radius: 6px; padding: 10px;
max-width: 340px; font-size: 12px; z-index: 20;
pointer-events: none; line-height: 1.4;
}}
#tooltip .tt-name {{ font-weight: bold; color: #fff; margin-bottom: 4px; }}
#tooltip .tt-desc {{ color: #bbb; }}
#stats {{
position: fixed; bottom: 12px; left: 12px; z-index: 10;
font-size: 11px; color: #666;
}}
</style>
</head>
<body>
<div id="controls">
<h3>{esc_title}</h3>
<input id="search" type="text" placeholder="Search nodes...">
<div id="legend"></div>
</div>
<div id="tooltip">
<div class="tt-name"></div>
<div class="tt-desc"></div>
</div>
<div id="stats"></div>
<div id="cy"></div>
<script>
const elements = {elements_json};
const schemaColors = {schema_colors_json};
const externalColor = "{_EXTERNAL_COLOR}";
// Build legend
const legend = document.getElementById("legend");
const hiddenSchemas = new Set();
Object.keys(schemaColors).sort().forEach(s => {{
const item = document.createElement("span");
item.className = "legend-item";
item.innerHTML = '<span class="legend-dot" style="background:' +
schemaColors[s] + '"></span>' + s;
item.onclick = () => {{
if (hiddenSchemas.has(s)) {{ hiddenSchemas.delete(s); item.classList.remove("hidden"); }}
else {{ hiddenSchemas.add(s); item.classList.add("hidden"); }}
applyVisibility();
}};
legend.appendChild(item);
}});
// Init Cytoscape
const cy = cytoscape({{
container: document.getElementById("cy"),
elements: [...elements.nodes, ...elements.edges],
style: [
{{
selector: "node.group",
style: {{
"background-color": "transparent",
"background-opacity": 0.05,
"border-width": 1,
"border-color": "#444",
"label": "data(label)",
"text-valign": "top",
"text-halign": "center",
"font-size": "14px",
"color": "#666",
"padding": "16px",
"shape": "round-rectangle",
}}
}},
{{
selector: "node.expr",
style: {{
"background-color": function(ele) {{
return schemaColors[ele.data("schema")] || "#888";
}},
"background-opacity": 0.15,
"border-width": 2,
"border-color": function(ele) {{
return schemaColors[ele.data("schema")] || "#888";
}},
"label": "data(label)",
"text-valign": "center",
"text-halign": "center",
"font-size": "9px",
"color": function(ele) {{
return schemaColors[ele.data("schema")] || "#888";
}},
"width": "label",
"height": 28,
"padding": "8px",
"shape": "round-rectangle",
}}
}},
{{
selector: "node.external",
style: {{
"background-color": "#2a2a2a",
"border-width": 1,
"border-style": "dashed",
"border-color": "#666",
"label": "data(label)",
"text-valign": "center",
"text-halign": "center",
"font-size": "8px",
"color": "#888",
"width": "label",
"height": 24,
"padding": "6px",
"shape": "round-rectangle",
}}
}},
{{
selector: "edge",
style: {{
"width": 1.5,
"line-color": "#444",
"target-arrow-color": "#444",
"target-arrow-shape": "triangle",
"curve-style": "bezier",
"arrow-scale": 0.8,
}}
}},
{{
selector: ".highlighted",
style: {{
"border-color": "#fff",
"border-width": 3,
"color": "#fff",
}}
}},
{{
selector: "edge.highlighted",
style: {{
"line-color": "#e0e0e0",
"target-arrow-color": "#e0e0e0",
"width": 2.5,
}}
}},
{{
selector: ".dimmed",
style: {{
"opacity": 0.15,
}}
}},
],
layout: {{
name: "dagre",
rankDir: "LR",
nodeSep: 30,
rankSep: 80,
edgeSep: 10,
animate: false,
}},
minZoom: 0.1,
maxZoom: 3,
}});
// Stats
const nNodes = elements.nodes.filter(n => !n.data.is_group).length;
const nEdges = elements.edges.length;
document.getElementById("stats").textContent =
nNodes + " nodes, " + nEdges + " edges";
// Tooltip
const tooltip = document.getElementById("tooltip");
const ttName = tooltip.querySelector(".tt-name");
const ttDesc = tooltip.querySelector(".tt-desc");
cy.on("mouseover", "node.expr, node.external", function(e) {{
const d = e.target.data();
ttName.textContent = d.id;
ttDesc.textContent = d.description || (d.is_external ? "External source table" : "");
tooltip.style.display = "block";
}});
cy.on("mouseout", "node.expr, node.external", function() {{
tooltip.style.display = "none";
}});
cy.on("mousemove", function(e) {{
tooltip.style.left = (e.originalEvent.clientX + 14) + "px";
tooltip.style.top = (e.originalEvent.clientY + 14) + "px";
}});
// Click to highlight ancestors + descendants
cy.on("tap", "node.expr, node.external", function(e) {{
const node = e.target;
const predecessors = node.predecessors();
const successors = node.successors();
const connected = predecessors.union(successors).union(node);
cy.elements().addClass("dimmed");
connected.removeClass("dimmed").addClass("highlighted");
}});
cy.on("tap", function(e) {{
if (e.target === cy) {{
cy.elements().removeClass("dimmed highlighted");
}}
}});
// Search
const searchInput = document.getElementById("search");
searchInput.addEventListener("input", function() {{
const q = this.value.toLowerCase();
if (!q) {{
cy.elements().removeClass("dimmed highlighted");
return;
}}
const matches = cy.nodes().filter(n =>
n.data("id").toLowerCase().includes(q) ||
(n.data("label") || "").toLowerCase().includes(q)
);
if (matches.length > 0) {{
cy.elements().addClass("dimmed");
matches.forEach(m => {{
const connected = m.predecessors().union(m.successors()).union(m);
connected.removeClass("dimmed").addClass("highlighted");
}});
}}
}});
// Schema visibility toggle
function applyVisibility() {{
cy.batch(() => {{
cy.nodes().forEach(n => {{
const s = n.data("schema") || n.data("label");
if (hiddenSchemas.has(s)) n.style("display", "none");
else n.style("display", "element");
}});
}});
}}
cy.fit(undefined, 40);
</script>
</body>
</html>
""")
# ---------------------------------------------------------------------------
# CLI
# ---------------------------------------------------------------------------
_ALL_PIPE_MODULES = [
"core",
"input_layer",
"hcc_suspecting",
"data_quality",
"quality_measures",
"pharmacy",
"readmissions",
"ahrq_measures",
"provider_attribution",
"main",
"claims_preprocessing",
"cclf",
"opps",
]
def _load_pipelines(names: list[str] | None = None) -> list[Pipeline]:
import importlib
targets = names or _ALL_PIPE_MODULES
pipelines = []
for name in targets:
mod = importlib.import_module(f"aco.pipe.{name}")
pipelines.append(mod.pipeline)
return pipelines
def main() -> None:
import argparse
import sys
parser = argparse.ArgumentParser(
description="DAG visualization from Pipeline introspection"
)
parser.add_argument(
"-p",
"--pipeline",
action="append",
help="Pipeline module name(s) (default: all)",
)
parser.add_argument("--html", metavar="FILE", help="Write interactive HTML")
parser.add_argument("--dot", metavar="FILE", help="Write DOT language")
parser.add_argument(
"--mermaid", action="store_true", help="Print Mermaid to stdout"
)
args = parser.parse_args()
pipelines = _load_pipelines(args.pipeline)
graph = build_graph(*pipelines)
if not (args.html or args.dot or args.mermaid):
# Summary mode
n_expr = sum(1 for n in graph.nodes if not n.is_external)
n_ext = sum(1 for n in graph.nodes if n.is_external)
print(f"Nodes: {len(graph.nodes)} ({n_expr} exprs, {n_ext} external)")
print(f"Edges: {len(graph.edges)}")
print(f"Schemas: {', '.join(sorted(graph.schemas()))}")
return
if args.html:
title = "ACO Pipeline DAG"
if args.pipeline:
title = f"{', '.join(args.pipeline)} DAG"
content = to_html(graph, title=title)
if args.html == "-":
sys.stdout.write(content)
else:
with open(args.html, "w") as f:
f.write(content)
print(f"Wrote {args.html}")
if args.dot:
content = to_dot(graph)
if args.dot == "-":
sys.stdout.write(content)
else:
with open(args.dot, "w") as f:
f.write(content)
print(f"Wrote {args.dot}")
if args.mermaid:
print(to_mermaid(graph))
if __name__ == "__main__":
main()