ec44e937f5
Wire cli/docs.py build/serve/generate commands to run extraction scripts and Docusaurus. Add extract_cli.py (27 CLI docs from typer help), extract_openapi.py (8 REST API docs from FastAPI schema), extract_citations.py (14 pipeline pages with bib references), extract_dag.py (pipeline + table lineage Mermaid diagrams). Closes #26, #27, #28, #29, #30.
127 lines
3.7 KiB
Python
127 lines
3.7 KiB
Python
"""Extract Zotero-sourced regulatory citations for pipeline docs.
|
|
|
|
For each pipeline, finds bib items tagged with the pipeline's module
|
|
namespace and generates a citations page with linked references.
|
|
|
|
Usage::
|
|
|
|
uv run python docs/scripts/extract_citations.py
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path("src").resolve()))
|
|
|
|
OUT_DIR = Path("docs/docs/pipelines")
|
|
|
|
|
|
def main() -> None:
|
|
from conf import path as _conf_path
|
|
|
|
bib_db = _conf_path("db.bib")
|
|
if not bib_db.exists():
|
|
print("bib.sqlite not found, skipping citations")
|
|
return
|
|
|
|
from aco.pipe import registry
|
|
from bib.store import Store
|
|
|
|
store = Store(str(bib_db))
|
|
|
|
if OUT_DIR.exists():
|
|
import shutil
|
|
|
|
shutil.rmtree(OUT_DIR)
|
|
OUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
cat_json = OUT_DIR / "_category_.json"
|
|
cat_json.write_text('{"label": "Pipelines", "position": 12}\n')
|
|
|
|
count = 0
|
|
|
|
# Overview page
|
|
overview = [
|
|
"---\ntitle: Pipelines\nsidebar_position: 1\n---\n",
|
|
"# Pipelines\n",
|
|
"| Pipeline | Steps | Description |",
|
|
"|----------|-------|-------------|",
|
|
]
|
|
for name, pipeline in sorted(registry.items()):
|
|
overview.append(f"| [{name}](./{name}) | {len(pipeline)} | |")
|
|
overview.append("")
|
|
(OUT_DIR / "index.md").write_text("\n".join(overview))
|
|
count += 1
|
|
|
|
# Per-pipeline pages with citations
|
|
all_items = store.list_items()
|
|
|
|
for pos, (name, pipeline) in enumerate(sorted(registry.items()), start=2):
|
|
lines = [
|
|
f"---\ntitle: {name}\nsidebar_position: {pos}\n---\n",
|
|
f"# {name}\n",
|
|
f"**Steps:** {len(pipeline)}\n",
|
|
]
|
|
|
|
# Step list
|
|
lines.append("## Steps\n")
|
|
for i, expr in enumerate(pipeline.exprs, 1):
|
|
doc = expr.fn.__doc__ or ""
|
|
doc_line = doc.strip().split("\n")[0] if doc else ""
|
|
lines.append(f"{i}. **{expr.name}**{f' — {doc_line}' if doc_line else ''}")
|
|
lines.append("")
|
|
|
|
# Find citations: items tagged with module:{pipeline_name} or
|
|
# related tags
|
|
module_tag = f"module:{name}"
|
|
cited_items = [
|
|
item for item in all_items if module_tag in item.tags
|
|
]
|
|
|
|
# Also check for aco-related tags
|
|
if not cited_items:
|
|
aco_tag = "module:aco"
|
|
cited_items = [
|
|
item for item in all_items if aco_tag in item.tags
|
|
][:5] # Limit to top 5 general references
|
|
|
|
if cited_items:
|
|
lines.append("## Regulatory References\n")
|
|
for item in cited_items:
|
|
title = item.title
|
|
url = item.url
|
|
date = item.date_published or ""
|
|
institution = item.institution or ""
|
|
|
|
if url:
|
|
lines.append(f"- [{title}]({url})")
|
|
else:
|
|
lines.append(f"- {title}")
|
|
|
|
meta_parts = []
|
|
if institution:
|
|
meta_parts.append(institution)
|
|
if date:
|
|
meta_parts.append(date)
|
|
if meta_parts:
|
|
lines.append(f" *{' | '.join(meta_parts)}*")
|
|
|
|
abstract = item.abstract or ""
|
|
if abstract:
|
|
short = abstract[:200] + "..." if len(abstract) > 200 else abstract
|
|
lines.append(f" {short}")
|
|
lines.append("")
|
|
|
|
out_path = OUT_DIR / f"{name}.md"
|
|
out_path.write_text("\n".join(lines))
|
|
count += 1
|
|
print(f" {name} -> {out_path}")
|
|
|
|
print(f"\nDone: {count} pipeline docs written to {OUT_DIR}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|