feat: sem.hooks — smart pre-commit with tracked hook architecture

Move pre-commit logic from shell into src/sem/hooks.py as a proper
Python module.  Shell hook (dev/hooks/pre-commit) is now a 3-line
wrapper.  git core.hooksPath points at dev/hooks/ so hooks are
version-controlled, not buried in .git/.

Smart test selection: only runs tests for changed src/ modules
(src/aco/ → tests/aco/), always includes structural invariants
(test_ast_coverage.py) on any src/ change, falls back to full
12k+ suite when infrastructure (conftest, pyproject.toml) changes.
Notebooks checked only when staged.

Force full suite: GIT_PRE_COMMIT_FULL=1 git commit

Also: register pytest 'stub' marker, move post-commit to tracked dir.

dev/hooks/post-commit

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Rebuild docs image and restart container in the background.
+# Runs after every commit so docs stay current with docstring
+# and bib.sqlite changes.
+
+(
+  cd "$(git rev-parse --show-toplevel)" || exit
+  docker build -q -t fhirworx/docs:latest -f docs/Dockerfile . \
+    && docker compose up -d docs
+) &>/dev/null &

dev/hooks/pre-commit

@@ -1,51 +1,4 @@
 #!/usr/bin/env bash
-set -e
+# Thin wrapper — all logic lives in src/sem/hooks.py (tracked).
-
+# Install: git config core.hooksPath dev/hooks
-# Auto-recover broken venv before running any checks
+exec uv run python -m sem.hooks
-if ! uv run python -c 'import sys' 2>/dev/null; then
-  echo "==> pre-commit: venv broken, running uv sync --dev"
-  uv sync --dev
-fi
-
-# Auto-regenerate derived config if stack.toml or generator changed
-CONF_CHANGED=$(git diff --cached --name-only -- stack.toml 'dev/scripts/gen_config.py' 'dev/scripts/backends/*.py')
-if [ -n "$CONF_CHANGED" ]; then
-  echo "==> pre-commit: regenerating config from stack.toml"
-  uv run python dev/scripts/gen_config.py
-  git add .woodpecker/*.yml .github/workflows/*.yml coredns/hosts coredns/Corefile 2>/dev/null || true
-fi
-
-echo "==> pre-commit: ruff check (staged)"
-# Only lint files that are staged for commit (avoid dirty-tree noise).
-STAGED=$(git diff --cached --name-only --diff-filter=d -- 'src/*.py' 'tests/*.py')
-if [ -n "$STAGED" ]; then
-  echo "$STAGED" | xargs uv run ruff check --quiet
-fi
-
-echo "==> pre-commit: ruff format --check (staged)"
-if [ -n "$STAGED" ]; then
-  echo "$STAGED" | xargs uv run ruff format --check --quiet
-fi
-
-echo "==> pre-commit: pytest"
-uv run python -m pytest tests/ --no-cov --tb=short -q
-
-echo "==> pre-commit: marimo check"
-uvx marimo check notebooks/*.py
-
-echo "==> pre-commit: notebook run check"
-# Only check notebooks that run standalone (no external services).
-# Notebooks needing containers (api, nessie, polaris) or optional
-# deps (vega_datasets, pyzotero) are excluded.
-for nb in \
-  notebooks/acodb_explorer.py \
-  notebooks/bib_explorer.py \
-  notebooks/cms_quality_measures.py \
-  notebooks/pfs_calcs.py \
-  notebooks/sql_generator.py \
-; do
-  echo "    running $nb"
-  uv run python "$nb" 2>&1 || { echo "FAILED: $nb"; exit 1; }
-done
-
-echo "==> pre-commit: all checks passed"

pyproject.toml

@@ -154,6 +154,9 @@ ignore = ["E501", "E741"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+markers = [
+    "stub: marks tests that report stub vs implemented status (deselect with '-m \"not stub\"')",
+]
 
 [tool.uv.build-backend]
 module-name = ["aco", "api", "bcda", "bib", "bls", "ccw", "cli", "cms", "conf", "opps", "perf", "pfs", "rex", "sem"]

src/sem/hooks.py

@@ -0,0 +1,292 @@
+"""Pre-commit hook logic — smart test selection via semantic analysis.
+
+This module implements the pre-commit orchestration for the stack repo.
+Instead of running all 12k+ tests on every commit, it analyses staged
+files to determine:
+
+1. Which ``src/`` modules changed → run matching ``tests/<module>/``
+2. Whether infrastructure changed (conftest, pyproject) → full suite
+3. Whether notebooks changed → marimo check + execution
+4. AST parse validation on every staged ``.py`` file
+
+The shell hook (``dev/hooks/pre-commit``) is a thin wrapper that calls
+``uv run python -m sem.hooks``.
+
+Environment variables
+---------------------
+GIT_PRE_COMMIT_FULL=1
+    Force full test suite regardless of what changed.
+"""
+
+from __future__ import annotations
+
+import ast
+import subprocess
+import sys
+from pathlib import Path
+
+
+def _staged_files() -> list[str]:
+    """Return list of staged file paths (excluding deletes)."""
+    result = subprocess.run(
+        ["git", "diff", "--cached", "--name-only", "--diff-filter=d"],
+        capture_output=True,
+        text=True,
+    )
+    return [f for f in result.stdout.strip().splitlines() if f]
+
+
+def _classify(staged: list[str]) -> dict[str, list[str]]:
+    """Partition staged files into categories."""
+    cats: dict[str, list[str]] = {
+        "src": [],
+        "tests": [],
+        "notebooks": [],
+        "infra": [],
+        "config": [],
+    }
+    infra_names = {"conftest.py", "pyproject.toml"}
+
+    for f in staged:
+        if f.startswith("src/") and f.endswith(".py"):
+            cats["src"].append(f)
+        elif f.startswith("tests/") and f.endswith(".py"):
+            cats["tests"].append(f)
+            if Path(f).name in infra_names:
+                cats["infra"].append(f)
+        elif f.startswith("notebooks/") and f.endswith(".py"):
+            cats["notebooks"].append(f)
+        elif Path(f).name in infra_names:
+            cats["infra"].append(f)
+        elif f in ("stack.toml", "dev/scripts/gen_config.py") or f.startswith(
+            "dev/scripts/backends/"
+        ):
+            cats["config"].append(f)
+    return cats
+
+
+def _changed_modules(src_files: list[str]) -> list[str]:
+    """Extract unique top-level module names from src/ paths.
+
+    ``src/aco/express/foo.py`` → ``aco``
+    ``src/sem/parse.py``       → ``sem``
+    """
+    mods: set[str] = set()
+    for f in src_files:
+        parts = f.split("/")
+        if len(parts) >= 2 and parts[0] == "src":
+            mods.add(parts[1])
+    return sorted(mods)
+
+
+def _changed_test_dirs(test_files: list[str]) -> list[str]:
+    """Extract unique test subdirectory names from test paths.
+
+    ``tests/aco/test_foo.py`` → ``tests/aco``
+    """
+    dirs: set[str] = set()
+    for f in test_files:
+        parts = f.split("/")
+        if len(parts) >= 3 and parts[0] == "tests":
+            dirs.add(f"tests/{parts[1]}")
+    return sorted(dirs)
+
+
+def _top_level_tests(test_files: list[str]) -> list[str]:
+    """Return test files directly under tests/ (not in subdirectories)."""
+    return [f for f in test_files if f.count("/") == 1]
+
+
+def check_syntax(src_files: list[str]) -> list[str]:
+    """Parse staged source files and return syntax errors."""
+    errors: list[str] = []
+    for f in src_files:
+        p = Path(f)
+        if not p.exists():
+            continue
+        try:
+            ast.parse(p.read_text(encoding="utf-8"), filename=f)
+        except SyntaxError as e:
+            errors.append(f"  {f}:{e.lineno}: {e.msg}")
+    return errors
+
+
+def compute_test_targets(
+    cats: dict[str, list[str]],
+    *,
+    force_full: bool = False,
+) -> tuple[list[str], str]:
+    """Determine which test paths to run.
+
+    Returns (test_paths, reason) where reason describes why this
+    scope was chosen.
+    """
+    if force_full:
+        return ["tests/"], "GIT_PRE_COMMIT_FULL=1"
+
+    if cats["infra"]:
+        return ["tests/"], "infrastructure files changed"
+
+    targets: list[str] = []
+    reasons: list[str] = []
+
+    # Map changed src/ modules to test directories
+    if cats["src"]:
+        for mod in _changed_modules(cats["src"]):
+            test_dir = Path("tests") / mod
+            if test_dir.is_dir():
+                targets.append(f"tests/{mod}/")
+                reasons.append(mod)
+
+        # Structural invariants on any src/ change
+        ast_cov = Path("tests/test_ast_coverage.py")
+        if ast_cov.exists():
+            targets.append(str(ast_cov))
+
+    # Include test dirs for directly-changed test files
+    if cats["tests"]:
+        for td in _changed_test_dirs(cats["tests"]):
+            if td + "/" not in targets and Path(td).is_dir():
+                targets.append(td + "/")
+                reasons.append(td.split("/")[-1])
+
+        for tf in _top_level_tests(cats["tests"]):
+            if tf not in targets:
+                targets.append(tf)
+
+    if not targets:
+        return [], "no testable changes"
+
+    # Deduplicate
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for t in targets:
+        if t not in seen:
+            seen.add(t)
+            deduped.append(t)
+
+    return deduped, ", ".join(reasons) if reasons else "changed files"
+
+
+def run_step(label: str, cmd: list[str], **kwargs) -> int:
+    """Run a subprocess step, printing the label."""
+    print(f"==> pre-commit: {label}")
+    result = subprocess.run(cmd, **kwargs)
+    return result.returncode
+
+
+def main() -> int:
+    """Entry point for pre-commit hook orchestration."""
+    import os
+
+    staged = _staged_files()
+    if not staged:
+        print("==> pre-commit: nothing staged — skipped")
+        return 0
+
+    cats = _classify(staged)
+    force_full = os.environ.get("GIT_PRE_COMMIT_FULL", "0") == "1"
+
+    # ── Venv health ──────────────────────────────────────────────────
+    venv_ok = subprocess.run(
+        ["uv", "run", "python", "-c", "import sys"],
+        capture_output=True,
+    ).returncode
+    if venv_ok != 0:
+        run_step("venv broken, running uv sync --dev", ["uv", "sync", "--dev"])
+
+    # ── Config regeneration ──────────────────────────────────────────
+    if cats["config"]:
+        rc = run_step(
+            "regenerating config from stack.toml",
+            ["uv", "run", "python", "dev/scripts/gen_config.py"],
+        )
+        if rc != 0:
+            return rc
+        subprocess.run(
+            [
+                "git",
+                "add",
+                ".woodpecker/*.yml",
+                ".github/workflows/*.yml",
+                ".gitea/workflows/*.yml",
+                "coredns/hosts",
+                "coredns/Corefile",
+            ],
+            capture_output=True,
+        )
+
+    # ── Ruff lint + format (staged Python only) ──────────────────────
+    all_py = cats["src"] + cats["tests"]
+    if all_py:
+        rc = run_step(
+            "ruff check (staged)",
+            ["uv", "run", "ruff", "check", "--quiet"] + all_py,
+        )
+        if rc != 0:
+            return rc
+
+        rc = run_step(
+            "ruff format --check (staged)",
+            ["uv", "run", "ruff", "format", "--check", "--quiet"] + all_py,
+        )
+        if rc != 0:
+            return rc
+
+    # ── AST parse check (staged src) ─────────────────────────────────
+    if cats["src"]:
+        errors = check_syntax(cats["src"])
+        if errors:
+            print("==> pre-commit: syntax errors in staged files")
+            print("\n".join(errors))
+            return 1
+        print(f"==> pre-commit: sem parse check ({len(cats['src'])} files OK)")
+
+    # ── Pytest: smart selection ──────────────────────────────────────
+    targets, reason = compute_test_targets(cats, force_full=force_full)
+    if targets:
+        rc = run_step(
+            f"pytest ({reason})",
+            ["uv", "run", "python", "-m", "pytest"]
+            + targets
+            + ["--no-cov", "--tb=short", "-q"],
+        )
+        if rc != 0:
+            return rc
+    else:
+        print(f"==> pre-commit: pytest ({reason} — skipped)")
+
+    # ── Notebooks (only when staged) ─────────────────────────────────
+    if cats["notebooks"]:
+        rc = run_step(
+            "marimo check (staged notebooks)",
+            ["uvx", "marimo", "check"] + cats["notebooks"],
+        )
+        if rc != 0:
+            return rc
+
+        # Only execute notebooks in the safe-to-run list
+        safe = {
+            "acodb_explorer.py",
+            "bib_explorer.py",
+            "cms_quality_measures.py",
+            "pfs_calcs.py",
+            "sql_generator.py",
+        }
+        for nb in cats["notebooks"]:
+            if Path(nb).name in safe:
+                rc = run_step(
+                    f"notebook run: {nb}",
+                    ["uv", "run", "python", nb],
+                )
+                if rc != 0:
+                    return rc
+    else:
+        print("==> pre-commit: notebooks (no changes — skipped)")
+
+    print("==> pre-commit: all checks passed")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/sem/test_hooks.py

@@ -0,0 +1,137 @@
+"""Tests for sem.hooks — pre-commit smart test selection."""
+
+from __future__ import annotations
+
+from sem.hooks import (
+    _changed_modules,
+    _changed_test_dirs,
+    _classify,
+    _top_level_tests,
+    check_syntax,
+    compute_test_targets,
+)
+
+
+class TestClassify:
+    def test_src_files(self):
+        cats = _classify(["src/aco/express/foo.py", "src/sem/parse.py"])
+        assert cats["src"] == ["src/aco/express/foo.py", "src/sem/parse.py"]
+        assert cats["tests"] == []
+
+    def test_test_files(self):
+        cats = _classify(["tests/aco/test_foo.py"])
+        assert cats["tests"] == ["tests/aco/test_foo.py"]
+
+    def test_notebooks(self):
+        cats = _classify(["notebooks/pfs_calcs.py"])
+        assert cats["notebooks"] == ["notebooks/pfs_calcs.py"]
+
+    def test_infra_conftest(self):
+        cats = _classify(["tests/conftest.py"])
+        assert "tests/conftest.py" in cats["infra"]
+
+    def test_infra_pyproject(self):
+        cats = _classify(["pyproject.toml"])
+        assert "pyproject.toml" in cats["infra"]
+
+    def test_config_stack_toml(self):
+        cats = _classify(["stack.toml"])
+        assert cats["config"] == ["stack.toml"]
+
+    def test_non_python_ignored(self):
+        cats = _classify(["README.md", "compose.yml", "src/aco/data.json"])
+        assert all(not v for v in cats.values())
+
+
+class TestChangedModules:
+    def test_extracts_top_level(self):
+        assert _changed_modules(["src/aco/express/foo.py"]) == ["aco"]
+        assert _changed_modules(["src/sem/parse.py"]) == ["sem"]
+
+    def test_deduplicates(self):
+        result = _changed_modules(
+            [
+                "src/aco/express/foo.py",
+                "src/aco/pipe/bar.py",
+            ]
+        )
+        assert result == ["aco"]
+
+    def test_multiple_modules(self):
+        result = _changed_modules(
+            [
+                "src/aco/foo.py",
+                "src/sem/bar.py",
+                "src/bib/baz.py",
+            ]
+        )
+        assert result == ["aco", "bib", "sem"]
+
+
+class TestChangedTestDirs:
+    def test_extracts_dirs(self):
+        result = _changed_test_dirs(["tests/aco/test_foo.py"])
+        assert result == ["tests/aco"]
+
+    def test_deduplicates(self):
+        result = _changed_test_dirs(
+            [
+                "tests/aco/test_a.py",
+                "tests/aco/test_b.py",
+            ]
+        )
+        assert result == ["tests/aco"]
+
+
+class TestTopLevelTests:
+    def test_finds_top_level(self):
+        result = _top_level_tests(
+            [
+                "tests/test_ast_coverage.py",
+                "tests/aco/test_foo.py",
+            ]
+        )
+        assert result == ["tests/test_ast_coverage.py"]
+
+
+class TestCheckSyntax:
+    def test_valid_file(self, tmp_path):
+        f = tmp_path / "good.py"
+        f.write_text("x = 1\n")
+        assert check_syntax([str(f)]) == []
+
+    def test_invalid_file(self, tmp_path):
+        f = tmp_path / "bad.py"
+        f.write_text("def f(\n")
+        errors = check_syntax([str(f)])
+        assert len(errors) == 1
+        assert "bad.py" in errors[0]
+
+    def test_missing_file_skipped(self):
+        assert check_syntax(["nonexistent_file_xyz.py"]) == []
+
+
+class TestComputeTargets:
+    def test_full_on_infra_change(self):
+        cats = _classify(["pyproject.toml", "src/sem/parse.py"])
+        targets, reason = compute_test_targets(cats)
+        assert targets == ["tests/"]
+        assert "infrastructure" in reason
+
+    def test_full_on_force(self):
+        cats = _classify(["src/sem/parse.py"])
+        targets, reason = compute_test_targets(cats, force_full=True)
+        assert targets == ["tests/"]
+
+    def test_targeted_on_src_change(self):
+        cats = _classify(["src/sem/parse.py"])
+        targets, reason = compute_test_targets(cats)
+        # Should include tests/sem/ and test_ast_coverage.py
+        assert any("tests/sem/" in t for t in targets)
+        assert any("test_ast_coverage" in t for t in targets)
+
+    def test_empty_on_no_python(self):
+        cats = _classify(["README.md"])
+        targets, reason = compute_test_targets(cats)
+        assert targets == []
+        assert "no testable" in reason