Files
stack/tests/aco/test_lake_governance.py
kert 97852e818c
Some checks failed
CI / skinny-install (api) (push) Successful in 24s
CI / lint-test (push) Successful in 1m18s
CI / skinny-install (bib) (push) Successful in 30s
CI / skinny-install (bls) (push) Successful in 28s
CI / skinny-install (aco) (push) Successful in 50s
CI / skinny-install (bcda) (push) Successful in 33s
CI / skinny-install (ccw) (push) Successful in 32s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (cli) (push) Successful in 36s
CI / skinny-install (conf) (push) Successful in 33s
CI / skinny-install (opps) (push) Successful in 30s
CI / skinny-install (perf) (push) Successful in 31s
CI / skinny-install (pfs) (push) Successful in 34s
CI / skinny-install (rex) (push) Has been cancelled
CI / skinny-install (aco) (pull_request) Successful in 51s
CI / lint-test (pull_request) Successful in 1m20s
CI / skinny-install (api) (pull_request) Successful in 29s
CI / skinny-install (bcda) (pull_request) Successful in 30s
CI / skinny-install (bib) (pull_request) Successful in 32s
CI / skinny-install (bls) (pull_request) Successful in 26s
CI / skinny-install (ccw) (pull_request) Successful in 27s
CI / skinny-install (cli) (pull_request) Successful in 29s
CI / skinny-install (cms) (pull_request) Successful in 34s
CI / skinny-install (conf) (pull_request) Successful in 22s
CI / skinny-install (opps) (pull_request) Successful in 33s
CI / skinny-install (perf) (pull_request) Successful in 31s
CI / skinny-install (pfs) (pull_request) Successful in 34s
CI / skinny-install (rex) (pull_request) Successful in 32s
Infra CI / notebooks (push) Successful in 8s
Infra CI / zotero (push) Successful in 6s
Infra CI / docs (push) Failing after 12s
Infra CI / api (push) Successful in 10s
Infra CI / mc (push) Successful in 9s
Infra CI / notebooks (pull_request) Successful in 7s
Infra CI / zotero (pull_request) Successful in 6s
Infra CI / docs (pull_request) Failing after 5s
Infra CI / api (pull_request) Successful in 6s
Infra CI / mc (pull_request) Successful in 6s
feat: deep Databricks SDK integration — governance, jobs, quality, secrets
Expand Databricks SDK usage from 6 to 18 WorkspaceClient services.

Tier 1 — Governance & Compliance:
- governance.py: declarative GovernancePolicy with apply + audit drift
- UnityClient: grants CRUD (ws.grants), secret management (ws.secrets),
  table constraints PK/FK (ws.table_constraints)
- sync_secrets.py: push env vars to Databricks scopes per stack.toml

Tier 2 — Job Orchestration:
- jobs.py: JobManager translates Pipeline → Databricks Job with task
  dependencies, run_now, get_run_status, list/delete
- UnityClient: warehouse lookup by name (ws.warehouses), start/stop

Tier 3 — Data Quality & Monitoring:
- quality.py: setup_monitors creates Lakehouse Monitoring profiles,
  list_monitors, run_refresh (ws.quality_monitors)
- UnityClient: system schema access, table lineage queries, audit log
  queries (ws.system_schemas, ws.statement_execution)

Config: stack.toml gains [databricks.warehouse], [databricks.secrets],
[databricks.governance], [databricks.quality] sections.

23 new tests, all passing.
2026-03-26 19:57:51 -04:00

168 lines
5.0 KiB
Python

"""Tests for aco.lake.governance — declarative grant management."""
from __future__ import annotations
from unittest.mock import MagicMock
from aco.lake.governance import (
GovernancePolicy,
GrantRule,
_resolve_schemas,
apply_governance,
audit_governance,
)
class TestResolveSchemas:
def test_wildcard_expands(self):
rule = GrantRule(group="g", schemas=["*"], privileges=["SELECT"])
assert _resolve_schemas(rule, ["core", "claims"]) == ["core", "claims"]
def test_explicit_schemas(self):
rule = GrantRule(group="g", schemas=["core"], privileges=["SELECT"])
assert _resolve_schemas(rule, ["core", "claims"]) == ["core"]
def test_missing_schema_filtered(self):
rule = GrantRule(group="g", schemas=["missing"], privileges=["SELECT"])
assert _resolve_schemas(rule, ["core"]) == []
class TestGovernancePolicyModel:
def test_basic_construction(self):
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(
group="analysts",
schemas=["readmissions"],
privileges=["SELECT"],
),
],
phi_schemas=["core"],
)
assert policy.catalog == "aco"
assert len(policy.rules) == 1
assert policy.phi_schemas == ["core"]
def test_empty_rules(self):
policy = GovernancePolicy(catalog="aco")
assert policy.rules == []
class TestApplyGovernance:
def test_dry_run_reports_actions(self):
client = MagicMock()
# Mock schemas list
schema_mock = MagicMock()
schema_mock.name = "readmissions"
client._ws.schemas.list.return_value = [schema_mock]
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(
group="analysts",
schemas=["readmissions"],
privileges=["SELECT"],
),
],
)
actions = apply_governance(client, policy, dry_run=True)
assert len(actions) == 1
assert "[dry-run]" in actions[0]
assert "GRANT SELECT" in actions[0]
assert "analysts" in actions[0]
# Should NOT call ws.grants.update in dry_run
client._ws.grants.update.assert_not_called()
def test_apply_calls_sdk(self):
client = MagicMock()
schema_mock = MagicMock()
schema_mock.name = "core"
client._ws.schemas.list.return_value = [schema_mock]
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(
group="pipeline_svc",
schemas=["core"],
privileges=["ALL_PRIVILEGES"],
),
],
)
actions = apply_governance(client, policy, dry_run=False)
assert len(actions) == 1
assert "GRANT" in actions[0]
client._ws.grants.update.assert_called_once()
def test_wildcard_expands_to_all(self):
client = MagicMock()
s1, s2 = MagicMock(), MagicMock()
s1.name, s2.name = "core", "claims"
client._ws.schemas.list.return_value = [s1, s2]
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(group="g", schemas=["*"], privileges=["SELECT"]),
],
)
actions = apply_governance(client, policy, dry_run=True)
assert len(actions) == 2
class TestAuditGovernance:
def test_detects_missing_grants(self):
client = MagicMock()
schema_mock = MagicMock()
schema_mock.name = "core"
client._ws.schemas.list.return_value = [schema_mock]
# No grants exist
resp = MagicMock()
resp.privilege_assignments = []
client._ws.grants.get.return_value = resp
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(group="analysts", schemas=["core"], privileges=["SELECT"]),
],
)
drifts = audit_governance(client, policy)
assert len(drifts) == 1
assert drifts[0].principal == "analysts"
assert drifts[0].action == "grant"
def test_no_drift_when_matching(self):
client = MagicMock()
schema_mock = MagicMock()
schema_mock.name = "core"
client._ws.schemas.list.return_value = [schema_mock]
# Grants match exactly
priv = MagicMock()
priv.privilege = MagicMock()
priv.privilege.value = "SELECT"
assignment = MagicMock()
assignment.principal = "analysts"
assignment.privileges = [priv]
resp = MagicMock()
resp.privilege_assignments = [assignment]
client._ws.grants.get.return_value = resp
policy = GovernancePolicy(
catalog="aco",
rules=[
GrantRule(group="analysts", schemas=["core"], privileges=["SELECT"]),
],
)
drifts = audit_governance(client, policy)
assert len(drifts) == 0