Some checks failed
CI / skinny-install (api) (push) Successful in 24s
CI / lint-test (push) Successful in 1m18s
CI / skinny-install (bib) (push) Successful in 30s
CI / skinny-install (bls) (push) Successful in 28s
CI / skinny-install (aco) (push) Successful in 50s
CI / skinny-install (bcda) (push) Successful in 33s
CI / skinny-install (ccw) (push) Successful in 32s
CI / skinny-install (cms) (push) Successful in 24s
CI / skinny-install (cli) (push) Successful in 36s
CI / skinny-install (conf) (push) Successful in 33s
CI / skinny-install (opps) (push) Successful in 30s
CI / skinny-install (perf) (push) Successful in 31s
CI / skinny-install (pfs) (push) Successful in 34s
CI / skinny-install (rex) (push) Has been cancelled
CI / skinny-install (aco) (pull_request) Successful in 51s
CI / lint-test (pull_request) Successful in 1m20s
CI / skinny-install (api) (pull_request) Successful in 29s
CI / skinny-install (bcda) (pull_request) Successful in 30s
CI / skinny-install (bib) (pull_request) Successful in 32s
CI / skinny-install (bls) (pull_request) Successful in 26s
CI / skinny-install (ccw) (pull_request) Successful in 27s
CI / skinny-install (cli) (pull_request) Successful in 29s
CI / skinny-install (cms) (pull_request) Successful in 34s
CI / skinny-install (conf) (pull_request) Successful in 22s
CI / skinny-install (opps) (pull_request) Successful in 33s
CI / skinny-install (perf) (pull_request) Successful in 31s
CI / skinny-install (pfs) (pull_request) Successful in 34s
CI / skinny-install (rex) (pull_request) Successful in 32s
Infra CI / notebooks (push) Successful in 8s
Infra CI / zotero (push) Successful in 6s
Infra CI / docs (push) Failing after 12s
Infra CI / api (push) Successful in 10s
Infra CI / mc (push) Successful in 9s
Infra CI / notebooks (pull_request) Successful in 7s
Infra CI / zotero (pull_request) Successful in 6s
Infra CI / docs (pull_request) Failing after 5s
Infra CI / api (pull_request) Successful in 6s
Infra CI / mc (pull_request) Successful in 6s
Expand Databricks SDK usage from 6 to 18 WorkspaceClient services. Tier 1 — Governance & Compliance: - governance.py: declarative GovernancePolicy with apply + audit drift - UnityClient: grants CRUD (ws.grants), secret management (ws.secrets), table constraints PK/FK (ws.table_constraints) - sync_secrets.py: push env vars to Databricks scopes per stack.toml Tier 2 — Job Orchestration: - jobs.py: JobManager translates Pipeline → Databricks Job with task dependencies, run_now, get_run_status, list/delete - UnityClient: warehouse lookup by name (ws.warehouses), start/stop Tier 3 — Data Quality & Monitoring: - quality.py: setup_monitors creates Lakehouse Monitoring profiles, list_monitors, run_refresh (ws.quality_monitors) - UnityClient: system schema access, table lineage queries, audit log queries (ws.system_schemas, ws.statement_execution) Config: stack.toml gains [databricks.warehouse], [databricks.secrets], [databricks.governance], [databricks.quality] sections. 23 new tests, all passing.
168 lines
5.0 KiB
Python
168 lines
5.0 KiB
Python
"""Tests for aco.lake.governance — declarative grant management."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from unittest.mock import MagicMock
|
|
|
|
from aco.lake.governance import (
|
|
GovernancePolicy,
|
|
GrantRule,
|
|
_resolve_schemas,
|
|
apply_governance,
|
|
audit_governance,
|
|
)
|
|
|
|
|
|
class TestResolveSchemas:
|
|
def test_wildcard_expands(self):
|
|
rule = GrantRule(group="g", schemas=["*"], privileges=["SELECT"])
|
|
assert _resolve_schemas(rule, ["core", "claims"]) == ["core", "claims"]
|
|
|
|
def test_explicit_schemas(self):
|
|
rule = GrantRule(group="g", schemas=["core"], privileges=["SELECT"])
|
|
assert _resolve_schemas(rule, ["core", "claims"]) == ["core"]
|
|
|
|
def test_missing_schema_filtered(self):
|
|
rule = GrantRule(group="g", schemas=["missing"], privileges=["SELECT"])
|
|
assert _resolve_schemas(rule, ["core"]) == []
|
|
|
|
|
|
class TestGovernancePolicyModel:
|
|
def test_basic_construction(self):
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(
|
|
group="analysts",
|
|
schemas=["readmissions"],
|
|
privileges=["SELECT"],
|
|
),
|
|
],
|
|
phi_schemas=["core"],
|
|
)
|
|
assert policy.catalog == "aco"
|
|
assert len(policy.rules) == 1
|
|
assert policy.phi_schemas == ["core"]
|
|
|
|
def test_empty_rules(self):
|
|
policy = GovernancePolicy(catalog="aco")
|
|
assert policy.rules == []
|
|
|
|
|
|
class TestApplyGovernance:
|
|
def test_dry_run_reports_actions(self):
|
|
client = MagicMock()
|
|
# Mock schemas list
|
|
schema_mock = MagicMock()
|
|
schema_mock.name = "readmissions"
|
|
client._ws.schemas.list.return_value = [schema_mock]
|
|
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(
|
|
group="analysts",
|
|
schemas=["readmissions"],
|
|
privileges=["SELECT"],
|
|
),
|
|
],
|
|
)
|
|
|
|
actions = apply_governance(client, policy, dry_run=True)
|
|
assert len(actions) == 1
|
|
assert "[dry-run]" in actions[0]
|
|
assert "GRANT SELECT" in actions[0]
|
|
assert "analysts" in actions[0]
|
|
# Should NOT call ws.grants.update in dry_run
|
|
client._ws.grants.update.assert_not_called()
|
|
|
|
def test_apply_calls_sdk(self):
|
|
client = MagicMock()
|
|
schema_mock = MagicMock()
|
|
schema_mock.name = "core"
|
|
client._ws.schemas.list.return_value = [schema_mock]
|
|
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(
|
|
group="pipeline_svc",
|
|
schemas=["core"],
|
|
privileges=["ALL_PRIVILEGES"],
|
|
),
|
|
],
|
|
)
|
|
|
|
actions = apply_governance(client, policy, dry_run=False)
|
|
assert len(actions) == 1
|
|
assert "GRANT" in actions[0]
|
|
client._ws.grants.update.assert_called_once()
|
|
|
|
def test_wildcard_expands_to_all(self):
|
|
client = MagicMock()
|
|
s1, s2 = MagicMock(), MagicMock()
|
|
s1.name, s2.name = "core", "claims"
|
|
client._ws.schemas.list.return_value = [s1, s2]
|
|
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(group="g", schemas=["*"], privileges=["SELECT"]),
|
|
],
|
|
)
|
|
|
|
actions = apply_governance(client, policy, dry_run=True)
|
|
assert len(actions) == 2
|
|
|
|
|
|
class TestAuditGovernance:
|
|
def test_detects_missing_grants(self):
|
|
client = MagicMock()
|
|
schema_mock = MagicMock()
|
|
schema_mock.name = "core"
|
|
client._ws.schemas.list.return_value = [schema_mock]
|
|
|
|
# No grants exist
|
|
resp = MagicMock()
|
|
resp.privilege_assignments = []
|
|
client._ws.grants.get.return_value = resp
|
|
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(group="analysts", schemas=["core"], privileges=["SELECT"]),
|
|
],
|
|
)
|
|
|
|
drifts = audit_governance(client, policy)
|
|
assert len(drifts) == 1
|
|
assert drifts[0].principal == "analysts"
|
|
assert drifts[0].action == "grant"
|
|
|
|
def test_no_drift_when_matching(self):
|
|
client = MagicMock()
|
|
schema_mock = MagicMock()
|
|
schema_mock.name = "core"
|
|
client._ws.schemas.list.return_value = [schema_mock]
|
|
|
|
# Grants match exactly
|
|
priv = MagicMock()
|
|
priv.privilege = MagicMock()
|
|
priv.privilege.value = "SELECT"
|
|
assignment = MagicMock()
|
|
assignment.principal = "analysts"
|
|
assignment.privileges = [priv]
|
|
resp = MagicMock()
|
|
resp.privilege_assignments = [assignment]
|
|
client._ws.grants.get.return_value = resp
|
|
|
|
policy = GovernancePolicy(
|
|
catalog="aco",
|
|
rules=[
|
|
GrantRule(group="analysts", schemas=["core"], privileges=["SELECT"]),
|
|
],
|
|
)
|
|
|
|
drifts = audit_governance(client, policy)
|
|
assert len(drifts) == 0
|