446f1b9398
All checks were successful
CI / lint (push) Successful in 34s
CI / notebooks-smoke (push) Successful in 1m28s
Deploy / notebooks (push) Has been skipped
Deploy / zotero (push) Has been skipped
Deploy / docs (push) Has been skipped
Deploy / api (push) Has been skipped
Deploy / mc (push) Has been skipped
Infra CI / notebooks (push) Successful in 53s
Infra CI / zotero (push) Successful in 16s
Infra CI / docs (push) Successful in 1m18s
Infra CI / api (push) Successful in 51s
Infra CI / mc (push) Successful in 12s
Deploy / report (push) Successful in 13s
CI / test (push) Successful in 15m7s
Notebooks Integration / notebooks-integration (push) Successful in 7m37s
aco.lake.DuckLakeContext: the concrete lake context for the M3 decision (#512). Subclasses DuckDBContext — load/save are inherited untouched; only the connection wiring differs (ducklake+httpfs extensions, S3 secret for RustFS, ATTACH the catalog, USE it as the default database so schema.table refs resolve in the lake). Postgres DSN for multi-writer prod, plain file path for dev/tests. Explicit close() added to the context family (releases locks/attachments). Pilot: dev/scripts/publish_opps_to_lake.py reads the OPPS tables from the replica and writes them through Context.save(mode=replace). Run against prod (postgres catalog db 'ducklake' + RustFS): addendum_b 219,665 rows in 2.6s, apc_weight 7,776, skin_sub_addendum_b 2,520 — all read-back verified through Context.load. Config: stack.toml [lake.ducklake] (catalog DSN sans password, data path, s3 endpoint). CI: ducklake added to the duckdb extension preinstall (same xdist install race class as #515). 6 new tests use a file catalog + local data path — same code paths, no services needed.
84 lines
3.0 KiB
Python
84 lines
3.0 KiB
Python
"""Tests for aco.lake.DuckLakeContext — file catalog + local data dir.
|
|
|
|
No postgres/S3 required: DuckLake accepts a local file catalog and a
|
|
local DATA_PATH, which exercises the exact same load/save code paths
|
|
as the prod postgres+RustFS configuration.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import narwhals as nw
|
|
import polars as pl
|
|
import pytest
|
|
|
|
from aco.lake import DuckLakeContext
|
|
|
|
|
|
def _writer(tmp_path) -> DuckLakeContext:
|
|
return DuckLakeContext(
|
|
catalog_dsn=str(tmp_path / "catalog.ducklake"),
|
|
data_path=str(tmp_path / "lake-data") + "/",
|
|
read_only=False,
|
|
)
|
|
|
|
|
|
def _frame() -> nw.DataFrame:
|
|
return nw.from_native(
|
|
pl.DataFrame(
|
|
{
|
|
"hcpcs": ["Q4186", "C5271"],
|
|
"year": [2026, 2026],
|
|
"payment_rate": [127.28, 127.28],
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
class TestDuckLakeContext:
|
|
def test_save_replace_and_load_roundtrip(self, tmp_path):
|
|
ctx = _writer(tmp_path)
|
|
ctx.save("opps.addendum_b", _frame(), mode="replace")
|
|
back = ctx.load("opps.addendum_b")
|
|
native = nw.to_native(back)
|
|
assert native.height == 2
|
|
assert set(native.columns) == {"hcpcs", "year", "payment_rate"}
|
|
|
|
def test_append_creates_then_appends(self, tmp_path):
|
|
ctx = _writer(tmp_path)
|
|
ctx.save("opps.addendum_b", _frame()) # append creates
|
|
ctx.save("opps.addendum_b", _frame()) # append adds
|
|
assert nw.to_native(ctx.load("opps.addendum_b")).height == 4
|
|
|
|
def test_replace_resets(self, tmp_path):
|
|
ctx = _writer(tmp_path)
|
|
ctx.save("opps.addendum_b", _frame())
|
|
ctx.save("opps.addendum_b", _frame(), mode="replace")
|
|
assert nw.to_native(ctx.load("opps.addendum_b")).height == 2
|
|
|
|
def test_read_only_guard(self, tmp_path):
|
|
writer = _writer(tmp_path)
|
|
writer.save("opps.addendum_b", _frame(), mode="replace")
|
|
writer.close() # release the file-catalog attachment
|
|
reader = DuckLakeContext(
|
|
catalog_dsn=str(tmp_path / "catalog.ducklake"),
|
|
data_path=str(tmp_path / "lake-data") + "/",
|
|
read_only=True,
|
|
)
|
|
assert nw.to_native(reader.load("opps.addendum_b")).height == 2
|
|
with pytest.raises(RuntimeError, match="read-only"):
|
|
reader.save("opps.addendum_b", _frame())
|
|
|
|
def test_unqualified_ref_rejected(self, tmp_path):
|
|
ctx = _writer(tmp_path)
|
|
with pytest.raises(ValueError, match="qualified"):
|
|
ctx.load("addendum_b")
|
|
with pytest.raises(ValueError, match="qualified"):
|
|
ctx.save("addendum_b", _frame())
|
|
|
|
def test_second_context_sees_committed_data(self, tmp_path):
|
|
"""Commits are visible to a separate connection through the shared
|
|
catalog — the property the monolith could never provide."""
|
|
_writer(tmp_path).save("opps.apc_weight", _frame(), mode="replace")
|
|
other = _writer(tmp_path)
|
|
assert nw.to_native(other.load("opps.apc_weight")).height == 2
|