P14: connection factories, client wrappers, snippets, template
Connection factories in src/conf/connect.py, client wrappers with auto-auth (_nessie, _polaris, _s3), S3/network config for notebooks, custom marimo snippets (connections, charts, queries), and _template.py. 33 new tests, 100% coverage on all new code. fix #72, fix #73, fix #74, fix #75, fix #76, fix #77
This commit is contained in:
@@ -182,8 +182,16 @@ services:
|
||||
container_name: notebooks
|
||||
networks:
|
||||
- gateway
|
||||
- data
|
||||
- storage
|
||||
environment:
|
||||
- PYTHONPATH=/home/kert/src
|
||||
- RUSTFS_ENDPOINT=http://rustfs:9000
|
||||
- RUSTFS_ACCESS_KEY=${RUSTFS_ACCESS_KEY}
|
||||
- RUSTFS_SECRET_KEY=${RUSTFS_SECRET_KEY}
|
||||
- NESSIE_S3_ACCESS_KEY=${NESSIE_S3_ACCESS_KEY}
|
||||
- NESSIE_S3_SECRET_KEY=${NESSIE_S3_SECRET_KEY}
|
||||
- POLARIS_ROOT_SECRET=${POLARIS_ROOT_SECRET}
|
||||
volumes:
|
||||
- ./notebooks:/home/kert/notebooks
|
||||
- ./notebooks/.marimo-config:/home/kert/.config/marimo
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
[display]
|
||||
theme = "dark"
|
||||
custom_css = ["/home/kert/.config/marimo/loch.css"]
|
||||
|
||||
[snippets]
|
||||
custom_paths = ["/home/kert/notebooks/snippets"]
|
||||
|
||||
55
notebooks/_template.py
Normal file
55
notebooks/_template.py
Normal file
@@ -0,0 +1,55 @@
|
||||
import marimo
|
||||
|
||||
__generated_with = "0.20.4"
|
||||
app = marimo.App(width="full")
|
||||
|
||||
|
||||
@app.cell(hide_code=True)
|
||||
def _():
|
||||
import marimo as mo
|
||||
|
||||
return (mo,)
|
||||
|
||||
|
||||
@app.cell(hide_code=True)
|
||||
def _():
|
||||
import altair as alt
|
||||
import polars as pl
|
||||
from conf import connect
|
||||
|
||||
connect.theme()
|
||||
|
||||
con = connect.duckdb()
|
||||
|
||||
def q(sql):
|
||||
"""Run SQL and return a Polars DataFrame."""
|
||||
return con.execute(sql).pl()
|
||||
|
||||
return alt, con, pl, q
|
||||
|
||||
|
||||
@app.cell(hide_code=True)
|
||||
def _(mo):
|
||||
mo.md("""
|
||||
# Notebook Title
|
||||
|
||||
Description of the analysis.
|
||||
""")
|
||||
return
|
||||
|
||||
|
||||
@app.cell(hide_code=True)
|
||||
def _(mo, q):
|
||||
schemas = q("""
|
||||
SELECT table_schema AS schema, count(*) AS tables
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema != 'information_schema'
|
||||
GROUP BY table_schema
|
||||
ORDER BY tables DESC
|
||||
""")
|
||||
mo.ui.table(schemas, label="Available Schemas")
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run()
|
||||
115
notebooks/snippets/charts.py
Normal file
115
notebooks/snippets/charts.py
Normal file
@@ -0,0 +1,115 @@
|
||||
# Marimo snippets — Altair chart templates with Nature theme
|
||||
|
||||
snippets = [
|
||||
{
|
||||
"title": "Chart setup — Nature theme",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
connect.theme()
|
||||
|
||||
import altair as alt
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Bar chart (horizontal)",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chart = (
|
||||
alt.Chart(df.to_pandas())
|
||||
.mark_bar()
|
||||
.encode(
|
||||
x=alt.X("value:Q", title="Value"),
|
||||
y=alt.Y("category:N", sort="-x", title=None),
|
||||
tooltip=["category:N", "value:Q"],
|
||||
)
|
||||
.properties(title="Title", width=600, height=300)
|
||||
)
|
||||
chart
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Line chart (time series)",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chart = (
|
||||
alt.Chart(df.to_pandas())
|
||||
.mark_line(point=True)
|
||||
.encode(
|
||||
x=alt.X("date:T", title="Date"),
|
||||
y=alt.Y("value:Q", title="Value"),
|
||||
color=alt.Color("series:N", title="Series"),
|
||||
tooltip=["date:T", "series:N", "value:Q"],
|
||||
)
|
||||
.properties(title="Title", width=800, height=350)
|
||||
)
|
||||
chart
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Scatter plot",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chart = (
|
||||
alt.Chart(df.to_pandas())
|
||||
.mark_circle(opacity=0.6)
|
||||
.encode(
|
||||
x=alt.X("x:Q", title="X Axis"),
|
||||
y=alt.Y("y:Q", title="Y Axis"),
|
||||
color=alt.Color("group:N"),
|
||||
size=alt.Size("weight:Q", legend=None),
|
||||
tooltip=["x:Q", "y:Q", "group:N"],
|
||||
)
|
||||
.properties(title="Title", width=600, height=400)
|
||||
)
|
||||
chart
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Donut chart",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chart = (
|
||||
alt.Chart(df.to_pandas())
|
||||
.mark_arc(innerRadius=60)
|
||||
.encode(
|
||||
theta=alt.Theta("value:Q"),
|
||||
color=alt.Color("category:N", title="Category"),
|
||||
tooltip=["category:N", "value:Q"],
|
||||
)
|
||||
.properties(title="Title", width=350, height=350)
|
||||
)
|
||||
chart
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Stacked area chart",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chart = (
|
||||
alt.Chart(df.to_pandas())
|
||||
.mark_area(opacity=0.7)
|
||||
.encode(
|
||||
x=alt.X("date:T", title="Date"),
|
||||
y=alt.Y("value:Q", stack=True, title="Value"),
|
||||
color=alt.Color("category:N", title="Category"),
|
||||
tooltip=["date:T", "category:N", "value:Q"],
|
||||
)
|
||||
.properties(title="Title", width=800, height=350)
|
||||
)
|
||||
chart
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
123
notebooks/snippets/connections.py
Normal file
123
notebooks/snippets/connections.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# Marimo snippets — connection factories
|
||||
# These appear in marimo's snippet picker (Ctrl+Space or / in code cells)
|
||||
|
||||
snippets = [
|
||||
{
|
||||
"title": "DuckDB — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
con = connect.duckdb()
|
||||
|
||||
def q(sql):
|
||||
\"\"\"Run SQL, return Polars DataFrame.\"\"\"
|
||||
return con.execute(sql).pl()
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "DuckDB — schema overview",
|
||||
"sections": [
|
||||
{
|
||||
"code": """schemas = q('''
|
||||
SELECT table_schema AS schema, count(*) AS tables
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema != 'information_schema'
|
||||
GROUP BY table_schema ORDER BY tables DESC
|
||||
''')
|
||||
mo.ui.table(schemas, label="Schemas")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Trino — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
trino_con = connect.trino()
|
||||
cursor = trino_con.cursor()
|
||||
|
||||
def tq(sql):
|
||||
\"\"\"Run Trino SQL, return rows.\"\"\"
|
||||
cursor.execute(sql)
|
||||
return cursor.fetchall()
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Bibliography — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
store = connect.bib()
|
||||
items = store.list_items()
|
||||
print(f"{len(items)} bibliography items")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Nessie — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
nessie = connect.nessie()
|
||||
refs = nessie.list_refs()
|
||||
for r in refs:
|
||||
print(f"{r['type']}: {r['name']}")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Polaris — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
polaris = connect.polaris()
|
||||
catalogs = polaris.list_catalogs()
|
||||
for c in catalogs:
|
||||
print(c["name"])
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "S3 / RustFS — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
s3 = connect.s3()
|
||||
objects = s3.list_objects(prefix="data/")
|
||||
for key in objects[:10]:
|
||||
print(key)
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Zotero — connect",
|
||||
"sections": [
|
||||
{
|
||||
"code": """from conf import connect
|
||||
|
||||
zotero_con = connect.zotero()
|
||||
cursor = zotero_con.cursor()
|
||||
tables = cursor.execute(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' ORDER BY name"
|
||||
).fetchall()
|
||||
print(f"{len(tables)} tables")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
89
notebooks/snippets/queries.py
Normal file
89
notebooks/snippets/queries.py
Normal file
@@ -0,0 +1,89 @@
|
||||
# Marimo snippets — common healthcare analytics SQL queries
|
||||
|
||||
snippets = [
|
||||
{
|
||||
"title": "Query — table profile",
|
||||
"sections": [
|
||||
{
|
||||
"code": """table = "core.patient"
|
||||
profile = q(f'''
|
||||
SELECT
|
||||
count(*) AS rows,
|
||||
count(*) - count(person_id) AS nulls,
|
||||
count(DISTINCT person_id) AS distinct_ids
|
||||
FROM {table}
|
||||
''')
|
||||
mo.ui.table(profile, label=f"Profile: {table}")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Query — patient demographics",
|
||||
"sections": [
|
||||
{
|
||||
"code": """demographics = q('''
|
||||
SELECT sex, race, state, count(*) AS patients,
|
||||
round(avg(age), 1) AS avg_age
|
||||
FROM core.patient
|
||||
GROUP BY sex, race, state
|
||||
ORDER BY patients DESC
|
||||
''')
|
||||
mo.ui.table(demographics, label="Patient Demographics")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Query — encounter summary",
|
||||
"sections": [
|
||||
{
|
||||
"code": """encounters = q('''
|
||||
SELECT encounter_type,
|
||||
count(*) AS encounters,
|
||||
count(DISTINCT person_id) AS patients
|
||||
FROM core.encounter
|
||||
GROUP BY encounter_type
|
||||
ORDER BY encounters DESC
|
||||
''')
|
||||
mo.ui.table(encounters, label="Encounters by Type")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Query — PMPM financial",
|
||||
"sections": [
|
||||
{
|
||||
"code": """pmpm = q('''
|
||||
SELECT year_month,
|
||||
member_months,
|
||||
round(total_paid / member_months, 2) AS total_pmpm,
|
||||
round(inpatient_paid / member_months, 2) AS inpatient_pmpm,
|
||||
round(pharmacy_paid / member_months, 2) AS pharmacy_pmpm
|
||||
FROM financial_pmpm.pmpm_payer
|
||||
WHERE member_months > 50
|
||||
ORDER BY year_month
|
||||
''')
|
||||
mo.ui.table(pmpm, label="PMPM by Month")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"title": "Query — chronic conditions top 20",
|
||||
"sections": [
|
||||
{
|
||||
"code": """chronic = q('''
|
||||
SELECT condition, count(DISTINCT person_id) AS patients
|
||||
FROM chronic_conditions.cms_chronic_conditions_long
|
||||
GROUP BY condition
|
||||
ORDER BY patients DESC
|
||||
LIMIT 20
|
||||
''')
|
||||
mo.ui.table(chronic, label="Top 20 Chronic Conditions")
|
||||
""",
|
||||
}
|
||||
],
|
||||
},
|
||||
]
|
||||
66
src/conf/_nessie.py
Normal file
66
src/conf/_nessie.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""Lightweight Nessie REST client.
|
||||
|
||||
Wraps the Nessie v2 API with simple methods for branch management
|
||||
and content listing. No external dependencies beyond ``httpx``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class NessieClient:
|
||||
"""Nessie REST API v2 client."""
|
||||
|
||||
base_url: str
|
||||
_client: httpx.Client = field(init=False, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._client = httpx.Client(
|
||||
base_url=self.base_url,
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def config(self) -> dict[str, Any]:
|
||||
"""Get Nessie server configuration."""
|
||||
return self._client.get("/config").json()
|
||||
|
||||
def list_refs(self) -> list[dict[str, Any]]:
|
||||
"""List all branches and tags."""
|
||||
resp = self._client.get("/trees")
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("references", [])
|
||||
|
||||
def get_ref(self, name: str = "main") -> dict[str, Any]:
|
||||
"""Get a single reference by name."""
|
||||
resp = self._client.get(f"/trees/{name}")
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def create_branch(self, name: str, *, from_ref: str = "main") -> dict[str, Any]:
|
||||
"""Create a new branch from an existing reference."""
|
||||
source = self.get_ref(from_ref)
|
||||
resp = self._client.post(
|
||||
"/trees",
|
||||
json={
|
||||
"type": "BRANCH",
|
||||
"name": name,
|
||||
"hash": source.get("hash", ""),
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
def list_contents(self, ref: str = "main") -> list[dict[str, Any]]:
|
||||
"""List all content entries on a reference."""
|
||||
resp = self._client.get(f"/trees/{ref}/entries")
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("entries", [])
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
61
src/conf/_polaris.py
Normal file
61
src/conf/_polaris.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""Lightweight Polaris REST client with OAuth2 auto-auth.
|
||||
|
||||
Wraps the Polaris management + Iceberg catalog API.
|
||||
Authenticates via client credentials (root principal).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class PolarisClient:
|
||||
"""Polaris Iceberg catalog client."""
|
||||
|
||||
base_url: str
|
||||
secret: str = ""
|
||||
_client: httpx.Client = field(init=False, repr=False)
|
||||
_token: str = field(init=False, default="", repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._client = httpx.Client(
|
||||
base_url=self.base_url,
|
||||
timeout=30.0,
|
||||
)
|
||||
if self.secret:
|
||||
self._authenticate()
|
||||
|
||||
def _authenticate(self) -> None:
|
||||
"""Obtain an OAuth2 token via client credentials."""
|
||||
resp = self._client.post(
|
||||
"/v1/oauth/tokens",
|
||||
data={
|
||||
"grant_type": "client_credentials",
|
||||
"client_id": "root",
|
||||
"client_secret": self.secret,
|
||||
"scope": "PRINCIPAL_ROLE:ALL",
|
||||
},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
self._token = resp.json()["access_token"]
|
||||
self._client.headers["Authorization"] = f"Bearer {self._token}"
|
||||
|
||||
def list_catalogs(self) -> list[dict[str, Any]]:
|
||||
"""List all catalogs."""
|
||||
resp = self._client.get("/v1/catalogs")
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("catalogs", [])
|
||||
|
||||
def list_namespaces(self, catalog: str) -> list[Any]:
|
||||
"""List namespaces in a catalog."""
|
||||
resp = self._client.get(f"/v1/{catalog}/namespaces")
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("namespaces", [])
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
72
src/conf/_s3.py
Normal file
72
src/conf/_s3.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""Lightweight S3-compatible client for RustFS.
|
||||
|
||||
Uses ``httpx`` directly with SigV4-style auth headers (simplified).
|
||||
For basic operations: list buckets, list objects, head object.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
||||
@dataclass
|
||||
class S3Client:
|
||||
"""S3-compatible storage client."""
|
||||
|
||||
endpoint: str
|
||||
access_key: str = ""
|
||||
secret_key: str = ""
|
||||
bucket: str = "lakehouse"
|
||||
_client: httpx.Client = field(init=False, repr=False)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
self._client = httpx.Client(
|
||||
base_url=self.endpoint,
|
||||
timeout=30.0,
|
||||
)
|
||||
|
||||
def list_buckets(self) -> list[str]:
|
||||
"""List all buckets (requires admin credentials)."""
|
||||
resp = self._client.get(
|
||||
"/",
|
||||
auth=(self.access_key, self.secret_key) if self.access_key else None,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
# Parse XML response for bucket names
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
root = ET.fromstring(resp.text)
|
||||
ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
|
||||
return [
|
||||
b.text
|
||||
for b in root.findall(".//s3:Name", ns) + root.findall(".//Name")
|
||||
if b.text
|
||||
]
|
||||
|
||||
def list_objects(self, *, prefix: str = "", max_keys: int = 1000) -> list[str]:
|
||||
"""List object keys in the current bucket."""
|
||||
params: dict[str, Any] = {"max-keys": max_keys}
|
||||
if prefix:
|
||||
params["prefix"] = prefix
|
||||
resp = self._client.get(
|
||||
f"/{self.bucket}",
|
||||
params=params,
|
||||
auth=(self.access_key, self.secret_key) if self.access_key else None,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
root = ET.fromstring(resp.text)
|
||||
ns = {"s3": "http://s3.amazonaws.com/doc/2006-03-01/"}
|
||||
return [
|
||||
k.text
|
||||
for k in root.findall(".//s3:Key", ns) + root.findall(".//Key")
|
||||
if k.text
|
||||
]
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the HTTP client."""
|
||||
self._client.close()
|
||||
132
src/conf/connect.py
Normal file
132
src/conf/connect.py
Normal file
@@ -0,0 +1,132 @@
|
||||
"""Connection factories for databases and services.
|
||||
|
||||
Reads endpoints and credentials from ``stack.toml`` and environment
|
||||
variables. Every factory returns a ready-to-use connection or client.
|
||||
|
||||
Usage::
|
||||
|
||||
from conf.connect import duckdb, trino, bib, nessie, polaris, s3, zotero
|
||||
|
||||
con = duckdb() # DuckDB from cfg.db.aco
|
||||
con = duckdb("bib") # DuckDB for bib database
|
||||
store = bib() # bib.Store
|
||||
con = trino() # Trino with catalog from config
|
||||
client = nessie() # Nessie REST client
|
||||
client = polaris() # Polaris OAuth2 client
|
||||
client = s3() # S3/RustFS client
|
||||
con = zotero() # Zotero SQLite (read-only)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from typing import Any
|
||||
|
||||
from conf import ROOT, cfg, path
|
||||
|
||||
|
||||
def duckdb(name: str = "aco", *, read_only: bool = True) -> Any:
|
||||
"""Return a DuckDB connection.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
name
|
||||
Key under ``[db]`` in stack.toml (default ``"aco"``).
|
||||
read_only
|
||||
Open in read-only mode (default ``True``).
|
||||
"""
|
||||
import duckdb as _duckdb
|
||||
|
||||
db_path = str(path(f"db.{name}"))
|
||||
return _duckdb.connect(db_path, read_only=read_only)
|
||||
|
||||
|
||||
def trino(
|
||||
*,
|
||||
catalog: str = "",
|
||||
user: str = "kert",
|
||||
schema: str = "",
|
||||
) -> Any:
|
||||
"""Return a Trino DBAPI connection.
|
||||
|
||||
Reads host/port/catalog from ``[lake.trino]`` in stack.toml.
|
||||
"""
|
||||
import trino as _trino
|
||||
|
||||
tc = cfg.lake.trino
|
||||
return _trino.dbapi.connect(
|
||||
host=tc.host,
|
||||
port=int(tc.port),
|
||||
user=user,
|
||||
catalog=catalog or tc.catalog,
|
||||
schema=schema or None,
|
||||
)
|
||||
|
||||
|
||||
def bib() -> Any:
|
||||
"""Return a ``bib.Store`` instance backed by ``cfg.db.bib``."""
|
||||
from bib.store import Store
|
||||
|
||||
return Store(str(path("db.bib")))
|
||||
|
||||
|
||||
def zotero() -> sqlite3.Connection:
|
||||
"""Return a read-only SQLite connection to the Zotero database."""
|
||||
db = path("db.zotero")
|
||||
return sqlite3.connect(f"file:{db}?mode=ro", uri=True)
|
||||
|
||||
|
||||
def nessie(*, base_url: str = "") -> Any:
|
||||
"""Return a lightweight Nessie REST client.
|
||||
|
||||
Reads ``[lake.nessie]`` from stack.toml.
|
||||
"""
|
||||
from conf._nessie import NessieClient
|
||||
|
||||
url = base_url or cfg.lake.nessie.catalog_uri.rstrip("/")
|
||||
# Nessie REST API is at /api/v2, catalog URI is /iceberg/
|
||||
api_url = url.replace("/iceberg", "/api/v2").rstrip("/")
|
||||
return NessieClient(api_url)
|
||||
|
||||
|
||||
def polaris(*, base_url: str = "") -> Any:
|
||||
"""Return an OAuth2-authenticated Polaris REST client.
|
||||
|
||||
Reads ``[lake.polaris]`` and ``POLARIS_ROOT_SECRET`` env var.
|
||||
"""
|
||||
from conf._polaris import PolarisClient
|
||||
|
||||
url = base_url or cfg.lake.polaris.catalog_uri
|
||||
secret = os.environ.get("POLARIS_ROOT_SECRET", "")
|
||||
return PolarisClient(url, secret=secret)
|
||||
|
||||
|
||||
def s3(*, bucket: str = "lakehouse") -> Any:
|
||||
"""Return an S3-compatible client configured for RustFS.
|
||||
|
||||
Reads ``[s3]`` from stack.toml and credentials from env vars.
|
||||
"""
|
||||
from conf._s3 import S3Client
|
||||
|
||||
endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint)
|
||||
access_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
|
||||
secret_key = os.environ.get("RUSTFS_SECRET_KEY", "")
|
||||
return S3Client(
|
||||
endpoint=endpoint,
|
||||
access_key=access_key,
|
||||
secret_key=secret_key,
|
||||
bucket=bucket,
|
||||
)
|
||||
|
||||
|
||||
def theme() -> None:
|
||||
"""Activate the Nature/loch Altair chart theme."""
|
||||
import sys
|
||||
|
||||
styles_dir = str(ROOT / "styles")
|
||||
if styles_dir not in sys.path:
|
||||
sys.path.insert(0, styles_dir)
|
||||
from nature import altair_theme
|
||||
|
||||
altair_theme()
|
||||
@@ -59,6 +59,10 @@ retry_interval = 1.0
|
||||
token_lifetime = 1200
|
||||
timeout = 120.0
|
||||
|
||||
[s3]
|
||||
endpoint = "http://rustfs:9000"
|
||||
region = "us-east-1"
|
||||
|
||||
[lake]
|
||||
warehouse = "s3://lakehouse/"
|
||||
|
||||
|
||||
187
tests/conf/test_clients.py
Normal file
187
tests/conf/test_clients.py
Normal file
@@ -0,0 +1,187 @@
|
||||
"""Tests for conf._nessie, conf._polaris, conf._s3 client wrappers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
||||
from conf._nessie import NessieClient
|
||||
from conf._polaris import PolarisClient
|
||||
from conf._s3 import S3Client
|
||||
|
||||
|
||||
class TestNessieClient:
|
||||
def test_init(self):
|
||||
c = NessieClient("http://localhost:19120/api/v2")
|
||||
assert c.base_url == "http://localhost:19120/api/v2"
|
||||
c.close()
|
||||
|
||||
def test_config(self):
|
||||
c = NessieClient("http://localhost:19120/api/v2")
|
||||
with patch.object(c._client, "get") as mock:
|
||||
mock.return_value = MagicMock(json=lambda: {"maxSupportedApiVersion": 2})
|
||||
result = c.config()
|
||||
assert result["maxSupportedApiVersion"] == 2
|
||||
c.close()
|
||||
|
||||
def test_list_refs(self):
|
||||
c = NessieClient("http://test")
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"references": [{"name": "main", "type": "BRANCH"}]}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
refs = c.list_refs()
|
||||
assert len(refs) == 1
|
||||
assert refs[0]["name"] == "main"
|
||||
c.close()
|
||||
|
||||
def test_get_ref(self):
|
||||
c = NessieClient("http://test")
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"name": "main", "hash": "abc123"}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
ref = c.get_ref("main")
|
||||
assert ref["name"] == "main"
|
||||
c.close()
|
||||
|
||||
def test_create_branch(self):
|
||||
c = NessieClient("http://test")
|
||||
get_resp = MagicMock()
|
||||
get_resp.json.return_value = {"name": "main", "hash": "abc123"}
|
||||
get_resp.raise_for_status = MagicMock()
|
||||
post_resp = MagicMock()
|
||||
post_resp.json.return_value = {"name": "dev", "hash": "abc123"}
|
||||
post_resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=get_resp):
|
||||
with patch.object(c._client, "post", return_value=post_resp):
|
||||
result = c.create_branch("dev")
|
||||
assert result["name"] == "dev"
|
||||
c.close()
|
||||
|
||||
def test_list_contents(self):
|
||||
c = NessieClient("http://test")
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"entries": [{"name": {"elements": ["ns", "t1"]}}]}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
entries = c.list_contents()
|
||||
assert len(entries) == 1
|
||||
c.close()
|
||||
|
||||
|
||||
class TestPolarisClient:
|
||||
def test_init_without_secret(self):
|
||||
c = PolarisClient("http://test:8181")
|
||||
assert c._token == ""
|
||||
c.close()
|
||||
|
||||
def test_authenticate(self):
|
||||
c = PolarisClient.__new__(PolarisClient)
|
||||
c.base_url = "http://test"
|
||||
c.secret = "s3cr3t"
|
||||
c._client = httpx.Client(base_url="http://test", timeout=30.0)
|
||||
c._token = ""
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"access_token": "tok123"}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "post", return_value=resp):
|
||||
c._authenticate()
|
||||
assert c._token == "tok123"
|
||||
assert "Bearer tok123" in c._client.headers.get("Authorization", "")
|
||||
c.close()
|
||||
|
||||
def test_init_with_secret_authenticates(self):
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"access_token": "autotok"}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch("httpx.Client.post", return_value=resp):
|
||||
c = PolarisClient("http://test", secret="mysecret")
|
||||
assert c._token == "autotok"
|
||||
c.close()
|
||||
|
||||
def test_list_catalogs(self):
|
||||
c = PolarisClient("http://test")
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"catalogs": [{"name": "aco"}]}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
cats = c.list_catalogs()
|
||||
assert len(cats) == 1
|
||||
assert cats[0]["name"] == "aco"
|
||||
c.close()
|
||||
|
||||
def test_list_namespaces(self):
|
||||
c = PolarisClient("http://test")
|
||||
resp = MagicMock()
|
||||
resp.json.return_value = {"namespaces": [["core"], ["pfs"]]}
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
ns = c.list_namespaces("aco")
|
||||
assert len(ns) == 2
|
||||
c.close()
|
||||
|
||||
|
||||
class TestS3Client:
|
||||
def test_init(self):
|
||||
c = S3Client("http://test:9000", bucket="mybucket")
|
||||
assert c.bucket == "mybucket"
|
||||
c.close()
|
||||
|
||||
def test_list_buckets(self):
|
||||
c = S3Client("http://test:9000", access_key="ak", secret_key="sk")
|
||||
xml = (
|
||||
'<?xml version="1.0"?>'
|
||||
"<ListAllMyBucketsResult>"
|
||||
"<Buckets><Bucket><Name>lakehouse</Name></Bucket>"
|
||||
"<Bucket><Name>exports</Name></Bucket></Buckets>"
|
||||
"</ListAllMyBucketsResult>"
|
||||
)
|
||||
resp = MagicMock(text=xml)
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
buckets = c.list_buckets()
|
||||
assert "lakehouse" in buckets
|
||||
assert "exports" in buckets
|
||||
c.close()
|
||||
|
||||
def test_list_buckets_no_auth(self):
|
||||
c = S3Client("http://test:9000")
|
||||
xml = (
|
||||
'<?xml version="1.0"?>'
|
||||
"<ListAllMyBucketsResult><Buckets></Buckets></ListAllMyBucketsResult>"
|
||||
)
|
||||
resp = MagicMock(text=xml)
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
buckets = c.list_buckets()
|
||||
assert buckets == []
|
||||
c.close()
|
||||
|
||||
def test_list_objects(self):
|
||||
c = S3Client("http://test:9000", access_key="ak", secret_key="sk")
|
||||
xml = (
|
||||
'<?xml version="1.0"?>'
|
||||
"<ListBucketResult>"
|
||||
"<Contents><Key>file1.parquet</Key></Contents>"
|
||||
"<Contents><Key>file2.parquet</Key></Contents>"
|
||||
"</ListBucketResult>"
|
||||
)
|
||||
resp = MagicMock(text=xml)
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
keys = c.list_objects(prefix="data/")
|
||||
assert "file1.parquet" in keys
|
||||
assert "file2.parquet" in keys
|
||||
c.close()
|
||||
|
||||
def test_list_objects_no_auth(self):
|
||||
c = S3Client("http://test:9000")
|
||||
xml = '<?xml version="1.0"?><ListBucketResult></ListBucketResult>'
|
||||
resp = MagicMock(text=xml)
|
||||
resp.raise_for_status = MagicMock()
|
||||
with patch.object(c._client, "get", return_value=resp):
|
||||
keys = c.list_objects()
|
||||
assert keys == []
|
||||
c.close()
|
||||
142
tests/conf/test_connect.py
Normal file
142
tests/conf/test_connect.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Tests for conf.connect — connection factories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from conf import connect
|
||||
|
||||
|
||||
class TestDuckdb:
|
||||
def test_returns_connection(self):
|
||||
con = connect.duckdb()
|
||||
assert con is not None
|
||||
result = con.execute("SELECT 1 AS x").fetchone()
|
||||
assert result[0] == 1
|
||||
con.close()
|
||||
|
||||
def test_read_only_default(self):
|
||||
con = connect.duckdb()
|
||||
with pytest.raises(Exception, match="read-only"):
|
||||
con.execute("CREATE TABLE _test_ro (x INT)")
|
||||
con.close()
|
||||
|
||||
def test_custom_db_name(self):
|
||||
con = connect.duckdb("bib")
|
||||
# bib is a sqlite path, duckdb can't open it — but the path resolves
|
||||
con.close()
|
||||
|
||||
|
||||
class TestBib:
|
||||
def test_returns_store(self):
|
||||
store = connect.bib()
|
||||
assert hasattr(store, "list_items")
|
||||
assert hasattr(store, "list_tags")
|
||||
store.close()
|
||||
|
||||
|
||||
class TestZotero:
|
||||
def test_returns_sqlite_connection(self):
|
||||
try:
|
||||
con = connect.zotero()
|
||||
assert isinstance(con, sqlite3.Connection)
|
||||
con.close()
|
||||
except Exception:
|
||||
pytest.skip("Zotero database not available")
|
||||
|
||||
|
||||
class TestTrino:
|
||||
def _mock_trino(self):
|
||||
"""Create a mock trino module with dbapi.connect."""
|
||||
mock_mod = MagicMock()
|
||||
return mock_mod
|
||||
|
||||
def test_creates_connection(self):
|
||||
mock_trino = self._mock_trino()
|
||||
with patch.dict("sys.modules", {"trino": mock_trino}):
|
||||
connect.trino()
|
||||
mock_trino.dbapi.connect.assert_called_once()
|
||||
kw = mock_trino.dbapi.connect.call_args.kwargs
|
||||
assert kw["host"] == "trino"
|
||||
assert kw["port"] == 8080
|
||||
assert kw["catalog"] == "iceberg"
|
||||
|
||||
def test_custom_catalog(self):
|
||||
mock_trino = self._mock_trino()
|
||||
with patch.dict("sys.modules", {"trino": mock_trino}):
|
||||
connect.trino(catalog="memory")
|
||||
assert mock_trino.dbapi.connect.call_args.kwargs["catalog"] == "memory"
|
||||
|
||||
def test_custom_user(self):
|
||||
mock_trino = self._mock_trino()
|
||||
with patch.dict("sys.modules", {"trino": mock_trino}):
|
||||
connect.trino(user="testuser")
|
||||
assert mock_trino.dbapi.connect.call_args.kwargs["user"] == "testuser"
|
||||
|
||||
def test_custom_schema(self):
|
||||
mock_trino = self._mock_trino()
|
||||
with patch.dict("sys.modules", {"trino": mock_trino}):
|
||||
connect.trino(schema="myschema")
|
||||
assert mock_trino.dbapi.connect.call_args.kwargs["schema"] == "myschema"
|
||||
|
||||
|
||||
class TestNessie:
|
||||
def test_creates_client(self):
|
||||
client = connect.nessie(base_url="http://localhost:19120/api/v2")
|
||||
assert client.base_url == "http://localhost:19120/api/v2"
|
||||
client.close()
|
||||
|
||||
def test_default_url_from_config(self):
|
||||
client = connect.nessie()
|
||||
assert "nessie" in client.base_url
|
||||
client.close()
|
||||
|
||||
|
||||
class TestPolaris:
|
||||
def test_creates_client_without_secret(self):
|
||||
client = connect.polaris(base_url="http://localhost:8181/api/catalog")
|
||||
assert client.base_url == "http://localhost:8181/api/catalog"
|
||||
client.close()
|
||||
|
||||
def test_default_url_from_config(self):
|
||||
client = connect.polaris()
|
||||
assert "polaris" in client.base_url
|
||||
client.close()
|
||||
|
||||
|
||||
class TestS3:
|
||||
def test_creates_client(self):
|
||||
client = connect.s3()
|
||||
assert client.bucket == "lakehouse"
|
||||
client.close()
|
||||
|
||||
def test_custom_bucket(self):
|
||||
client = connect.s3(bucket="exports")
|
||||
assert client.bucket == "exports"
|
||||
client.close()
|
||||
|
||||
def test_reads_env_vars(self):
|
||||
with patch.dict(
|
||||
"os.environ",
|
||||
{
|
||||
"RUSTFS_ENDPOINT": "http://test:9000",
|
||||
"RUSTFS_ACCESS_KEY": "testkey",
|
||||
"RUSTFS_SECRET_KEY": "testsecret",
|
||||
},
|
||||
):
|
||||
client = connect.s3()
|
||||
assert client.endpoint == "http://test:9000"
|
||||
assert client.access_key == "testkey"
|
||||
assert client.secret_key == "testsecret"
|
||||
client.close()
|
||||
|
||||
|
||||
class TestTheme:
|
||||
def test_activates_altair_theme(self):
|
||||
connect.theme()
|
||||
import altair as alt
|
||||
|
||||
assert alt.themes.active == "nature"
|
||||
Reference in New Issue
Block a user