cb530c25d9
Bundled commit across unrelated threads: perf collector/meter/logs/export
updates, SSO bootstrap and traefik/nginx/gitea infra, dev backend scripts
(gitea/github/woodpecker), diag test updates, and misc config.
Also fix test pollution: five tests were doing raw sys.modules.pop("conf")
to simulate ImportError on `import conf`, but raw dict mutation isn't
tracked by monkeypatch and leaked the eviction into subsequent tests.
Downstream tests that did `from conf import secret` at module level held
references to the pre-eviction conf, while re-imports inside tests got a
fresh conf, so patches to conf.cfg._data didn't land on the secret()
closure's cfg — causing tests/conf/test_conf.py::TestSecret and
tests/conf/test_connect.py::TestDuckdb::test_custom_db_name to fail.
Fix: use monkeypatch.delitem(sys.modules, "conf", raising=False) so the
eviction is reverted on teardown. Applied to test_diag_ci.py, test_init.py,
test_tracer.py, and test_resource.py (two sites).
133 lines
4.5 KiB
Python
133 lines
4.5 KiB
Python
"""Tests for conf.storage — multi-cloud filesystem dispatch."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
import types
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
from conf.storage import get_filesystem
|
|
|
|
|
|
class _Ctx:
|
|
def __init__(self, backend: str) -> None:
|
|
self.storage_backend = backend
|
|
|
|
|
|
class TestLocalBackend:
|
|
def test_local_returns_fsspec_local(self):
|
|
with patch("conf.context", return_value=_Ctx("local")):
|
|
fs = get_filesystem()
|
|
# fsspec("file") returns LocalFileSystem
|
|
assert hasattr(fs, "ls")
|
|
|
|
|
|
class TestS3Backend:
|
|
def test_s3_uses_env_credentials(self, monkeypatch):
|
|
monkeypatch.setenv("RUSTFS_ENDPOINT", "http://rustfs:9000")
|
|
monkeypatch.setenv("RUSTFS_ACCESS_KEY", "ak")
|
|
monkeypatch.setenv("RUSTFS_SECRET_KEY", "sk")
|
|
|
|
with patch("conf.context", return_value=_Ctx("s3")):
|
|
fs = get_filesystem()
|
|
# fsspec returns S3FileSystem; minimal check that it's not None
|
|
assert fs is not None
|
|
|
|
|
|
class TestGcsBackend:
|
|
def test_gcs_imports_gcsfs(self, monkeypatch):
|
|
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "my-project")
|
|
fake_gcs_fs = MagicMock()
|
|
fake_module = types.ModuleType("gcsfs")
|
|
fake_module.GCSFileSystem = MagicMock(return_value=fake_gcs_fs)
|
|
monkeypatch.setitem(sys.modules, "gcsfs", fake_module)
|
|
|
|
with patch("conf.context", return_value=_Ctx("gcs")):
|
|
fs = get_filesystem()
|
|
assert fs is fake_gcs_fs
|
|
fake_module.GCSFileSystem.assert_called_once_with(project="my-project")
|
|
|
|
def test_gcs_missing_package_raises_with_hint(self, monkeypatch):
|
|
# Force the gcsfs import to fail.
|
|
real_import = (
|
|
__builtins__["__import__"]
|
|
if isinstance(__builtins__, dict)
|
|
else __builtins__.__import__
|
|
)
|
|
|
|
def fake_import(name, *a, **kw):
|
|
if name == "gcsfs":
|
|
raise ImportError("not installed")
|
|
return real_import(name, *a, **kw)
|
|
|
|
monkeypatch.setattr("builtins.__import__", fake_import)
|
|
sys.modules.pop("gcsfs", None)
|
|
with patch("conf.context", return_value=_Ctx("gcs")):
|
|
with pytest.raises(ImportError, match="stack\\[gcp\\]"):
|
|
get_filesystem()
|
|
|
|
|
|
class TestAbfsBackend:
|
|
def test_abfs_imports_adlfs(self, monkeypatch):
|
|
monkeypatch.setenv("AZURE_STORAGE_ACCOUNT", "myacct")
|
|
fake_az_fs = MagicMock()
|
|
fake_module = types.ModuleType("adlfs")
|
|
fake_module.AzureBlobFileSystem = MagicMock(return_value=fake_az_fs)
|
|
monkeypatch.setitem(sys.modules, "adlfs", fake_module)
|
|
|
|
with patch("conf.context", return_value=_Ctx("abfs")):
|
|
fs = get_filesystem()
|
|
assert fs is fake_az_fs
|
|
fake_module.AzureBlobFileSystem.assert_called_once_with(account_name="myacct")
|
|
|
|
def test_abfs_missing_package_raises_with_hint(self, monkeypatch):
|
|
real_import = (
|
|
__builtins__["__import__"]
|
|
if isinstance(__builtins__, dict)
|
|
else __builtins__.__import__
|
|
)
|
|
|
|
def fake_import(name, *a, **kw):
|
|
if name == "adlfs":
|
|
raise ImportError("not installed")
|
|
return real_import(name, *a, **kw)
|
|
|
|
monkeypatch.setattr("builtins.__import__", fake_import)
|
|
sys.modules.pop("adlfs", None)
|
|
with patch("conf.context", return_value=_Ctx("abfs")):
|
|
with pytest.raises(ImportError, match="stack\\[azure\\]"):
|
|
get_filesystem()
|
|
|
|
|
|
class TestDbfsBackend:
|
|
def test_dbfs_uses_fsspec(self, monkeypatch):
|
|
# fsspec doesn't ship a dbfs filesystem by default; stub it.
|
|
import fsspec
|
|
|
|
fake_fs = MagicMock()
|
|
monkeypatch.setattr(fsspec, "filesystem", lambda name, **kw: fake_fs)
|
|
with patch("conf.context", return_value=_Ctx("dbfs")):
|
|
fs = get_filesystem()
|
|
assert fs is fake_fs
|
|
|
|
|
|
class TestUnknownBackend:
|
|
def test_raises_value_error(self):
|
|
with patch("conf.context", return_value=_Ctx("warpdrive")):
|
|
with pytest.raises(ValueError, match="Unknown storage backend"):
|
|
get_filesystem()
|
|
|
|
|
|
class TestContextNameOverride:
|
|
def test_uses_named_context(self, monkeypatch):
|
|
# Drive context_name path: cfg.context["lake"] returns _Ctx("local").
|
|
from conf import cfg
|
|
|
|
monkeypatch.setattr(
|
|
type(cfg), "context", {"lake": _Ctx("local")}, raising=False
|
|
)
|
|
fs = get_filesystem(context_name="lake")
|
|
assert fs is not None
|