fix obstore S3: use constructor with allow_http, not from_url
Some checks failed
ci/woodpecker/push/infra-ci Pipeline was successful
ci/woodpecker/push/deploy Pipeline failed
ci/woodpecker/push/ci Pipeline failed

obstore's from_url requires aws_-prefixed keys and doesn't support
allow_http. Direct constructor with unprefixed config works for
HTTP endpoints (RustFS/MinIO). 36 objects verified in container.
This commit is contained in:
kert
2026-03-21 21:12:16 -04:00
parent 013fad2ae9
commit 9c71794ee4
2 changed files with 22 additions and 18 deletions

View File

@@ -131,12 +131,15 @@ def obstore(*, bucket: str = "lakehouse") -> Any:
endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint) endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint)
access_key = os.environ.get("RUSTFS_ACCESS_KEY", "") access_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
secret_key = os.environ.get("RUSTFS_SECRET_KEY", "") secret_key = os.environ.get("RUSTFS_SECRET_KEY", "")
return S3Store.from_url( return S3Store(
f"s3://{bucket}", bucket,
endpoint_url=endpoint, config={
access_key_id=access_key, "endpoint": endpoint,
secret_access_key=secret_key, "access_key_id": access_key,
region="us-east-1", "secret_access_key": secret_key,
"region": "us-east-1",
"allow_http": "true",
},
) )

View File

@@ -148,15 +148,18 @@ class TestObstore:
}, },
): ):
with patch( with patch(
"obstore.store.S3Store.from_url", return_value=mock_store "obstore.store.S3Store", return_value=mock_store
) as mock_from_url: ) as mock_cls:
result = connect.obstore(bucket="mybucket") result = connect.obstore(bucket="mybucket")
mock_from_url.assert_called_once_with( mock_cls.assert_called_once_with(
"s3://mybucket", "mybucket",
endpoint_url="http://test:9000", config={
access_key_id="ak", "endpoint": "http://test:9000",
secret_access_key="sk", "access_key_id": "ak",
region="us-east-1", "secret_access_key": "sk",
"region": "us-east-1",
"allow_http": "true",
},
) )
assert result is mock_store assert result is mock_store
@@ -164,11 +167,9 @@ class TestObstore:
mock_store = MagicMock() mock_store = MagicMock()
with patch("conf.connect.cfg") as mock_cfg: with patch("conf.connect.cfg") as mock_cfg:
mock_cfg.s3.endpoint = "http://rustfs:9000" mock_cfg.s3.endpoint = "http://rustfs:9000"
with patch( with patch("obstore.store.S3Store", return_value=mock_store) as mock_cls:
"obstore.store.S3Store.from_url", return_value=mock_store
) as mock_from_url:
connect.obstore() connect.obstore()
assert mock_from_url.call_args.args[0] == "s3://lakehouse" assert mock_cls.call_args.args[0] == "lakehouse"
class TestTheme: class TestTheme: