fix obstore S3: use constructor with allow_http, not from_url
obstore's from_url requires aws_-prefixed keys and doesn't support allow_http. Direct constructor with unprefixed config works for HTTP endpoints (RustFS/MinIO). 36 objects verified in container.
This commit is contained in:
@@ -131,12 +131,15 @@ def obstore(*, bucket: str = "lakehouse") -> Any:
|
||||
endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint)
|
||||
access_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
|
||||
secret_key = os.environ.get("RUSTFS_SECRET_KEY", "")
|
||||
return S3Store.from_url(
|
||||
f"s3://{bucket}",
|
||||
endpoint_url=endpoint,
|
||||
access_key_id=access_key,
|
||||
secret_access_key=secret_key,
|
||||
region="us-east-1",
|
||||
return S3Store(
|
||||
bucket,
|
||||
config={
|
||||
"endpoint": endpoint,
|
||||
"access_key_id": access_key,
|
||||
"secret_access_key": secret_key,
|
||||
"region": "us-east-1",
|
||||
"allow_http": "true",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -148,15 +148,18 @@ class TestObstore:
|
||||
},
|
||||
):
|
||||
with patch(
|
||||
"obstore.store.S3Store.from_url", return_value=mock_store
|
||||
) as mock_from_url:
|
||||
"obstore.store.S3Store", return_value=mock_store
|
||||
) as mock_cls:
|
||||
result = connect.obstore(bucket="mybucket")
|
||||
mock_from_url.assert_called_once_with(
|
||||
"s3://mybucket",
|
||||
endpoint_url="http://test:9000",
|
||||
access_key_id="ak",
|
||||
secret_access_key="sk",
|
||||
region="us-east-1",
|
||||
mock_cls.assert_called_once_with(
|
||||
"mybucket",
|
||||
config={
|
||||
"endpoint": "http://test:9000",
|
||||
"access_key_id": "ak",
|
||||
"secret_access_key": "sk",
|
||||
"region": "us-east-1",
|
||||
"allow_http": "true",
|
||||
},
|
||||
)
|
||||
assert result is mock_store
|
||||
|
||||
@@ -164,11 +167,9 @@ class TestObstore:
|
||||
mock_store = MagicMock()
|
||||
with patch("conf.connect.cfg") as mock_cfg:
|
||||
mock_cfg.s3.endpoint = "http://rustfs:9000"
|
||||
with patch(
|
||||
"obstore.store.S3Store.from_url", return_value=mock_store
|
||||
) as mock_from_url:
|
||||
with patch("obstore.store.S3Store", return_value=mock_store) as mock_cls:
|
||||
connect.obstore()
|
||||
assert mock_from_url.call_args.args[0] == "s3://lakehouse"
|
||||
assert mock_cls.call_args.args[0] == "lakehouse"
|
||||
|
||||
|
||||
class TestTheme:
|
||||
|
||||
Reference in New Issue
Block a user