fix obstore S3: use constructor with allow_http, not from_url
obstore's from_url requires aws_-prefixed keys and doesn't support allow_http. Direct constructor with unprefixed config works for HTTP endpoints (RustFS/MinIO). 36 objects verified in container.
This commit is contained in:
@@ -131,12 +131,15 @@ def obstore(*, bucket: str = "lakehouse") -> Any:
|
|||||||
endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint)
|
endpoint = os.environ.get("RUSTFS_ENDPOINT", cfg.s3.endpoint)
|
||||||
access_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
|
access_key = os.environ.get("RUSTFS_ACCESS_KEY", "")
|
||||||
secret_key = os.environ.get("RUSTFS_SECRET_KEY", "")
|
secret_key = os.environ.get("RUSTFS_SECRET_KEY", "")
|
||||||
return S3Store.from_url(
|
return S3Store(
|
||||||
f"s3://{bucket}",
|
bucket,
|
||||||
endpoint_url=endpoint,
|
config={
|
||||||
access_key_id=access_key,
|
"endpoint": endpoint,
|
||||||
secret_access_key=secret_key,
|
"access_key_id": access_key,
|
||||||
region="us-east-1",
|
"secret_access_key": secret_key,
|
||||||
|
"region": "us-east-1",
|
||||||
|
"allow_http": "true",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -148,15 +148,18 @@ class TestObstore:
|
|||||||
},
|
},
|
||||||
):
|
):
|
||||||
with patch(
|
with patch(
|
||||||
"obstore.store.S3Store.from_url", return_value=mock_store
|
"obstore.store.S3Store", return_value=mock_store
|
||||||
) as mock_from_url:
|
) as mock_cls:
|
||||||
result = connect.obstore(bucket="mybucket")
|
result = connect.obstore(bucket="mybucket")
|
||||||
mock_from_url.assert_called_once_with(
|
mock_cls.assert_called_once_with(
|
||||||
"s3://mybucket",
|
"mybucket",
|
||||||
endpoint_url="http://test:9000",
|
config={
|
||||||
access_key_id="ak",
|
"endpoint": "http://test:9000",
|
||||||
secret_access_key="sk",
|
"access_key_id": "ak",
|
||||||
region="us-east-1",
|
"secret_access_key": "sk",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"allow_http": "true",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
assert result is mock_store
|
assert result is mock_store
|
||||||
|
|
||||||
@@ -164,11 +167,9 @@ class TestObstore:
|
|||||||
mock_store = MagicMock()
|
mock_store = MagicMock()
|
||||||
with patch("conf.connect.cfg") as mock_cfg:
|
with patch("conf.connect.cfg") as mock_cfg:
|
||||||
mock_cfg.s3.endpoint = "http://rustfs:9000"
|
mock_cfg.s3.endpoint = "http://rustfs:9000"
|
||||||
with patch(
|
with patch("obstore.store.S3Store", return_value=mock_store) as mock_cls:
|
||||||
"obstore.store.S3Store.from_url", return_value=mock_store
|
|
||||||
) as mock_from_url:
|
|
||||||
connect.obstore()
|
connect.obstore()
|
||||||
assert mock_from_url.call_args.args[0] == "s3://lakehouse"
|
assert mock_cls.call_args.args[0] == "lakehouse"
|
||||||
|
|
||||||
|
|
||||||
class TestTheme:
|
class TestTheme:
|
||||||
|
|||||||
Reference in New Issue
Block a user