mirror of
https://github.com/lancedb/lancedb.git
synced 2026-09-12 16:22:24 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b9291e4b56 | ||
|
|
5eff90bba7 |
@@ -65,6 +65,42 @@ def _check_s3_bucket_with_dots(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_s3_storage_options(
|
||||||
|
uri: str, storage_options: Optional[Dict[str, str]]
|
||||||
|
) -> Optional[Dict[str, str]]:
|
||||||
|
"""Translate Python S3 compatibility options to native storage options."""
|
||||||
|
if not isinstance(uri, str) or not uri.startswith(("s3://", "s3+ddb://")):
|
||||||
|
return storage_options
|
||||||
|
if not storage_options:
|
||||||
|
return storage_options
|
||||||
|
|
||||||
|
verify_key = next(
|
||||||
|
(key for key in storage_options if key.casefold() == "verify"), None
|
||||||
|
)
|
||||||
|
if verify_key is None:
|
||||||
|
return storage_options
|
||||||
|
|
||||||
|
normalized = dict(storage_options)
|
||||||
|
verify = normalized.pop(verify_key)
|
||||||
|
native_option_keys = {
|
||||||
|
"allow_invalid_certificates",
|
||||||
|
"aws_allow_invalid_certificates",
|
||||||
|
}
|
||||||
|
has_native_option = any(key.casefold() in native_option_keys for key in normalized)
|
||||||
|
if not has_native_option:
|
||||||
|
verify_value = verify.casefold()
|
||||||
|
if verify_value == "false":
|
||||||
|
normalized["allow_invalid_certificates"] = "true"
|
||||||
|
elif verify_value == "true":
|
||||||
|
normalized["allow_invalid_certificates"] = "false"
|
||||||
|
else:
|
||||||
|
raise ValueError(
|
||||||
|
"S3 storage option 'verify' must be 'true' or 'false'; "
|
||||||
|
"use 'allow_invalid_certificates' to configure TLS verification"
|
||||||
|
)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
def connect(
|
def connect(
|
||||||
uri: Optional[URI] = None,
|
uri: Optional[URI] = None,
|
||||||
*,
|
*,
|
||||||
@@ -120,6 +156,11 @@ def connect(
|
|||||||
storage_options: dict, optional
|
storage_options: dict, optional
|
||||||
Additional options for the storage backend. See available options at
|
Additional options for the storage backend. See available options at
|
||||||
<https://docs.lancedb.com/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
|
|
||||||
|
For S3-compatible endpoints with self-signed TLS certificates, set
|
||||||
|
``allow_invalid_certificates`` to ``"true"``. The boto3-compatible
|
||||||
|
``verify="false"`` spelling is also accepted. Disabling certificate
|
||||||
|
validation is insecure and should only be used for testing.
|
||||||
manifest_enabled : bool, default False
|
manifest_enabled : bool, default False
|
||||||
When true for local/native connections, use directory namespace
|
When true for local/native connections, use directory namespace
|
||||||
manifests as the source of truth for table metadata. Existing
|
manifests as the source of truth for table metadata. Existing
|
||||||
@@ -184,6 +225,8 @@ def connect(
|
|||||||
conn : DBConnection
|
conn : DBConnection
|
||||||
A connection to a LanceDB database.
|
A connection to a LanceDB database.
|
||||||
"""
|
"""
|
||||||
|
storage_options = _normalize_s3_storage_options(str(uri), storage_options)
|
||||||
|
|
||||||
if namespace_client_impl is not None:
|
if namespace_client_impl is not None:
|
||||||
if namespace_client_properties is None:
|
if namespace_client_properties is None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -426,6 +469,11 @@ async def connect_async(
|
|||||||
storage_options: dict, optional
|
storage_options: dict, optional
|
||||||
Additional options for the storage backend. See available options at
|
Additional options for the storage backend. See available options at
|
||||||
<https://docs.lancedb.com/storage/>
|
<https://docs.lancedb.com/storage/>
|
||||||
|
|
||||||
|
For S3-compatible endpoints with self-signed TLS certificates, set
|
||||||
|
``allow_invalid_certificates`` to ``"true"``. The boto3-compatible
|
||||||
|
``verify="false"`` spelling is also accepted. Disabling certificate
|
||||||
|
validation is insecure and should only be used for testing.
|
||||||
session: Session, optional
|
session: Session, optional
|
||||||
(For LanceDB OSS only)
|
(For LanceDB OSS only)
|
||||||
A session to use for this connection. Sessions allow you to configure
|
A session to use for this connection. Sessions allow you to configure
|
||||||
@@ -467,6 +515,8 @@ async def connect_async(
|
|||||||
conn : AsyncConnection
|
conn : AsyncConnection
|
||||||
A connection to a LanceDB database.
|
A connection to a LanceDB database.
|
||||||
"""
|
"""
|
||||||
|
storage_options = _normalize_s3_storage_options(str(uri), storage_options)
|
||||||
|
|
||||||
if read_consistency_interval is not None:
|
if read_consistency_interval is not None:
|
||||||
read_consistency_interval_secs = read_consistency_interval.total_seconds()
|
read_consistency_interval_secs = read_consistency_interval.total_seconds()
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -23,6 +23,52 @@ CONFIG = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_s3_verify_false_is_normalized(monkeypatch):
|
||||||
|
captured_options = None
|
||||||
|
|
||||||
|
async def capture_connect(*args):
|
||||||
|
nonlocal captured_options
|
||||||
|
captured_options = args[6]
|
||||||
|
return object()
|
||||||
|
|
||||||
|
monkeypatch.setattr("lancedb.db.lancedb_connect", capture_connect)
|
||||||
|
options = {"endpoint": "https://minio.example", "verify": "false"}
|
||||||
|
|
||||||
|
lancedb.connect("s3://bucket/database", storage_options=options)
|
||||||
|
|
||||||
|
assert captured_options == {
|
||||||
|
"endpoint": "https://minio.example",
|
||||||
|
"allow_invalid_certificates": "true",
|
||||||
|
}
|
||||||
|
assert options == {"endpoint": "https://minio.example", "verify": "false"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"native_key",
|
||||||
|
["allow_invalid_certificates", "aws_allow_invalid_certificates"],
|
||||||
|
)
|
||||||
|
async def test_s3_native_tls_option_takes_precedence(monkeypatch, native_key):
|
||||||
|
captured_options = None
|
||||||
|
|
||||||
|
async def capture_connect(*args):
|
||||||
|
nonlocal captured_options
|
||||||
|
captured_options = args[6]
|
||||||
|
return object()
|
||||||
|
|
||||||
|
monkeypatch.setattr(lancedb, "lancedb_connect", capture_connect)
|
||||||
|
|
||||||
|
await lancedb.connect_async(
|
||||||
|
"s3://bucket/database",
|
||||||
|
storage_options={
|
||||||
|
"verify": "false",
|
||||||
|
native_key: "false",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured_options == {native_key: "false"}
|
||||||
|
|
||||||
|
|
||||||
def get_boto3_client(*args, **kwargs):
|
def get_boto3_client(*args, **kwargs):
|
||||||
import boto3
|
import boto3
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user