From 5eff90bba74abaa6c9916170c4a66b9916cbe000 Mon Sep 17 00:00:00 2001 From: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:57:30 +0000 Subject: [PATCH] fix(python): support boto3 TLS verification option --- python/python/lancedb/__init__.py | 48 +++++++++++++++++++++++++++++++ python/python/tests/test_s3.py | 42 +++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 235049f97..35d3661d0 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -65,6 +65,40 @@ def _check_s3_bucket_with_dots( ) +def _normalize_s3_storage_options( + uri: str, storage_options: Optional[Dict[str, str]] +) -> Optional[Dict[str, str]]: + """Translate Python S3 compatibility options to native storage options.""" + if not isinstance(uri, str) or not uri.startswith(("s3://", "s3+ddb://")): + return storage_options + if not storage_options: + return storage_options + + verify_key = next( + (key for key in storage_options if key.casefold() == "verify"), None + ) + if verify_key is None: + return storage_options + + normalized = dict(storage_options) + verify = normalized.pop(verify_key) + has_native_option = any( + key.casefold() == "allow_invalid_certificates" for key in normalized + ) + if not has_native_option: + verify_value = verify.casefold() + if verify_value == "false": + normalized["allow_invalid_certificates"] = "true" + elif verify_value == "true": + normalized["allow_invalid_certificates"] = "false" + else: + raise ValueError( + "S3 storage option 'verify' must be 'true' or 'false'; " + "use 'allow_invalid_certificates' to configure TLS verification" + ) + return normalized + + def connect( uri: Optional[URI] = None, *, @@ -120,6 +154,11 @@ def connect( storage_options: dict, optional Additional options for the storage backend. See available options at + + For S3-compatible endpoints with self-signed TLS certificates, set + ``allow_invalid_certificates`` to ``"true"``. The boto3-compatible + ``verify="false"`` spelling is also accepted. Disabling certificate + validation is insecure and should only be used for testing. manifest_enabled : bool, default False When true for local/native connections, use directory namespace manifests as the source of truth for table metadata. Existing @@ -184,6 +223,8 @@ def connect( conn : DBConnection A connection to a LanceDB database. """ + storage_options = _normalize_s3_storage_options(str(uri), storage_options) + if namespace_client_impl is not None: if namespace_client_properties is None: raise ValueError( @@ -426,6 +467,11 @@ async def connect_async( storage_options: dict, optional Additional options for the storage backend. See available options at + + For S3-compatible endpoints with self-signed TLS certificates, set + ``allow_invalid_certificates`` to ``"true"``. The boto3-compatible + ``verify="false"`` spelling is also accepted. Disabling certificate + validation is insecure and should only be used for testing. session: Session, optional (For LanceDB OSS only) A session to use for this connection. Sessions allow you to configure @@ -467,6 +513,8 @@ async def connect_async( conn : AsyncConnection A connection to a LanceDB database. """ + storage_options = _normalize_s3_storage_options(str(uri), storage_options) + if read_consistency_interval is not None: read_consistency_interval_secs = read_consistency_interval.total_seconds() else: diff --git a/python/python/tests/test_s3.py b/python/python/tests/test_s3.py index 256ccb1d4..fd9b4e3bd 100644 --- a/python/python/tests/test_s3.py +++ b/python/python/tests/test_s3.py @@ -23,6 +23,48 @@ CONFIG = { } +def test_s3_verify_false_is_normalized(monkeypatch): + captured_options = None + + async def capture_connect(*args): + nonlocal captured_options + captured_options = args[6] + return object() + + monkeypatch.setattr("lancedb.db.lancedb_connect", capture_connect) + options = {"endpoint": "https://minio.example", "verify": "false"} + + lancedb.connect("s3://bucket/database", storage_options=options) + + assert captured_options == { + "endpoint": "https://minio.example", + "allow_invalid_certificates": "true", + } + assert options == {"endpoint": "https://minio.example", "verify": "false"} + + +@pytest.mark.asyncio +async def test_s3_native_tls_option_takes_precedence(monkeypatch): + captured_options = None + + async def capture_connect(*args): + nonlocal captured_options + captured_options = args[6] + return object() + + monkeypatch.setattr(lancedb, "lancedb_connect", capture_connect) + + await lancedb.connect_async( + "s3://bucket/database", + storage_options={ + "verify": "false", + "allow_invalid_certificates": "false", + }, + ) + + assert captured_options == {"allow_invalid_certificates": "false"} + + def get_boto3_client(*args, **kwargs): import boto3