feat(python): support exist_ok for storage integrations (#4986)

Adds `exist_ok=False` to sync and async Python storage integration
creation. `exist_ok=True` sends `CREATE STORAGE INTEGRATION IF NOT
EXISTS` and returns the server's stored configuration, preserving the
prefix, credentials and external ID.

```python
integration = db.create_storage_integration(
    "raw", storage_provider="S3", storage_prefix="s3://customer-bucket/raw/",
    aws_role_arn="arn:aws:iam::123456789012:role/access", exist_ok=True,
)
```

The non-exhaustive Rust request also supports `.exist_ok(true)` for
every provider. Requires a SQL server supporting this syntax. Includes
protocol and provider SQL coverage. No local builds or tests were run,
as requested.
This commit is contained in:
Jack Ye
2026-11-14 10:37:08 +08:00
committed by GitHub
parent 2c3c30dbdf
commit a0eb6b4b71
6 changed files with 185 additions and 4 deletions
+13
View File
@@ -432,8 +432,12 @@ class DBConnection(EnforceOverrides):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register storage access, preserving existing configuration with
``exist_ok=True``. Existing names are errors by default.
"""
raise NotImplementedError
def describe_storage_integration(
@@ -2503,6 +2507,7 @@ class LanceDBConnection(DBConnection):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -2520,6 +2525,7 @@ class LanceDBConnection(DBConnection):
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
huggingface_token_secret=huggingface_token_secret,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
@@ -3888,6 +3894,7 @@ class AsyncConnection(object):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register cloud storage access using database-owned credential Secrets.
@@ -3902,6 +3909,10 @@ class AsyncConnection(object):
optionally set ``huggingface_token_secret`` to a Secret containing the
token; omitting it selects anonymous access at the default rate limits.
Registration returns setup instructions without verifying storage access.
``exist_ok=True`` returns the stored configuration unchanged when the
integration exists, preserving its provider, prefix and credential
bindings. Existing names are errors by default.
"""
azure_options = (
azure_tenant_id_secret,
@@ -4010,6 +4021,8 @@ class AsyncConnection(object):
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
}
if exist_ok:
request["exist_ok"] = True
info = await self._inner.create_storage_integration(
validate_secret_name(name),
json.dumps(request),
+2
View File
@@ -1091,6 +1091,7 @@ class RemoteDBConnection(DBConnection):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -1108,6 +1109,7 @@ class RemoteDBConnection(DBConnection):
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
huggingface_token_secret=huggingface_token_secret,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
from pathlib import Path
import pyarrow as pa
import pytest
import lancedb
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
("exist_ok", "existing"),
[(None, False), (False, False), (True, False), (True, True)],
)
async def test_storage_integration_exist_ok_preserves_server_metadata(
asynchronous, exist_ok, existing
):
flight = pytest.importorskip("pyarrow.flight")
fixture = (
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/storage_integration.json"
)
stored = json.loads(fixture.read_text())
stored["namespace_path"] = ["team"]
columns = {}
for key, value in stored.items():
if isinstance(value, (dict, list)):
value = json.dumps(value)
columns[key] = pa.array([None], type=pa.string()) if value is None else [value]
result = pa.table(columns)
prefix = "s3://replacement-bucket/raw/" if existing else stored["storage_prefix"]
role = (
"arn:aws:iam::999999999999:role/replacement"
if existing
else stored["aws_role_arn"]
)
class IntegrationServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.requests = []
def do_get(self, context, ticket):
self.requests.append(ticket.ticket)
return flight.RecordBatchStream(result)
with IntegrationServer() as server:
connection_options = {
"api_key": "test-key",
"host_override": "http://localhost:10024",
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
}
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
kwargs.update(
storage_provider="S3",
storage_prefix=prefix,
aws_role_arn=role,
namespace_path=["team"],
)
uri = f"db://{stored['database']}"
if asynchronous:
db = await lancedb.connect_async(uri, **connection_options)
info = await db.create_storage_integration(stored["name"], **kwargs)
else:
db = lancedb.connect(uri, **connection_options)
info = db.create_storage_integration(stored["name"], **kwargs)
(request,) = server.requests
command = (
"CREATE STORAGE INTEGRATION IF NOT EXISTS"
if exist_ok
else "CREATE STORAGE INTEGRATION"
)
assert f'{command} "raw"'.encode() in request
assert prefix.encode() in request
assert role.encode() in request
assert info.storage_prefix == stored["storage_prefix"]
assert info.aws_role_arn == stored["aws_role_arn"]
assert info.aws_external_id == stored["aws_external_id"]
assert info.created_at_millis == stored["created_at_millis"]
assert info.namespace_path == ["team"]