mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok for storage integrations (#4986)
Adds `exist_ok=False` to sync and async Python storage integration
creation. `exist_ok=True` sends `CREATE STORAGE INTEGRATION IF NOT
EXISTS` and returns the server's stored configuration, preserving the
prefix, credentials and external ID.
```python
integration = db.create_storage_integration(
"raw", storage_provider="S3", storage_prefix="s3://customer-bucket/raw/",
aws_role_arn="arn:aws:iam::123456789012:role/access", exist_ok=True,
)
```
The non-exhaustive Rust request also supports `.exist_ok(true)` for
every provider. Requires a SQL server supporting this syntax. Includes
protocol and provider SQL coverage. No local builds or tests were run,
as requested.
This commit is contained in:
@@ -432,8 +432,12 @@ class DBConnection(EnforceOverrides):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
"""Register storage access, preserving existing configuration with
|
||||
``exist_ok=True``. Existing names are errors by default.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def describe_storage_integration(
|
||||
@@ -2503,6 +2507,7 @@ class LanceDBConnection(DBConnection):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
return LOOP.run(
|
||||
@@ -2520,6 +2525,7 @@ class LanceDBConnection(DBConnection):
|
||||
azure_client_secret_secret=azure_client_secret_secret,
|
||||
gcp_credentials_json_secret=gcp_credentials_json_secret,
|
||||
huggingface_token_secret=huggingface_token_secret,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
@@ -3888,6 +3894,7 @@ class AsyncConnection(object):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
"""Register cloud storage access using database-owned credential Secrets.
|
||||
@@ -3902,6 +3909,10 @@ class AsyncConnection(object):
|
||||
optionally set ``huggingface_token_secret`` to a Secret containing the
|
||||
token; omitting it selects anonymous access at the default rate limits.
|
||||
Registration returns setup instructions without verifying storage access.
|
||||
|
||||
``exist_ok=True`` returns the stored configuration unchanged when the
|
||||
integration exists, preserving its provider, prefix and credential
|
||||
bindings. Existing names are errors by default.
|
||||
"""
|
||||
azure_options = (
|
||||
azure_tenant_id_secret,
|
||||
@@ -4010,6 +4021,8 @@ class AsyncConnection(object):
|
||||
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
|
||||
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
|
||||
}
|
||||
if exist_ok:
|
||||
request["exist_ok"] = True
|
||||
info = await self._inner.create_storage_integration(
|
||||
validate_secret_name(name),
|
||||
json.dumps(request),
|
||||
|
||||
@@ -1091,6 +1091,7 @@ class RemoteDBConnection(DBConnection):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
return LOOP.run(
|
||||
@@ -1108,6 +1109,7 @@ class RemoteDBConnection(DBConnection):
|
||||
azure_client_secret_secret=azure_client_secret_secret,
|
||||
gcp_credentials_json_secret=gcp_credentials_json_secret,
|
||||
huggingface_token_secret=huggingface_token_secret,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("exist_ok", "existing"),
|
||||
[(None, False), (False, False), (True, False), (True, True)],
|
||||
)
|
||||
async def test_storage_integration_exist_ok_preserves_server_metadata(
|
||||
asynchronous, exist_ok, existing
|
||||
):
|
||||
flight = pytest.importorskip("pyarrow.flight")
|
||||
fixture = (
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/storage_integration.json"
|
||||
)
|
||||
stored = json.loads(fixture.read_text())
|
||||
stored["namespace_path"] = ["team"]
|
||||
columns = {}
|
||||
for key, value in stored.items():
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
columns[key] = pa.array([None], type=pa.string()) if value is None else [value]
|
||||
result = pa.table(columns)
|
||||
prefix = "s3://replacement-bucket/raw/" if existing else stored["storage_prefix"]
|
||||
role = (
|
||||
"arn:aws:iam::999999999999:role/replacement"
|
||||
if existing
|
||||
else stored["aws_role_arn"]
|
||||
)
|
||||
|
||||
class IntegrationServer(flight.FlightServerBase):
|
||||
def __init__(self):
|
||||
super().__init__("grpc://127.0.0.1:0")
|
||||
self.requests = []
|
||||
|
||||
def do_get(self, context, ticket):
|
||||
self.requests.append(ticket.ticket)
|
||||
return flight.RecordBatchStream(result)
|
||||
|
||||
with IntegrationServer() as server:
|
||||
connection_options = {
|
||||
"api_key": "test-key",
|
||||
"host_override": "http://localhost:10024",
|
||||
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
|
||||
}
|
||||
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
|
||||
kwargs.update(
|
||||
storage_provider="S3",
|
||||
storage_prefix=prefix,
|
||||
aws_role_arn=role,
|
||||
namespace_path=["team"],
|
||||
)
|
||||
uri = f"db://{stored['database']}"
|
||||
if asynchronous:
|
||||
db = await lancedb.connect_async(uri, **connection_options)
|
||||
info = await db.create_storage_integration(stored["name"], **kwargs)
|
||||
else:
|
||||
db = lancedb.connect(uri, **connection_options)
|
||||
info = db.create_storage_integration(stored["name"], **kwargs)
|
||||
(request,) = server.requests
|
||||
|
||||
command = (
|
||||
"CREATE STORAGE INTEGRATION IF NOT EXISTS"
|
||||
if exist_ok
|
||||
else "CREATE STORAGE INTEGRATION"
|
||||
)
|
||||
assert f'{command} "raw"'.encode() in request
|
||||
assert prefix.encode() in request
|
||||
assert role.encode() in request
|
||||
assert info.storage_prefix == stored["storage_prefix"]
|
||||
assert info.aws_role_arn == stored["aws_role_arn"]
|
||||
assert info.aws_external_id == stored["aws_external_id"]
|
||||
assert info.created_at_millis == stored["created_at_millis"]
|
||||
assert info.namespace_path == ["team"]
|
||||
Reference in New Issue
Block a user