feat(python): support exist_ok for storage integrations (#4986)

Adds `exist_ok=False` to sync and async Python storage integration
creation. `exist_ok=True` sends `CREATE STORAGE INTEGRATION IF NOT
EXISTS` and returns the server's stored configuration, preserving the
prefix, credentials and external ID.

```python
integration = db.create_storage_integration(
    "raw", storage_provider="S3", storage_prefix="s3://customer-bucket/raw/",
    aws_role_arn="arn:aws:iam::123456789012:role/access", exist_ok=True,
)
```

The non-exhaustive Rust request also supports `.exist_ok(true)` for
every provider. Requires a SQL server supporting this syntax. Includes
protocol and provider SQL coverage. No local builds or tests were run,
as requested.
This commit is contained in:
Jack Ye
2026-11-14 10:37:08 +08:00
committed by GitHub
parent 2c3c30dbdf
commit a0eb6b4b71
6 changed files with 185 additions and 4 deletions
+10
View File
@@ -28,6 +28,16 @@ CREATE STORAGE INTEGRATION <name>
```
</CodeGroup>
{/* Draft for the storage-integration exist_ok release.
`IF NOT EXISTS` returns an existing integration unchanged, including its provider,
storage prefix, credential references and external ID. In the Python SDK, pass
`exist_ok=True` to `create_storage_integration`. The default reports an error when
the name already exists.
Returning an existing integration requires `USAGE` on that integration.
*/}
`STORAGE_PROVIDER` and `STORAGE_PREFIX` are required for every provider.
`STORAGE_PREFIX` is the URI prefix the integration covers -- a query only
matches an integration whose prefix is a prefix of the URI it reads.
+13
View File
@@ -432,8 +432,12 @@ class DBConnection(EnforceOverrides):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register storage access, preserving existing configuration with
``exist_ok=True``. Existing names are errors by default.
"""
raise NotImplementedError
def describe_storage_integration(
@@ -2503,6 +2507,7 @@ class LanceDBConnection(DBConnection):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -2520,6 +2525,7 @@ class LanceDBConnection(DBConnection):
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
huggingface_token_secret=huggingface_token_secret,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
@@ -3888,6 +3894,7 @@ class AsyncConnection(object):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
"""Register cloud storage access using database-owned credential Secrets.
@@ -3902,6 +3909,10 @@ class AsyncConnection(object):
optionally set ``huggingface_token_secret`` to a Secret containing the
token; omitting it selects anonymous access at the default rate limits.
Registration returns setup instructions without verifying storage access.
``exist_ok=True`` returns the stored configuration unchanged when the
integration exists, preserving its provider, prefix and credential
bindings. Existing names are errors by default.
"""
azure_options = (
azure_tenant_id_secret,
@@ -4010,6 +4021,8 @@ class AsyncConnection(object):
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
}
if exist_ok:
request["exist_ok"] = True
info = await self._inner.create_storage_integration(
validate_secret_name(name),
json.dumps(request),
+2
View File
@@ -1091,6 +1091,7 @@ class RemoteDBConnection(DBConnection):
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
exist_ok: bool = False,
namespace_path: Optional[List[str]] = None,
) -> StorageIntegrationInfo:
return LOOP.run(
@@ -1108,6 +1109,7 @@ class RemoteDBConnection(DBConnection):
azure_client_secret_secret=azure_client_secret_secret,
gcp_credentials_json_secret=gcp_credentials_json_secret,
huggingface_token_secret=huggingface_token_secret,
exist_ok=exist_ok,
namespace_path=namespace_path,
)
)
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import json
from pathlib import Path
import pyarrow as pa
import pytest
import lancedb
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
("exist_ok", "existing"),
[(None, False), (False, False), (True, False), (True, True)],
)
async def test_storage_integration_exist_ok_preserves_server_metadata(
asynchronous, exist_ok, existing
):
flight = pytest.importorskip("pyarrow.flight")
fixture = (
Path(__file__).parents[3]
/ "rust/lancedb/tests/fixtures/storage_integration.json"
)
stored = json.loads(fixture.read_text())
stored["namespace_path"] = ["team"]
columns = {}
for key, value in stored.items():
if isinstance(value, (dict, list)):
value = json.dumps(value)
columns[key] = pa.array([None], type=pa.string()) if value is None else [value]
result = pa.table(columns)
prefix = "s3://replacement-bucket/raw/" if existing else stored["storage_prefix"]
role = (
"arn:aws:iam::999999999999:role/replacement"
if existing
else stored["aws_role_arn"]
)
class IntegrationServer(flight.FlightServerBase):
def __init__(self):
super().__init__("grpc://127.0.0.1:0")
self.requests = []
def do_get(self, context, ticket):
self.requests.append(ticket.ticket)
return flight.RecordBatchStream(result)
with IntegrationServer() as server:
connection_options = {
"api_key": "test-key",
"host_override": "http://localhost:10024",
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
}
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
kwargs.update(
storage_provider="S3",
storage_prefix=prefix,
aws_role_arn=role,
namespace_path=["team"],
)
uri = f"db://{stored['database']}"
if asynchronous:
db = await lancedb.connect_async(uri, **connection_options)
info = await db.create_storage_integration(stored["name"], **kwargs)
else:
db = lancedb.connect(uri, **connection_options)
info = db.create_storage_integration(stored["name"], **kwargs)
(request,) = server.requests
command = (
"CREATE STORAGE INTEGRATION IF NOT EXISTS"
if exist_ok
else "CREATE STORAGE INTEGRATION"
)
assert f'{command} "raw"'.encode() in request
assert prefix.encode() in request
assert role.encode() in request
assert info.storage_prefix == stored["storage_prefix"]
assert info.aws_role_arn == stored["aws_role_arn"]
assert info.aws_external_id == stored["aws_external_id"]
assert info.created_at_millis == stored["created_at_millis"]
assert info.namespace_path == ["team"]
+49 -4
View File
@@ -54,6 +54,11 @@ pub(super) fn create_sql(
namespace: &[String],
request: &CreateStorageIntegrationRequest,
) -> Result<String> {
let if_not_exists = if request.exist_ok {
" IF NOT EXISTS"
} else {
""
};
if request.storage_provider != StorageProvider::HuggingFace
&& request.huggingface_token_secret.is_some()
{
@@ -77,7 +82,7 @@ pub(super) fn create_sql(
));
}
let mut sql = format!(
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}",
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}",
object_name(name, namespace)?,
literal(&request.storage_prefix)
);
@@ -115,7 +120,7 @@ pub(super) fn create_sql(
.as_ref()
.ok_or_else(|| invalid("GCP integrations require gcp_credentials_json_secret"))?;
return Ok(format!(
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}",
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}",
object_name(name, namespace)?,
literal(&request.storage_prefix),
secret_name(reference)?
@@ -135,7 +140,7 @@ pub(super) fn create_sql(
.filter(|account| !account.is_empty())
.ok_or_else(|| invalid("Azure integrations require azure_storage_account"))?;
let mut sql = format!(
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}",
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}",
object_name(name, namespace)?,
literal(&request.storage_prefix),
literal(account)
@@ -169,7 +174,7 @@ pub(super) fn create_sql(
return Err(invalid("S3 integrations cannot contain Azure options"));
}
let mut sql = format!(
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}",
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}",
object_name(name, namespace)?,
literal(&request.storage_prefix)
);
@@ -283,6 +288,46 @@ mod tests {
use super::*;
#[test]
fn create_if_not_exists_preserves_provider_options() {
let secret = SecretReference::new("credential");
let requests = [
CreateStorageIntegrationRequest::s3("s3://bucket/", "role"),
CreateStorageIntegrationRequest::s3_with_secrets(
"s3://bucket/",
secret.clone(),
secret.clone(),
),
CreateStorageIntegrationRequest::azure(
"az://container/",
"account",
secret.clone(),
secret.clone(),
secret.clone(),
),
CreateStorageIntegrationRequest::gcp("gs://bucket/", secret),
CreateStorageIntegrationRequest::huggingface("hf://datasets/acme/data/"),
];
for request in requests {
let legacy_json = serde_json::to_value(&request).unwrap();
assert!(legacy_json.get("exist_ok").is_none());
let decoded: CreateStorageIntegrationRequest =
serde_json::from_value(legacy_json).unwrap();
assert!(!decoded.exist_ok);
let strict = create_sql("raw", &[], &request).unwrap();
let guarded = request.exist_ok(true);
assert_eq!(serde_json::to_value(&guarded).unwrap()["exist_ok"], true);
assert_eq!(
create_sql("raw", &[], &guarded).unwrap(),
strict.replacen(
"CREATE STORAGE INTEGRATION ",
"CREATE STORAGE INTEGRATION IF NOT EXISTS ",
1,
)
);
}
}
#[test]
fn sql_quotes_values_and_keeps_secret_root_namespace_explicit() {
let role = CreateStorageIntegrationRequest::s3("s3://bucket/it's/", "role'value");
+26
View File
@@ -40,6 +40,9 @@ impl SecretReference {
#[derive(Debug, Clone, Serialize, Deserialize)]
#[non_exhaustive]
pub struct CreateStorageIntegrationRequest {
/// Return the stored configuration unchanged when the integration already exists.
#[serde(default, skip_serializing_if = "is_false")]
pub exist_ok: bool,
pub storage_provider: StorageProvider,
pub storage_prefix: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
@@ -77,12 +80,31 @@ pub enum StorageProvider {
HuggingFace,
}
fn is_false(value: &bool) -> bool {
!*value
}
impl CreateStorageIntegrationRequest {
/// Return an existing integration without changing its configuration.
/// Defaults to false, which reports an error for an existing name.
///
/// ```
/// use lancedb::storage_integration::CreateStorageIntegrationRequest;
/// let config = CreateStorageIntegrationRequest::huggingface(
/// "hf://datasets/acme/data/",
/// ).exist_ok(true);
/// ```
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
self.exist_ok = exist_ok;
self
}
/// Register a prefix and the customer-owned IAM role authorized to access it.
pub fn s3(storage_prefix: impl Into<String>, aws_role_arn: impl Into<String>) -> Self {
Self {
storage_provider: StorageProvider::S3,
storage_prefix: storage_prefix.into(),
exist_ok: false,
aws_role_arn: Some(aws_role_arn.into()),
aws_access_key_id_secret: None,
aws_secret_access_key_secret: None,
@@ -105,6 +127,7 @@ impl CreateStorageIntegrationRequest {
Self {
storage_provider: StorageProvider::S3,
storage_prefix: storage_prefix.into(),
exist_ok: false,
aws_role_arn: None,
aws_access_key_id_secret: Some(access_key),
aws_secret_access_key_secret: Some(secret_key),
@@ -138,6 +161,7 @@ impl CreateStorageIntegrationRequest {
Self {
storage_provider: StorageProvider::Azure,
storage_prefix: storage_prefix.into(),
exist_ok: false,
azure_storage_account: Some(storage_account.into()),
azure_tenant_id_secret: Some(tenant_id),
azure_client_id_secret: Some(client_id),
@@ -163,6 +187,7 @@ impl CreateStorageIntegrationRequest {
Self {
storage_provider: StorageProvider::Gcp,
storage_prefix: storage_prefix.into(),
exist_ok: false,
gcp_credentials_json_secret: Some(credentials_json),
huggingface_token_secret: None,
aws_role_arn: None,
@@ -189,6 +214,7 @@ impl CreateStorageIntegrationRequest {
Self {
storage_provider: StorageProvider::HuggingFace,
storage_prefix: storage_prefix.into(),
exist_ok: false,
huggingface_token_secret: None,
gcp_credentials_json_secret: None,
aws_role_arn: None,