diff --git a/docs/web/sql/ddl/storage-integration.mdx b/docs/web/sql/ddl/storage-integration.mdx index 37351f971..9444824ac 100644 --- a/docs/web/sql/ddl/storage-integration.mdx +++ b/docs/web/sql/ddl/storage-integration.mdx @@ -28,6 +28,16 @@ CREATE STORAGE INTEGRATION ``` +{/* Draft for the storage-integration exist_ok release. + +`IF NOT EXISTS` returns an existing integration unchanged, including its provider, +storage prefix, credential references and external ID. In the Python SDK, pass +`exist_ok=True` to `create_storage_integration`. The default reports an error when +the name already exists. +Returning an existing integration requires `USAGE` on that integration. + +*/} + `STORAGE_PROVIDER` and `STORAGE_PREFIX` are required for every provider. `STORAGE_PREFIX` is the URI prefix the integration covers -- a query only matches an integration whose prefix is a prefix of the URI it reads. diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 050b20727..39c9eb1d5 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -432,8 +432,12 @@ class DBConnection(EnforceOverrides): azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, huggingface_token_secret: Optional[Union[str, SecretReference]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: + """Register storage access, preserving existing configuration with + ``exist_ok=True``. Existing names are errors by default. + """ raise NotImplementedError def describe_storage_integration( @@ -2503,6 +2507,7 @@ class LanceDBConnection(DBConnection): azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, huggingface_token_secret: Optional[Union[str, SecretReference]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: return LOOP.run( @@ -2520,6 +2525,7 @@ class LanceDBConnection(DBConnection): azure_client_secret_secret=azure_client_secret_secret, gcp_credentials_json_secret=gcp_credentials_json_secret, huggingface_token_secret=huggingface_token_secret, + exist_ok=exist_ok, namespace_path=namespace_path, ) ) @@ -3888,6 +3894,7 @@ class AsyncConnection(object): azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, huggingface_token_secret: Optional[Union[str, SecretReference]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: """Register cloud storage access using database-owned credential Secrets. @@ -3902,6 +3909,10 @@ class AsyncConnection(object): optionally set ``huggingface_token_secret`` to a Secret containing the token; omitting it selects anonymous access at the default rate limits. Registration returns setup instructions without verifying storage access. + + ``exist_ok=True`` returns the stored configuration unchanged when the + integration exists, preserving its provider, prefix and credential + bindings. Existing names are errors by default. """ azure_options = ( azure_tenant_id_secret, @@ -4010,6 +4021,8 @@ class AsyncConnection(object): "aws_access_key_id_secret": reference(aws_access_key_id_secret), "aws_secret_access_key_secret": reference(aws_secret_access_key_secret), } + if exist_ok: + request["exist_ok"] = True info = await self._inner.create_storage_integration( validate_secret_name(name), json.dumps(request), diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 870b85b1e..9221c7ceb 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -1091,6 +1091,7 @@ class RemoteDBConnection(DBConnection): azure_client_secret_secret: Optional[Union[str, SecretReference]] = None, gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None, huggingface_token_secret: Optional[Union[str, SecretReference]] = None, + exist_ok: bool = False, namespace_path: Optional[List[str]] = None, ) -> StorageIntegrationInfo: return LOOP.run( @@ -1108,6 +1109,7 @@ class RemoteDBConnection(DBConnection): azure_client_secret_secret=azure_client_secret_secret, gcp_credentials_json_secret=gcp_credentials_json_secret, huggingface_token_secret=huggingface_token_secret, + exist_ok=exist_ok, namespace_path=namespace_path, ) ) diff --git a/python/python/tests/test_storage_integration_creation.py b/python/python/tests/test_storage_integration_creation.py new file mode 100644 index 000000000..13b94d741 --- /dev/null +++ b/python/python/tests/test_storage_integration_creation.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import json +from pathlib import Path + +import pyarrow as pa +import pytest + +import lancedb + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize( + ("exist_ok", "existing"), + [(None, False), (False, False), (True, False), (True, True)], +) +async def test_storage_integration_exist_ok_preserves_server_metadata( + asynchronous, exist_ok, existing +): + flight = pytest.importorskip("pyarrow.flight") + fixture = ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/storage_integration.json" + ) + stored = json.loads(fixture.read_text()) + stored["namespace_path"] = ["team"] + columns = {} + for key, value in stored.items(): + if isinstance(value, (dict, list)): + value = json.dumps(value) + columns[key] = pa.array([None], type=pa.string()) if value is None else [value] + result = pa.table(columns) + prefix = "s3://replacement-bucket/raw/" if existing else stored["storage_prefix"] + role = ( + "arn:aws:iam::999999999999:role/replacement" + if existing + else stored["aws_role_arn"] + ) + + class IntegrationServer(flight.FlightServerBase): + def __init__(self): + super().__init__("grpc://127.0.0.1:0") + self.requests = [] + + def do_get(self, context, ticket): + self.requests.append(ticket.ticket) + return flight.RecordBatchStream(result) + + with IntegrationServer() as server: + connection_options = { + "api_key": "test-key", + "host_override": "http://localhost:10024", + "sql_host_override": f"grpc://127.0.0.1:{server.port}", + } + kwargs = {} if exist_ok is None else {"exist_ok": exist_ok} + kwargs.update( + storage_provider="S3", + storage_prefix=prefix, + aws_role_arn=role, + namespace_path=["team"], + ) + uri = f"db://{stored['database']}" + if asynchronous: + db = await lancedb.connect_async(uri, **connection_options) + info = await db.create_storage_integration(stored["name"], **kwargs) + else: + db = lancedb.connect(uri, **connection_options) + info = db.create_storage_integration(stored["name"], **kwargs) + (request,) = server.requests + + command = ( + "CREATE STORAGE INTEGRATION IF NOT EXISTS" + if exist_ok + else "CREATE STORAGE INTEGRATION" + ) + assert f'{command} "raw"'.encode() in request + assert prefix.encode() in request + assert role.encode() in request + assert info.storage_prefix == stored["storage_prefix"] + assert info.aws_role_arn == stored["aws_role_arn"] + assert info.aws_external_id == stored["aws_external_id"] + assert info.created_at_millis == stored["created_at_millis"] + assert info.namespace_path == ["team"] diff --git a/rust/lancedb/src/remote/storage_integration.rs b/rust/lancedb/src/remote/storage_integration.rs index bdba4f1df..48dab4c6a 100644 --- a/rust/lancedb/src/remote/storage_integration.rs +++ b/rust/lancedb/src/remote/storage_integration.rs @@ -54,6 +54,11 @@ pub(super) fn create_sql( namespace: &[String], request: &CreateStorageIntegrationRequest, ) -> Result { + let if_not_exists = if request.exist_ok { + " IF NOT EXISTS" + } else { + "" + }; if request.storage_provider != StorageProvider::HuggingFace && request.huggingface_token_secret.is_some() { @@ -77,7 +82,7 @@ pub(super) fn create_sql( )); } let mut sql = format!( - "CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}", + "CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}", object_name(name, namespace)?, literal(&request.storage_prefix) ); @@ -115,7 +120,7 @@ pub(super) fn create_sql( .as_ref() .ok_or_else(|| invalid("GCP integrations require gcp_credentials_json_secret"))?; return Ok(format!( - "CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}", + "CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}", object_name(name, namespace)?, literal(&request.storage_prefix), secret_name(reference)? @@ -135,7 +140,7 @@ pub(super) fn create_sql( .filter(|account| !account.is_empty()) .ok_or_else(|| invalid("Azure integrations require azure_storage_account"))?; let mut sql = format!( - "CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}", + "CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}", object_name(name, namespace)?, literal(&request.storage_prefix), literal(account) @@ -169,7 +174,7 @@ pub(super) fn create_sql( return Err(invalid("S3 integrations cannot contain Azure options")); } let mut sql = format!( - "CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}", + "CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}", object_name(name, namespace)?, literal(&request.storage_prefix) ); @@ -283,6 +288,46 @@ mod tests { use super::*; + #[test] + fn create_if_not_exists_preserves_provider_options() { + let secret = SecretReference::new("credential"); + let requests = [ + CreateStorageIntegrationRequest::s3("s3://bucket/", "role"), + CreateStorageIntegrationRequest::s3_with_secrets( + "s3://bucket/", + secret.clone(), + secret.clone(), + ), + CreateStorageIntegrationRequest::azure( + "az://container/", + "account", + secret.clone(), + secret.clone(), + secret.clone(), + ), + CreateStorageIntegrationRequest::gcp("gs://bucket/", secret), + CreateStorageIntegrationRequest::huggingface("hf://datasets/acme/data/"), + ]; + for request in requests { + let legacy_json = serde_json::to_value(&request).unwrap(); + assert!(legacy_json.get("exist_ok").is_none()); + let decoded: CreateStorageIntegrationRequest = + serde_json::from_value(legacy_json).unwrap(); + assert!(!decoded.exist_ok); + let strict = create_sql("raw", &[], &request).unwrap(); + let guarded = request.exist_ok(true); + assert_eq!(serde_json::to_value(&guarded).unwrap()["exist_ok"], true); + assert_eq!( + create_sql("raw", &[], &guarded).unwrap(), + strict.replacen( + "CREATE STORAGE INTEGRATION ", + "CREATE STORAGE INTEGRATION IF NOT EXISTS ", + 1, + ) + ); + } + } + #[test] fn sql_quotes_values_and_keeps_secret_root_namespace_explicit() { let role = CreateStorageIntegrationRequest::s3("s3://bucket/it's/", "role'value"); diff --git a/rust/lancedb/src/storage_integration.rs b/rust/lancedb/src/storage_integration.rs index 50bfc0aca..587528037 100644 --- a/rust/lancedb/src/storage_integration.rs +++ b/rust/lancedb/src/storage_integration.rs @@ -40,6 +40,9 @@ impl SecretReference { #[derive(Debug, Clone, Serialize, Deserialize)] #[non_exhaustive] pub struct CreateStorageIntegrationRequest { + /// Return the stored configuration unchanged when the integration already exists. + #[serde(default, skip_serializing_if = "is_false")] + pub exist_ok: bool, pub storage_provider: StorageProvider, pub storage_prefix: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -77,12 +80,31 @@ pub enum StorageProvider { HuggingFace, } +fn is_false(value: &bool) -> bool { + !*value +} + impl CreateStorageIntegrationRequest { + /// Return an existing integration without changing its configuration. + /// Defaults to false, which reports an error for an existing name. + /// + /// ``` + /// use lancedb::storage_integration::CreateStorageIntegrationRequest; + /// let config = CreateStorageIntegrationRequest::huggingface( + /// "hf://datasets/acme/data/", + /// ).exist_ok(true); + /// ``` + pub fn exist_ok(mut self, exist_ok: bool) -> Self { + self.exist_ok = exist_ok; + self + } + /// Register a prefix and the customer-owned IAM role authorized to access it. pub fn s3(storage_prefix: impl Into, aws_role_arn: impl Into) -> Self { Self { storage_provider: StorageProvider::S3, storage_prefix: storage_prefix.into(), + exist_ok: false, aws_role_arn: Some(aws_role_arn.into()), aws_access_key_id_secret: None, aws_secret_access_key_secret: None, @@ -105,6 +127,7 @@ impl CreateStorageIntegrationRequest { Self { storage_provider: StorageProvider::S3, storage_prefix: storage_prefix.into(), + exist_ok: false, aws_role_arn: None, aws_access_key_id_secret: Some(access_key), aws_secret_access_key_secret: Some(secret_key), @@ -138,6 +161,7 @@ impl CreateStorageIntegrationRequest { Self { storage_provider: StorageProvider::Azure, storage_prefix: storage_prefix.into(), + exist_ok: false, azure_storage_account: Some(storage_account.into()), azure_tenant_id_secret: Some(tenant_id), azure_client_id_secret: Some(client_id), @@ -163,6 +187,7 @@ impl CreateStorageIntegrationRequest { Self { storage_provider: StorageProvider::Gcp, storage_prefix: storage_prefix.into(), + exist_ok: false, gcp_credentials_json_secret: Some(credentials_json), huggingface_token_secret: None, aws_role_arn: None, @@ -189,6 +214,7 @@ impl CreateStorageIntegrationRequest { Self { storage_provider: StorageProvider::HuggingFace, storage_prefix: storage_prefix.into(), + exist_ok: false, huggingface_token_secret: None, gcp_credentials_json_secret: None, aws_role_arn: None,