mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-19 17:14:00 +00:00
feat(python): support exist_ok for storage integrations (#4986)
Adds `exist_ok=False` to sync and async Python storage integration
creation. `exist_ok=True` sends `CREATE STORAGE INTEGRATION IF NOT
EXISTS` and returns the server's stored configuration, preserving the
prefix, credentials and external ID.
```python
integration = db.create_storage_integration(
"raw", storage_provider="S3", storage_prefix="s3://customer-bucket/raw/",
aws_role_arn="arn:aws:iam::123456789012:role/access", exist_ok=True,
)
```
The non-exhaustive Rust request also supports `.exist_ok(true)` for
every provider. Requires a SQL server supporting this syntax. Includes
protocol and provider SQL coverage. No local builds or tests were run,
as requested.
This commit is contained in:
@@ -28,6 +28,16 @@ CREATE STORAGE INTEGRATION <name>
|
||||
```
|
||||
</CodeGroup>
|
||||
|
||||
{/* Draft for the storage-integration exist_ok release.
|
||||
|
||||
`IF NOT EXISTS` returns an existing integration unchanged, including its provider,
|
||||
storage prefix, credential references and external ID. In the Python SDK, pass
|
||||
`exist_ok=True` to `create_storage_integration`. The default reports an error when
|
||||
the name already exists.
|
||||
Returning an existing integration requires `USAGE` on that integration.
|
||||
|
||||
*/}
|
||||
|
||||
`STORAGE_PROVIDER` and `STORAGE_PREFIX` are required for every provider.
|
||||
`STORAGE_PREFIX` is the URI prefix the integration covers -- a query only
|
||||
matches an integration whose prefix is a prefix of the URI it reads.
|
||||
|
||||
@@ -432,8 +432,12 @@ class DBConnection(EnforceOverrides):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
"""Register storage access, preserving existing configuration with
|
||||
``exist_ok=True``. Existing names are errors by default.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def describe_storage_integration(
|
||||
@@ -2503,6 +2507,7 @@ class LanceDBConnection(DBConnection):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
return LOOP.run(
|
||||
@@ -2520,6 +2525,7 @@ class LanceDBConnection(DBConnection):
|
||||
azure_client_secret_secret=azure_client_secret_secret,
|
||||
gcp_credentials_json_secret=gcp_credentials_json_secret,
|
||||
huggingface_token_secret=huggingface_token_secret,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
@@ -3888,6 +3894,7 @@ class AsyncConnection(object):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
"""Register cloud storage access using database-owned credential Secrets.
|
||||
@@ -3902,6 +3909,10 @@ class AsyncConnection(object):
|
||||
optionally set ``huggingface_token_secret`` to a Secret containing the
|
||||
token; omitting it selects anonymous access at the default rate limits.
|
||||
Registration returns setup instructions without verifying storage access.
|
||||
|
||||
``exist_ok=True`` returns the stored configuration unchanged when the
|
||||
integration exists, preserving its provider, prefix and credential
|
||||
bindings. Existing names are errors by default.
|
||||
"""
|
||||
azure_options = (
|
||||
azure_tenant_id_secret,
|
||||
@@ -4010,6 +4021,8 @@ class AsyncConnection(object):
|
||||
"aws_access_key_id_secret": reference(aws_access_key_id_secret),
|
||||
"aws_secret_access_key_secret": reference(aws_secret_access_key_secret),
|
||||
}
|
||||
if exist_ok:
|
||||
request["exist_ok"] = True
|
||||
info = await self._inner.create_storage_integration(
|
||||
validate_secret_name(name),
|
||||
json.dumps(request),
|
||||
|
||||
@@ -1091,6 +1091,7 @@ class RemoteDBConnection(DBConnection):
|
||||
azure_client_secret_secret: Optional[Union[str, SecretReference]] = None,
|
||||
gcp_credentials_json_secret: Optional[Union[str, SecretReference]] = None,
|
||||
huggingface_token_secret: Optional[Union[str, SecretReference]] = None,
|
||||
exist_ok: bool = False,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> StorageIntegrationInfo:
|
||||
return LOOP.run(
|
||||
@@ -1108,6 +1109,7 @@ class RemoteDBConnection(DBConnection):
|
||||
azure_client_secret_secret=azure_client_secret_secret,
|
||||
gcp_credentials_json_secret=gcp_credentials_json_secret,
|
||||
huggingface_token_secret=huggingface_token_secret,
|
||||
exist_ok=exist_ok,
|
||||
namespace_path=namespace_path,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pyarrow as pa
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("exist_ok", "existing"),
|
||||
[(None, False), (False, False), (True, False), (True, True)],
|
||||
)
|
||||
async def test_storage_integration_exist_ok_preserves_server_metadata(
|
||||
asynchronous, exist_ok, existing
|
||||
):
|
||||
flight = pytest.importorskip("pyarrow.flight")
|
||||
fixture = (
|
||||
Path(__file__).parents[3]
|
||||
/ "rust/lancedb/tests/fixtures/storage_integration.json"
|
||||
)
|
||||
stored = json.loads(fixture.read_text())
|
||||
stored["namespace_path"] = ["team"]
|
||||
columns = {}
|
||||
for key, value in stored.items():
|
||||
if isinstance(value, (dict, list)):
|
||||
value = json.dumps(value)
|
||||
columns[key] = pa.array([None], type=pa.string()) if value is None else [value]
|
||||
result = pa.table(columns)
|
||||
prefix = "s3://replacement-bucket/raw/" if existing else stored["storage_prefix"]
|
||||
role = (
|
||||
"arn:aws:iam::999999999999:role/replacement"
|
||||
if existing
|
||||
else stored["aws_role_arn"]
|
||||
)
|
||||
|
||||
class IntegrationServer(flight.FlightServerBase):
|
||||
def __init__(self):
|
||||
super().__init__("grpc://127.0.0.1:0")
|
||||
self.requests = []
|
||||
|
||||
def do_get(self, context, ticket):
|
||||
self.requests.append(ticket.ticket)
|
||||
return flight.RecordBatchStream(result)
|
||||
|
||||
with IntegrationServer() as server:
|
||||
connection_options = {
|
||||
"api_key": "test-key",
|
||||
"host_override": "http://localhost:10024",
|
||||
"sql_host_override": f"grpc://127.0.0.1:{server.port}",
|
||||
}
|
||||
kwargs = {} if exist_ok is None else {"exist_ok": exist_ok}
|
||||
kwargs.update(
|
||||
storage_provider="S3",
|
||||
storage_prefix=prefix,
|
||||
aws_role_arn=role,
|
||||
namespace_path=["team"],
|
||||
)
|
||||
uri = f"db://{stored['database']}"
|
||||
if asynchronous:
|
||||
db = await lancedb.connect_async(uri, **connection_options)
|
||||
info = await db.create_storage_integration(stored["name"], **kwargs)
|
||||
else:
|
||||
db = lancedb.connect(uri, **connection_options)
|
||||
info = db.create_storage_integration(stored["name"], **kwargs)
|
||||
(request,) = server.requests
|
||||
|
||||
command = (
|
||||
"CREATE STORAGE INTEGRATION IF NOT EXISTS"
|
||||
if exist_ok
|
||||
else "CREATE STORAGE INTEGRATION"
|
||||
)
|
||||
assert f'{command} "raw"'.encode() in request
|
||||
assert prefix.encode() in request
|
||||
assert role.encode() in request
|
||||
assert info.storage_prefix == stored["storage_prefix"]
|
||||
assert info.aws_role_arn == stored["aws_role_arn"]
|
||||
assert info.aws_external_id == stored["aws_external_id"]
|
||||
assert info.created_at_millis == stored["created_at_millis"]
|
||||
assert info.namespace_path == ["team"]
|
||||
@@ -54,6 +54,11 @@ pub(super) fn create_sql(
|
||||
namespace: &[String],
|
||||
request: &CreateStorageIntegrationRequest,
|
||||
) -> Result<String> {
|
||||
let if_not_exists = if request.exist_ok {
|
||||
" IF NOT EXISTS"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
if request.storage_provider != StorageProvider::HuggingFace
|
||||
&& request.huggingface_token_secret.is_some()
|
||||
{
|
||||
@@ -77,7 +82,7 @@ pub(super) fn create_sql(
|
||||
));
|
||||
}
|
||||
let mut sql = format!(
|
||||
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}",
|
||||
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'HUGGINGFACE' STORAGE_PREFIX = {}",
|
||||
object_name(name, namespace)?,
|
||||
literal(&request.storage_prefix)
|
||||
);
|
||||
@@ -115,7 +120,7 @@ pub(super) fn create_sql(
|
||||
.as_ref()
|
||||
.ok_or_else(|| invalid("GCP integrations require gcp_credentials_json_secret"))?;
|
||||
return Ok(format!(
|
||||
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}",
|
||||
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'GCP' STORAGE_PREFIX = {} GCP_CREDENTIALS_JSON_SECRET = {}",
|
||||
object_name(name, namespace)?,
|
||||
literal(&request.storage_prefix),
|
||||
secret_name(reference)?
|
||||
@@ -135,7 +140,7 @@ pub(super) fn create_sql(
|
||||
.filter(|account| !account.is_empty())
|
||||
.ok_or_else(|| invalid("Azure integrations require azure_storage_account"))?;
|
||||
let mut sql = format!(
|
||||
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}",
|
||||
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'AZURE' STORAGE_PREFIX = {} STORAGE_AZURE_STORAGE_ACCOUNT = {}",
|
||||
object_name(name, namespace)?,
|
||||
literal(&request.storage_prefix),
|
||||
literal(account)
|
||||
@@ -169,7 +174,7 @@ pub(super) fn create_sql(
|
||||
return Err(invalid("S3 integrations cannot contain Azure options"));
|
||||
}
|
||||
let mut sql = format!(
|
||||
"CREATE STORAGE INTEGRATION {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}",
|
||||
"CREATE STORAGE INTEGRATION{if_not_exists} {} STORAGE_PROVIDER = 'S3' STORAGE_PREFIX = {}",
|
||||
object_name(name, namespace)?,
|
||||
literal(&request.storage_prefix)
|
||||
);
|
||||
@@ -283,6 +288,46 @@ mod tests {
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn create_if_not_exists_preserves_provider_options() {
|
||||
let secret = SecretReference::new("credential");
|
||||
let requests = [
|
||||
CreateStorageIntegrationRequest::s3("s3://bucket/", "role"),
|
||||
CreateStorageIntegrationRequest::s3_with_secrets(
|
||||
"s3://bucket/",
|
||||
secret.clone(),
|
||||
secret.clone(),
|
||||
),
|
||||
CreateStorageIntegrationRequest::azure(
|
||||
"az://container/",
|
||||
"account",
|
||||
secret.clone(),
|
||||
secret.clone(),
|
||||
secret.clone(),
|
||||
),
|
||||
CreateStorageIntegrationRequest::gcp("gs://bucket/", secret),
|
||||
CreateStorageIntegrationRequest::huggingface("hf://datasets/acme/data/"),
|
||||
];
|
||||
for request in requests {
|
||||
let legacy_json = serde_json::to_value(&request).unwrap();
|
||||
assert!(legacy_json.get("exist_ok").is_none());
|
||||
let decoded: CreateStorageIntegrationRequest =
|
||||
serde_json::from_value(legacy_json).unwrap();
|
||||
assert!(!decoded.exist_ok);
|
||||
let strict = create_sql("raw", &[], &request).unwrap();
|
||||
let guarded = request.exist_ok(true);
|
||||
assert_eq!(serde_json::to_value(&guarded).unwrap()["exist_ok"], true);
|
||||
assert_eq!(
|
||||
create_sql("raw", &[], &guarded).unwrap(),
|
||||
strict.replacen(
|
||||
"CREATE STORAGE INTEGRATION ",
|
||||
"CREATE STORAGE INTEGRATION IF NOT EXISTS ",
|
||||
1,
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sql_quotes_values_and_keeps_secret_root_namespace_explicit() {
|
||||
let role = CreateStorageIntegrationRequest::s3("s3://bucket/it's/", "role'value");
|
||||
|
||||
@@ -40,6 +40,9 @@ impl SecretReference {
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[non_exhaustive]
|
||||
pub struct CreateStorageIntegrationRequest {
|
||||
/// Return the stored configuration unchanged when the integration already exists.
|
||||
#[serde(default, skip_serializing_if = "is_false")]
|
||||
pub exist_ok: bool,
|
||||
pub storage_provider: StorageProvider,
|
||||
pub storage_prefix: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
@@ -77,12 +80,31 @@ pub enum StorageProvider {
|
||||
HuggingFace,
|
||||
}
|
||||
|
||||
fn is_false(value: &bool) -> bool {
|
||||
!*value
|
||||
}
|
||||
|
||||
impl CreateStorageIntegrationRequest {
|
||||
/// Return an existing integration without changing its configuration.
|
||||
/// Defaults to false, which reports an error for an existing name.
|
||||
///
|
||||
/// ```
|
||||
/// use lancedb::storage_integration::CreateStorageIntegrationRequest;
|
||||
/// let config = CreateStorageIntegrationRequest::huggingface(
|
||||
/// "hf://datasets/acme/data/",
|
||||
/// ).exist_ok(true);
|
||||
/// ```
|
||||
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
|
||||
self.exist_ok = exist_ok;
|
||||
self
|
||||
}
|
||||
|
||||
/// Register a prefix and the customer-owned IAM role authorized to access it.
|
||||
pub fn s3(storage_prefix: impl Into<String>, aws_role_arn: impl Into<String>) -> Self {
|
||||
Self {
|
||||
storage_provider: StorageProvider::S3,
|
||||
storage_prefix: storage_prefix.into(),
|
||||
exist_ok: false,
|
||||
aws_role_arn: Some(aws_role_arn.into()),
|
||||
aws_access_key_id_secret: None,
|
||||
aws_secret_access_key_secret: None,
|
||||
@@ -105,6 +127,7 @@ impl CreateStorageIntegrationRequest {
|
||||
Self {
|
||||
storage_provider: StorageProvider::S3,
|
||||
storage_prefix: storage_prefix.into(),
|
||||
exist_ok: false,
|
||||
aws_role_arn: None,
|
||||
aws_access_key_id_secret: Some(access_key),
|
||||
aws_secret_access_key_secret: Some(secret_key),
|
||||
@@ -138,6 +161,7 @@ impl CreateStorageIntegrationRequest {
|
||||
Self {
|
||||
storage_provider: StorageProvider::Azure,
|
||||
storage_prefix: storage_prefix.into(),
|
||||
exist_ok: false,
|
||||
azure_storage_account: Some(storage_account.into()),
|
||||
azure_tenant_id_secret: Some(tenant_id),
|
||||
azure_client_id_secret: Some(client_id),
|
||||
@@ -163,6 +187,7 @@ impl CreateStorageIntegrationRequest {
|
||||
Self {
|
||||
storage_provider: StorageProvider::Gcp,
|
||||
storage_prefix: storage_prefix.into(),
|
||||
exist_ok: false,
|
||||
gcp_credentials_json_secret: Some(credentials_json),
|
||||
huggingface_token_secret: None,
|
||||
aws_role_arn: None,
|
||||
@@ -189,6 +214,7 @@ impl CreateStorageIntegrationRequest {
|
||||
Self {
|
||||
storage_provider: StorageProvider::HuggingFace,
|
||||
storage_prefix: storage_prefix.into(),
|
||||
exist_ok: false,
|
||||
huggingface_token_secret: None,
|
||||
gcp_credentials_json_secret: None,
|
||||
aws_role_arn: None,
|
||||
|
||||
Reference in New Issue
Block a user