mirror of
https://github.com/lancedb/lancedb.git
synced 2026-11-16 07:36:50 +00:00
feat(python): support exist_ok when cloning tables (#4990)
Adds `exist_ok=False` to Python table cloning and `.exist_ok(true)` to
the Rust clone builder. An existing target is returned with its data and
version history unchanged; strict duplicate errors remain the default.
```python
snapshot = db.clone_table("snapshot", source_uri, exist_ok=True)
```
Embedded cloning skips source reads for an existing target and handles
typed creation conflicts. Remote cloning forwards the option to the
server. Includes sync/async preservation and remote request coverage,
plus SQL syntax documentation for `CREATE TABLE IF NOT EXISTS target
CLONE source`. Requires a server supporting the option for remote
clones.
BREAKING CHANGE: Rust `CloneTableRequest` struct literals must
initialize the new `exist_ok` field; `CloneTableRequest::new` remains
compatible.
No local builds or tests were run, as requested.
This commit is contained in:
@@ -270,6 +270,18 @@ Renaming is not supported on a local directory connection. In LanceDB Enterprise
|
||||
whether a table can be renamed or cloned at all depends on how the deployment
|
||||
stores its catalog.
|
||||
|
||||
{/* Draft for the table-clone exist_ok release.
|
||||
|
||||
Pass `exist_ok=True` to keep an existing target table, including its data and
|
||||
version history, instead of reporting a duplicate. The source is used only when
|
||||
the target is missing. Remote use requires a deployment that supports this option.
|
||||
|
||||
```python
|
||||
db.clone_table("docs_snapshot", source_uri="s3://bucket/db/documents.lance", exist_ok=True)
|
||||
```
|
||||
|
||||
*/}
|
||||
|
||||
A **shallow** clone (the default) references the source's data files rather
|
||||
than copying them, so it is fast and cheap regardless of table size. It is the
|
||||
practical way to branch an experiment off a known version, or to pin a training
|
||||
|
||||
@@ -205,6 +205,16 @@ Dropping, renaming, or retyping a column is an SDK operation
|
||||
|
||||
## CLONE {#clone}
|
||||
|
||||
{/* Draft for the table-clone exist_ok release.
|
||||
|
||||
Use `CREATE TABLE IF NOT EXISTS target CLONE source` to preserve an existing
|
||||
target unchanged. The Python equivalent is
|
||||
`db.clone_table("target", source_uri, exist_ok=True)`. Omitting the option reports
|
||||
an error for an existing target. A skipped clone does not copy rows or change
|
||||
the target's version history.
|
||||
|
||||
*/}
|
||||
|
||||
<CodeGroup>
|
||||
```sql SQL icon="database"
|
||||
CREATE TABLE docs_snapshot CLONE documents
|
||||
|
||||
@@ -347,6 +347,8 @@ class Connection(object):
|
||||
source_version: Optional[int] = None,
|
||||
source_tag: Optional[str] = None,
|
||||
is_shallow: bool = True,
|
||||
*,
|
||||
exist_ok: bool = False,
|
||||
) -> Table: ...
|
||||
async def rename_table(
|
||||
self,
|
||||
|
||||
@@ -2271,6 +2271,7 @@ class LanceDBConnection(DBConnection):
|
||||
source_version: Optional[int] = None,
|
||||
source_tag: Optional[str] = None,
|
||||
is_shallow: bool = True,
|
||||
exist_ok: bool = False,
|
||||
) -> LanceTable:
|
||||
"""Clone a table from a source table.
|
||||
|
||||
@@ -2295,6 +2296,8 @@ class LanceDBConnection(DBConnection):
|
||||
is_shallow: bool, default True
|
||||
Whether to perform a shallow clone (True) or deep clone (False).
|
||||
Currently only shallow clone is supported.
|
||||
exist_ok: bool, default False
|
||||
Return the existing target unchanged instead of cloning over it.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -2310,6 +2313,7 @@ class LanceDBConnection(DBConnection):
|
||||
source_version=source_version,
|
||||
source_tag=source_tag,
|
||||
is_shallow=is_shallow,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
)
|
||||
return LanceTable.open(
|
||||
@@ -3595,6 +3599,7 @@ class AsyncConnection(object):
|
||||
source_version: Optional[int] = None,
|
||||
source_tag: Optional[str] = None,
|
||||
is_shallow: bool = True,
|
||||
exist_ok: bool = False,
|
||||
) -> AsyncTable:
|
||||
"""Clone a table from a source table.
|
||||
|
||||
@@ -3619,6 +3624,8 @@ class AsyncConnection(object):
|
||||
is_shallow: bool, default True
|
||||
Whether to perform a shallow clone (True) or deep clone (False).
|
||||
Currently only shallow clone is supported.
|
||||
exist_ok: bool, default False
|
||||
Return the existing target unchanged instead of cloning over it.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -3633,6 +3640,7 @@ class AsyncConnection(object):
|
||||
source_version=source_version,
|
||||
source_tag=source_tag,
|
||||
is_shallow=is_shallow,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
return AsyncTable(table)
|
||||
|
||||
|
||||
@@ -576,6 +576,7 @@ class RemoteDBConnection(DBConnection):
|
||||
source_version: Optional[int] = None,
|
||||
source_tag: Optional[str] = None,
|
||||
is_shallow: bool = True,
|
||||
exist_ok: bool = False,
|
||||
) -> Table:
|
||||
"""Clone a table from a source table.
|
||||
|
||||
@@ -595,6 +596,8 @@ class RemoteDBConnection(DBConnection):
|
||||
is_shallow: bool, default True
|
||||
Whether to perform a shallow clone (True) or deep clone (False).
|
||||
Currently only shallow clone is supported.
|
||||
exist_ok: bool, default False
|
||||
Return the existing target unchanged instead of cloning over it.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -612,6 +615,7 @@ class RemoteDBConnection(DBConnection):
|
||||
source_version=source_version,
|
||||
source_tag=source_tag,
|
||||
is_shallow=is_shallow,
|
||||
exist_ok=exist_ok,
|
||||
)
|
||||
)
|
||||
return RemoteTable(
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
||||
|
||||
import pytest
|
||||
|
||||
import lancedb
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
async def test_clone_exist_ok_preserves_target(tmp_path, asynchronous):
|
||||
source_uri = str(tmp_path / "source.lance")
|
||||
if asynchronous:
|
||||
db = await lancedb.connect_async(tmp_path)
|
||||
await db.create_table("source", [{"id": 1}])
|
||||
table = await db.clone_table("target", source_uri, exist_ok=True)
|
||||
assert (await table.to_arrow()).to_pylist() == [{"id": 1}]
|
||||
await table.add([{"id": 2}])
|
||||
version = await table.version()
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
await db.clone_table("target", source_uri)
|
||||
kept = await db.clone_table("target", "missing.lance", exist_ok=True)
|
||||
assert await kept.version() == version
|
||||
assert (await kept.to_arrow()).to_pylist() == [{"id": 1}, {"id": 2}]
|
||||
else:
|
||||
db = lancedb.connect(tmp_path)
|
||||
db.create_table("source", [{"id": 1}])
|
||||
table = db.clone_table("target", source_uri, exist_ok=True)
|
||||
assert table.to_arrow().to_pylist() == [{"id": 1}]
|
||||
table.add([{"id": 2}])
|
||||
version = table.version
|
||||
with pytest.raises(ValueError, match="already exists"):
|
||||
db.clone_table("target", source_uri)
|
||||
kept = db.clone_table("target", "missing.lance", exist_ok=True)
|
||||
assert kept.version == version
|
||||
assert kept.to_arrow().to_pylist() == [{"id": 1}, {"id": 2}]
|
||||
@@ -457,7 +457,8 @@ impl Connection {
|
||||
})
|
||||
}
|
||||
|
||||
#[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true))]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true, *, exist_ok=false))]
|
||||
pub fn clone_table(
|
||||
self_: PyRef<'_, Self>,
|
||||
target_table_name: String,
|
||||
@@ -466,6 +467,7 @@ impl Connection {
|
||||
source_version: Option<u64>,
|
||||
source_tag: Option<String>,
|
||||
is_shallow: bool,
|
||||
exist_ok: bool,
|
||||
) -> PyResult<Bound<'_, PyAny>> {
|
||||
let inner = self_.get_inner()?.clone();
|
||||
|
||||
@@ -477,7 +479,7 @@ impl Connection {
|
||||
if let Some(tag) = source_tag {
|
||||
builder = builder.source_tag(tag);
|
||||
}
|
||||
builder = builder.is_shallow(is_shallow);
|
||||
builder = builder.is_shallow(is_shallow).exist_ok(exist_ok);
|
||||
|
||||
future_into_py(self_.py(), async move {
|
||||
let table = builder.execute().await.infer_error()?;
|
||||
|
||||
@@ -433,6 +433,21 @@ impl CloneTableBuilder {
|
||||
}
|
||||
}
|
||||
|
||||
/// Preserve an existing target without reading or cloning the source.
|
||||
/// Defaults to false, which reports a duplicate-name error.
|
||||
///
|
||||
/// ```no_run
|
||||
/// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> {
|
||||
/// let table = db.clone_table("snapshot", "s3://bucket/source.lance")
|
||||
/// .exist_ok(true).execute().await?;
|
||||
/// # Ok(())
|
||||
/// # }
|
||||
/// ```
|
||||
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
|
||||
self.request.exist_ok = exist_ok;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the source version to clone from
|
||||
pub fn source_version(mut self, version: u64) -> Self {
|
||||
self.request.source_version = Some(version);
|
||||
@@ -633,7 +648,8 @@ impl Connection {
|
||||
/// Creates a new table by cloning from an existing source table.
|
||||
/// By default, this performs a shallow clone where the new table shares
|
||||
/// the underlying data files with the source table.
|
||||
/// The target table name must be unused. An existing table is left unchanged.
|
||||
/// The target table name must be unused by default. Use
|
||||
/// [`CloneTableBuilder::exist_ok`] to open an existing target unchanged.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `target_table_name`: The name of the new table to create
|
||||
@@ -2787,6 +2803,14 @@ mod tests {
|
||||
"unexpected clone error: {err:?}"
|
||||
);
|
||||
|
||||
let preserved = db
|
||||
.clone_table(target, "missing-source.lance")
|
||||
.exist_ok(true)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(preserved.name(), target);
|
||||
|
||||
// Reopen both tables to verify that the failed clone left their data and versions intact.
|
||||
for (name, expected_data) in [("source", source_data), ("target", target_data)] {
|
||||
let table = db.open_table(name).execute().await.unwrap();
|
||||
@@ -2804,8 +2828,9 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_clone_table() {
|
||||
async fn test_clone_table(#[values(false, true)] exist_ok: bool) {
|
||||
let tmp_dir = tempdir().unwrap();
|
||||
let uri = tmp_dir.path().to_str().unwrap();
|
||||
let db = connect(uri).execute().await.unwrap();
|
||||
@@ -2830,6 +2855,7 @@ mod tests {
|
||||
// Clone the table
|
||||
let cloned_table = db
|
||||
.clone_table("cloned_table", source_uri)
|
||||
.exist_ok(exist_ok)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
@@ -163,6 +163,8 @@ impl CreateTableRequest {
|
||||
/// sharing the same data, deletion, and index files.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CloneTableRequest {
|
||||
/// Preserve an existing target rather than returning a duplicate-name error.
|
||||
pub exist_ok: bool,
|
||||
/// The name of the target table to create
|
||||
pub target_table_name: String,
|
||||
/// The namespace path for the target table. Empty list represents root namespace.
|
||||
@@ -184,6 +186,7 @@ pub struct CloneTableRequest {
|
||||
impl CloneTableRequest {
|
||||
pub fn new(target_table_name: String, source_uri: String) -> Self {
|
||||
Self {
|
||||
exist_ok: false,
|
||||
target_table_name,
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
|
||||
@@ -1119,40 +1119,47 @@ impl Database for ListingDatabase {
|
||||
};
|
||||
|
||||
let target_uri = self.table_uri(&request.target_table_name)?;
|
||||
match DatasetBuilder::from_uri(&target_uri)
|
||||
let exists = match DatasetBuilder::from_uri(&target_uri)
|
||||
.with_read_params(read_params.clone())
|
||||
.load()
|
||||
.await
|
||||
{
|
||||
Ok(_) => {
|
||||
Ok(_) if !request.exist_ok => {
|
||||
return Err(Error::TableAlreadyExists {
|
||||
name: request.target_table_name,
|
||||
});
|
||||
}
|
||||
Err(lance::Error::DatasetNotFound { .. }) => {}
|
||||
Ok(_) => true,
|
||||
Err(lance::Error::DatasetNotFound { .. }) => false,
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
|
||||
if !exists {
|
||||
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
|
||||
.with_read_params(read_params.clone())
|
||||
.load()
|
||||
.await
|
||||
.map_err(|e| -> Error { e.into() })?;
|
||||
|
||||
let version_ref = match (request.source_version, request.source_tag) {
|
||||
(Some(v), None) => Ok(Ref::Version(None, Some(v))),
|
||||
(None, Some(tag)) => Ok(Ref::Tag(tag)),
|
||||
(None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))),
|
||||
_ => Err(Error::InvalidInput {
|
||||
message: "Cannot specify both source_version and source_tag".to_string(),
|
||||
}),
|
||||
}?;
|
||||
|
||||
match source_dataset
|
||||
.shallow_clone(&target_uri, version_ref, Some(storage_params))
|
||||
.await
|
||||
{
|
||||
Ok(_) => {}
|
||||
Err(lance::Error::DatasetAlreadyExists { .. }) if request.exist_ok => {}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
|
||||
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
|
||||
.with_read_params(read_params.clone())
|
||||
.load()
|
||||
.await
|
||||
.map_err(|e| -> Error { e.into() })?;
|
||||
|
||||
let version_ref = match (request.source_version, request.source_tag) {
|
||||
(Some(v), None) => Ok(Ref::Version(None, Some(v))),
|
||||
(None, Some(tag)) => Ok(Ref::Tag(tag)),
|
||||
(None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))),
|
||||
_ => Err(Error::InvalidInput {
|
||||
message: "Cannot specify both source_version and source_tag".to_string(),
|
||||
}),
|
||||
}?;
|
||||
|
||||
source_dataset
|
||||
.shallow_clone(&target_uri, version_ref, Some(storage_params))
|
||||
.await
|
||||
.map_err(|e| -> Error { e.into() })?;
|
||||
|
||||
let cloned_table = NativeTable::open_with_params(
|
||||
&target_uri,
|
||||
&request.target_table_name,
|
||||
@@ -2252,6 +2259,7 @@ mod tests {
|
||||
// Clone the table
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned_table".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri: source_uri.clone(),
|
||||
@@ -2313,6 +2321,7 @@ mod tests {
|
||||
// Clone the table
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned_with_data".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2375,6 +2384,7 @@ mod tests {
|
||||
// Clone should work with storage options
|
||||
let cloned = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2412,6 +2422,7 @@ mod tests {
|
||||
// Try deep clone (should fail)
|
||||
let result = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2453,6 +2464,7 @@ mod tests {
|
||||
// Try clone with namespace (should fail for listing database)
|
||||
let result = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned".to_string(),
|
||||
target_namespace_path: vec!["namespace".to_string()], // Non-empty namespace
|
||||
source_uri,
|
||||
@@ -2494,6 +2506,7 @@ mod tests {
|
||||
// Try clone with invalid target name
|
||||
let result = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "invalid/name".to_string(), // Invalid name with slash
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2514,6 +2527,7 @@ mod tests {
|
||||
// Try to clone from non-existent source
|
||||
let result = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri: "/nonexistent/table.lance".to_string(),
|
||||
@@ -2551,6 +2565,7 @@ mod tests {
|
||||
// Try clone with both version and tag (should fail)
|
||||
let result = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2625,6 +2640,7 @@ mod tests {
|
||||
// Clone from the initial version (should have only 2 rows)
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned_from_version".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2704,6 +2720,7 @@ mod tests {
|
||||
// Clone from the tag (should have only 2 rows)
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned_from_tag".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2756,6 +2773,7 @@ mod tests {
|
||||
// Clone the table
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "independent_clone".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
@@ -2849,6 +2867,7 @@ mod tests {
|
||||
// Clone without specifying version or tag (should get latest)
|
||||
let cloned_table = db
|
||||
.clone_table(CloneTableRequest {
|
||||
exist_ok: false,
|
||||
target_table_name: "cloned_latest".to_string(),
|
||||
target_namespace_path: vec![],
|
||||
source_uri,
|
||||
|
||||
@@ -1612,10 +1612,13 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
|
||||
is_shallow: Some(request.is_shallow),
|
||||
};
|
||||
|
||||
let req = self
|
||||
let mut req = self
|
||||
.client
|
||||
.post(&format!("/v1/table/{}/clone", table_identifier.clone()))
|
||||
.json(&remote_request);
|
||||
if request.exist_ok {
|
||||
req = req.query(&[("exist_ok", true)]);
|
||||
}
|
||||
|
||||
let (request_id, rsp) = self.client.send(req).await?;
|
||||
|
||||
@@ -3651,9 +3654,10 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[tokio::test]
|
||||
async fn test_clone_table() {
|
||||
let conn = Connection::new_with_handler(|request| {
|
||||
async fn test_clone_table(#[values(false, true)] exist_ok: bool) {
|
||||
let conn = Connection::new_with_handler(move |request| {
|
||||
assert_eq!(request.method(), &reqwest::Method::POST);
|
||||
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
|
||||
assert_eq!(
|
||||
@@ -3665,12 +3669,18 @@ mod tests {
|
||||
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
|
||||
assert_eq!(body["source_location"], "s3://bucket/source_table");
|
||||
assert_eq!(body["is_shallow"], true);
|
||||
if exist_ok {
|
||||
assert_eq!(request.url().query(), Some("exist_ok=true"));
|
||||
} else {
|
||||
assert!(request.url().query().is_none());
|
||||
}
|
||||
|
||||
http::Response::builder().status(200).body("").unwrap()
|
||||
});
|
||||
|
||||
let table = conn
|
||||
.clone_table("cloned_table", "s3://bucket/source_table")
|
||||
.exist_ok(exist_ok)
|
||||
.execute()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
Reference in New Issue
Block a user