feat(python): support exist_ok when cloning tables (#4990)

Adds `exist_ok=False` to Python table cloning and `.exist_ok(true)` to
the Rust clone builder. An existing target is returned with its data and
version history unchanged; strict duplicate errors remain the default.

```python
snapshot = db.clone_table("snapshot", source_uri, exist_ok=True)
```

Embedded cloning skips source reads for an existing target and handles
typed creation conflicts. Remote cloning forwards the option to the
server. Includes sync/async preservation and remote request coverage,
plus SQL syntax documentation for `CREATE TABLE IF NOT EXISTS target
CLONE source`. Requires a server supporting the option for remote
clones.

BREAKING CHANGE: Rust `CloneTableRequest` struct literals must
initialize the new `exist_ok` field; `CloneTableRequest::new` remains
compatible.

No local builds or tests were run, as requested.
This commit is contained in:
Jack Ye
2026-11-14 10:48:26 +08:00
committed by GitHub
parent a0eb6b4b71
commit 4db0d5c7f5
11 changed files with 162 additions and 30 deletions
+12
View File
@@ -270,6 +270,18 @@ Renaming is not supported on a local directory connection. In LanceDB Enterprise
whether a table can be renamed or cloned at all depends on how the deployment
stores its catalog.
{/* Draft for the table-clone exist_ok release.
Pass `exist_ok=True` to keep an existing target table, including its data and
version history, instead of reporting a duplicate. The source is used only when
the target is missing. Remote use requires a deployment that supports this option.
```python
db.clone_table("docs_snapshot", source_uri="s3://bucket/db/documents.lance", exist_ok=True)
```
*/}
A **shallow** clone (the default) references the source's data files rather
than copying them, so it is fast and cheap regardless of table size. It is the
practical way to branch an experiment off a known version, or to pin a training
+10
View File
@@ -205,6 +205,16 @@ Dropping, renaming, or retyping a column is an SDK operation
## CLONE {#clone}
{/* Draft for the table-clone exist_ok release.
Use `CREATE TABLE IF NOT EXISTS target CLONE source` to preserve an existing
target unchanged. The Python equivalent is
`db.clone_table("target", source_uri, exist_ok=True)`. Omitting the option reports
an error for an existing target. A skipped clone does not copy rows or change
the target's version history.
*/}
<CodeGroup>
```sql SQL icon="database"
CREATE TABLE docs_snapshot CLONE documents
+2
View File
@@ -347,6 +347,8 @@ class Connection(object):
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
*,
exist_ok: bool = False,
) -> Table: ...
async def rename_table(
self,
+8
View File
@@ -2271,6 +2271,7 @@ class LanceDBConnection(DBConnection):
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
exist_ok: bool = False,
) -> LanceTable:
"""Clone a table from a source table.
@@ -2295,6 +2296,8 @@ class LanceDBConnection(DBConnection):
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
exist_ok: bool, default False
Return the existing target unchanged instead of cloning over it.
Returns
-------
@@ -2310,6 +2313,7 @@ class LanceDBConnection(DBConnection):
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
exist_ok=exist_ok,
)
)
return LanceTable.open(
@@ -3595,6 +3599,7 @@ class AsyncConnection(object):
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
exist_ok: bool = False,
) -> AsyncTable:
"""Clone a table from a source table.
@@ -3619,6 +3624,8 @@ class AsyncConnection(object):
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
exist_ok: bool, default False
Return the existing target unchanged instead of cloning over it.
Returns
-------
@@ -3633,6 +3640,7 @@ class AsyncConnection(object):
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
exist_ok=exist_ok,
)
return AsyncTable(table)
+4
View File
@@ -576,6 +576,7 @@ class RemoteDBConnection(DBConnection):
source_version: Optional[int] = None,
source_tag: Optional[str] = None,
is_shallow: bool = True,
exist_ok: bool = False,
) -> Table:
"""Clone a table from a source table.
@@ -595,6 +596,8 @@ class RemoteDBConnection(DBConnection):
is_shallow: bool, default True
Whether to perform a shallow clone (True) or deep clone (False).
Currently only shallow clone is supported.
exist_ok: bool, default False
Return the existing target unchanged instead of cloning over it.
Returns
-------
@@ -612,6 +615,7 @@ class RemoteDBConnection(DBConnection):
source_version=source_version,
source_tag=source_tag,
is_shallow=is_shallow,
exist_ok=exist_ok,
)
)
return RemoteTable(
@@ -0,0 +1,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import pytest
import lancedb
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
async def test_clone_exist_ok_preserves_target(tmp_path, asynchronous):
source_uri = str(tmp_path / "source.lance")
if asynchronous:
db = await lancedb.connect_async(tmp_path)
await db.create_table("source", [{"id": 1}])
table = await db.clone_table("target", source_uri, exist_ok=True)
assert (await table.to_arrow()).to_pylist() == [{"id": 1}]
await table.add([{"id": 2}])
version = await table.version()
with pytest.raises(ValueError, match="already exists"):
await db.clone_table("target", source_uri)
kept = await db.clone_table("target", "missing.lance", exist_ok=True)
assert await kept.version() == version
assert (await kept.to_arrow()).to_pylist() == [{"id": 1}, {"id": 2}]
else:
db = lancedb.connect(tmp_path)
db.create_table("source", [{"id": 1}])
table = db.clone_table("target", source_uri, exist_ok=True)
assert table.to_arrow().to_pylist() == [{"id": 1}]
table.add([{"id": 2}])
version = table.version
with pytest.raises(ValueError, match="already exists"):
db.clone_table("target", source_uri)
kept = db.clone_table("target", "missing.lance", exist_ok=True)
assert kept.version == version
assert kept.to_arrow().to_pylist() == [{"id": 1}, {"id": 2}]
+4 -2
View File
@@ -457,7 +457,8 @@ impl Connection {
})
}
#[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true))]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true, *, exist_ok=false))]
pub fn clone_table(
self_: PyRef<'_, Self>,
target_table_name: String,
@@ -466,6 +467,7 @@ impl Connection {
source_version: Option<u64>,
source_tag: Option<String>,
is_shallow: bool,
exist_ok: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
@@ -477,7 +479,7 @@ impl Connection {
if let Some(tag) = source_tag {
builder = builder.source_tag(tag);
}
builder = builder.is_shallow(is_shallow);
builder = builder.is_shallow(is_shallow).exist_ok(exist_ok);
future_into_py(self_.py(), async move {
let table = builder.execute().await.infer_error()?;
+28 -2
View File
@@ -433,6 +433,21 @@ impl CloneTableBuilder {
}
}
/// Preserve an existing target without reading or cloning the source.
/// Defaults to false, which reports a duplicate-name error.
///
/// ```no_run
/// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> {
/// let table = db.clone_table("snapshot", "s3://bucket/source.lance")
/// .exist_ok(true).execute().await?;
/// # Ok(())
/// # }
/// ```
pub fn exist_ok(mut self, exist_ok: bool) -> Self {
self.request.exist_ok = exist_ok;
self
}
/// Set the source version to clone from
pub fn source_version(mut self, version: u64) -> Self {
self.request.source_version = Some(version);
@@ -633,7 +648,8 @@ impl Connection {
/// Creates a new table by cloning from an existing source table.
/// By default, this performs a shallow clone where the new table shares
/// the underlying data files with the source table.
/// The target table name must be unused. An existing table is left unchanged.
/// The target table name must be unused by default. Use
/// [`CloneTableBuilder::exist_ok`] to open an existing target unchanged.
///
/// # Parameters
/// - `target_table_name`: The name of the new table to create
@@ -2787,6 +2803,14 @@ mod tests {
"unexpected clone error: {err:?}"
);
let preserved = db
.clone_table(target, "missing-source.lance")
.exist_ok(true)
.execute()
.await
.unwrap();
assert_eq!(preserved.name(), target);
// Reopen both tables to verify that the failed clone left their data and versions intact.
for (name, expected_data) in [("source", source_data), ("target", target_data)] {
let table = db.open_table(name).execute().await.unwrap();
@@ -2804,8 +2828,9 @@ mod tests {
}
}
#[rstest::rstest]
#[tokio::test]
async fn test_clone_table() {
async fn test_clone_table(#[values(false, true)] exist_ok: bool) {
let tmp_dir = tempdir().unwrap();
let uri = tmp_dir.path().to_str().unwrap();
let db = connect(uri).execute().await.unwrap();
@@ -2830,6 +2855,7 @@ mod tests {
// Clone the table
let cloned_table = db
.clone_table("cloned_table", source_uri)
.exist_ok(exist_ok)
.execute()
.await
.unwrap();
+3
View File
@@ -163,6 +163,8 @@ impl CreateTableRequest {
/// sharing the same data, deletion, and index files.
#[derive(Clone, Debug)]
pub struct CloneTableRequest {
/// Preserve an existing target rather than returning a duplicate-name error.
pub exist_ok: bool,
/// The name of the target table to create
pub target_table_name: String,
/// The namespace path for the target table. Empty list represents root namespace.
@@ -184,6 +186,7 @@ pub struct CloneTableRequest {
impl CloneTableRequest {
pub fn new(target_table_name: String, source_uri: String) -> Self {
Self {
exist_ok: false,
target_table_name,
target_namespace_path: vec![],
source_uri,
+42 -23
View File
@@ -1119,40 +1119,47 @@ impl Database for ListingDatabase {
};
let target_uri = self.table_uri(&request.target_table_name)?;
match DatasetBuilder::from_uri(&target_uri)
let exists = match DatasetBuilder::from_uri(&target_uri)
.with_read_params(read_params.clone())
.load()
.await
{
Ok(_) => {
Ok(_) if !request.exist_ok => {
return Err(Error::TableAlreadyExists {
name: request.target_table_name,
});
}
Err(lance::Error::DatasetNotFound { .. }) => {}
Ok(_) => true,
Err(lance::Error::DatasetNotFound { .. }) => false,
Err(err) => return Err(err.into()),
};
if !exists {
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
.with_read_params(read_params.clone())
.load()
.await
.map_err(|e| -> Error { e.into() })?;
let version_ref = match (request.source_version, request.source_tag) {
(Some(v), None) => Ok(Ref::Version(None, Some(v))),
(None, Some(tag)) => Ok(Ref::Tag(tag)),
(None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))),
_ => Err(Error::InvalidInput {
message: "Cannot specify both source_version and source_tag".to_string(),
}),
}?;
match source_dataset
.shallow_clone(&target_uri, version_ref, Some(storage_params))
.await
{
Ok(_) => {}
Err(lance::Error::DatasetAlreadyExists { .. }) if request.exist_ok => {}
Err(error) => return Err(error.into()),
}
}
let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri)
.with_read_params(read_params.clone())
.load()
.await
.map_err(|e| -> Error { e.into() })?;
let version_ref = match (request.source_version, request.source_tag) {
(Some(v), None) => Ok(Ref::Version(None, Some(v))),
(None, Some(tag)) => Ok(Ref::Tag(tag)),
(None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))),
_ => Err(Error::InvalidInput {
message: "Cannot specify both source_version and source_tag".to_string(),
}),
}?;
source_dataset
.shallow_clone(&target_uri, version_ref, Some(storage_params))
.await
.map_err(|e| -> Error { e.into() })?;
let cloned_table = NativeTable::open_with_params(
&target_uri,
&request.target_table_name,
@@ -2252,6 +2259,7 @@ mod tests {
// Clone the table
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned_table".to_string(),
target_namespace_path: vec![],
source_uri: source_uri.clone(),
@@ -2313,6 +2321,7 @@ mod tests {
// Clone the table
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned_with_data".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2375,6 +2384,7 @@ mod tests {
// Clone should work with storage options
let cloned = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2412,6 +2422,7 @@ mod tests {
// Try deep clone (should fail)
let result = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2453,6 +2464,7 @@ mod tests {
// Try clone with namespace (should fail for listing database)
let result = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned".to_string(),
target_namespace_path: vec!["namespace".to_string()], // Non-empty namespace
source_uri,
@@ -2494,6 +2506,7 @@ mod tests {
// Try clone with invalid target name
let result = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "invalid/name".to_string(), // Invalid name with slash
target_namespace_path: vec![],
source_uri,
@@ -2514,6 +2527,7 @@ mod tests {
// Try to clone from non-existent source
let result = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned".to_string(),
target_namespace_path: vec![],
source_uri: "/nonexistent/table.lance".to_string(),
@@ -2551,6 +2565,7 @@ mod tests {
// Try clone with both version and tag (should fail)
let result = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2625,6 +2640,7 @@ mod tests {
// Clone from the initial version (should have only 2 rows)
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned_from_version".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2704,6 +2720,7 @@ mod tests {
// Clone from the tag (should have only 2 rows)
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned_from_tag".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2756,6 +2773,7 @@ mod tests {
// Clone the table
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "independent_clone".to_string(),
target_namespace_path: vec![],
source_uri,
@@ -2849,6 +2867,7 @@ mod tests {
// Clone without specifying version or tag (should get latest)
let cloned_table = db
.clone_table(CloneTableRequest {
exist_ok: false,
target_table_name: "cloned_latest".to_string(),
target_namespace_path: vec![],
source_uri,
+13 -3
View File
@@ -1612,10 +1612,13 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
is_shallow: Some(request.is_shallow),
};
let req = self
let mut req = self
.client
.post(&format!("/v1/table/{}/clone", table_identifier.clone()))
.json(&remote_request);
if request.exist_ok {
req = req.query(&[("exist_ok", true)]);
}
let (request_id, rsp) = self.client.send(req).await?;
@@ -3651,9 +3654,10 @@ mod tests {
}
}
#[rstest::rstest]
#[tokio::test]
async fn test_clone_table() {
let conn = Connection::new_with_handler(|request| {
async fn test_clone_table(#[values(false, true)] exist_ok: bool) {
let conn = Connection::new_with_handler(move |request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/table/cloned_table/clone");
assert_eq!(
@@ -3665,12 +3669,18 @@ mod tests {
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["source_location"], "s3://bucket/source_table");
assert_eq!(body["is_shallow"], true);
if exist_ok {
assert_eq!(request.url().query(), Some("exist_ok=true"));
} else {
assert!(request.url().query().is_none());
}
http::Response::builder().status(200).body("").unwrap()
});
let table = conn
.clone_table("cloned_table", "s3://bucket/source_table")
.exist_ok(exist_ok)
.execute()
.await
.unwrap();