diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index bf6bf3edd..8031a4c49 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -270,6 +270,18 @@ Renaming is not supported on a local directory connection. In LanceDB Enterprise whether a table can be renamed or cloned at all depends on how the deployment stores its catalog. +{/* Draft for the table-clone exist_ok release. + +Pass `exist_ok=True` to keep an existing target table, including its data and +version history, instead of reporting a duplicate. The source is used only when +the target is missing. Remote use requires a deployment that supports this option. + +```python +db.clone_table("docs_snapshot", source_uri="s3://bucket/db/documents.lance", exist_ok=True) +``` + +*/} + A **shallow** clone (the default) references the source's data files rather than copying them, so it is fast and cheap regardless of table size. It is the practical way to branch an experiment off a known version, or to pin a training diff --git a/docs/web/sql/ddl/table.mdx b/docs/web/sql/ddl/table.mdx index ab0ff64bf..c8c42d4e0 100644 --- a/docs/web/sql/ddl/table.mdx +++ b/docs/web/sql/ddl/table.mdx @@ -205,6 +205,16 @@ Dropping, renaming, or retyping a column is an SDK operation ## CLONE {#clone} +{/* Draft for the table-clone exist_ok release. + +Use `CREATE TABLE IF NOT EXISTS target CLONE source` to preserve an existing +target unchanged. The Python equivalent is +`db.clone_table("target", source_uri, exist_ok=True)`. Omitting the option reports +an error for an existing target. A skipped clone does not copy rows or change +the target's version history. + +*/} + ```sql SQL icon="database" CREATE TABLE docs_snapshot CLONE documents diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 44523fd42..d5c024959 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -347,6 +347,8 @@ class Connection(object): source_version: Optional[int] = None, source_tag: Optional[str] = None, is_shallow: bool = True, + *, + exist_ok: bool = False, ) -> Table: ... async def rename_table( self, diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 39c9eb1d5..18ca9cf6f 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -2271,6 +2271,7 @@ class LanceDBConnection(DBConnection): source_version: Optional[int] = None, source_tag: Optional[str] = None, is_shallow: bool = True, + exist_ok: bool = False, ) -> LanceTable: """Clone a table from a source table. @@ -2295,6 +2296,8 @@ class LanceDBConnection(DBConnection): is_shallow: bool, default True Whether to perform a shallow clone (True) or deep clone (False). Currently only shallow clone is supported. + exist_ok: bool, default False + Return the existing target unchanged instead of cloning over it. Returns ------- @@ -2310,6 +2313,7 @@ class LanceDBConnection(DBConnection): source_version=source_version, source_tag=source_tag, is_shallow=is_shallow, + exist_ok=exist_ok, ) ) return LanceTable.open( @@ -3595,6 +3599,7 @@ class AsyncConnection(object): source_version: Optional[int] = None, source_tag: Optional[str] = None, is_shallow: bool = True, + exist_ok: bool = False, ) -> AsyncTable: """Clone a table from a source table. @@ -3619,6 +3624,8 @@ class AsyncConnection(object): is_shallow: bool, default True Whether to perform a shallow clone (True) or deep clone (False). Currently only shallow clone is supported. + exist_ok: bool, default False + Return the existing target unchanged instead of cloning over it. Returns ------- @@ -3633,6 +3640,7 @@ class AsyncConnection(object): source_version=source_version, source_tag=source_tag, is_shallow=is_shallow, + exist_ok=exist_ok, ) return AsyncTable(table) diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 9221c7ceb..69b691004 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -576,6 +576,7 @@ class RemoteDBConnection(DBConnection): source_version: Optional[int] = None, source_tag: Optional[str] = None, is_shallow: bool = True, + exist_ok: bool = False, ) -> Table: """Clone a table from a source table. @@ -595,6 +596,8 @@ class RemoteDBConnection(DBConnection): is_shallow: bool, default True Whether to perform a shallow clone (True) or deep clone (False). Currently only shallow clone is supported. + exist_ok: bool, default False + Return the existing target unchanged instead of cloning over it. Returns ------- @@ -612,6 +615,7 @@ class RemoteDBConnection(DBConnection): source_version=source_version, source_tag=source_tag, is_shallow=is_shallow, + exist_ok=exist_ok, ) ) return RemoteTable( diff --git a/python/python/tests/test_clone_creation.py b/python/python/tests/test_clone_creation.py new file mode 100644 index 000000000..6c12420c5 --- /dev/null +++ b/python/python/tests/test_clone_creation.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import pytest + +import lancedb + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_clone_exist_ok_preserves_target(tmp_path, asynchronous): + source_uri = str(tmp_path / "source.lance") + if asynchronous: + db = await lancedb.connect_async(tmp_path) + await db.create_table("source", [{"id": 1}]) + table = await db.clone_table("target", source_uri, exist_ok=True) + assert (await table.to_arrow()).to_pylist() == [{"id": 1}] + await table.add([{"id": 2}]) + version = await table.version() + with pytest.raises(ValueError, match="already exists"): + await db.clone_table("target", source_uri) + kept = await db.clone_table("target", "missing.lance", exist_ok=True) + assert await kept.version() == version + assert (await kept.to_arrow()).to_pylist() == [{"id": 1}, {"id": 2}] + else: + db = lancedb.connect(tmp_path) + db.create_table("source", [{"id": 1}]) + table = db.clone_table("target", source_uri, exist_ok=True) + assert table.to_arrow().to_pylist() == [{"id": 1}] + table.add([{"id": 2}]) + version = table.version + with pytest.raises(ValueError, match="already exists"): + db.clone_table("target", source_uri) + kept = db.clone_table("target", "missing.lance", exist_ok=True) + assert kept.version == version + assert kept.to_arrow().to_pylist() == [{"id": 1}, {"id": 2}] diff --git a/python/src/connection.rs b/python/src/connection.rs index dabecc048..8763822cc 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -457,7 +457,8 @@ impl Connection { }) } - #[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true))] + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (target_table_name, source_uri, target_namespace_path=None, source_version=None, source_tag=None, is_shallow=true, *, exist_ok=false))] pub fn clone_table( self_: PyRef<'_, Self>, target_table_name: String, @@ -466,6 +467,7 @@ impl Connection { source_version: Option, source_tag: Option, is_shallow: bool, + exist_ok: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); @@ -477,7 +479,7 @@ impl Connection { if let Some(tag) = source_tag { builder = builder.source_tag(tag); } - builder = builder.is_shallow(is_shallow); + builder = builder.is_shallow(is_shallow).exist_ok(exist_ok); future_into_py(self_.py(), async move { let table = builder.execute().await.infer_error()?; diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index c7a058502..6dacbbbea 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -433,6 +433,21 @@ impl CloneTableBuilder { } } + /// Preserve an existing target without reading or cloning the source. + /// Defaults to false, which reports a duplicate-name error. + /// + /// ```no_run + /// # async fn example(db: &lancedb::Connection) -> lancedb::Result<()> { + /// let table = db.clone_table("snapshot", "s3://bucket/source.lance") + /// .exist_ok(true).execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn exist_ok(mut self, exist_ok: bool) -> Self { + self.request.exist_ok = exist_ok; + self + } + /// Set the source version to clone from pub fn source_version(mut self, version: u64) -> Self { self.request.source_version = Some(version); @@ -633,7 +648,8 @@ impl Connection { /// Creates a new table by cloning from an existing source table. /// By default, this performs a shallow clone where the new table shares /// the underlying data files with the source table. - /// The target table name must be unused. An existing table is left unchanged. + /// The target table name must be unused by default. Use + /// [`CloneTableBuilder::exist_ok`] to open an existing target unchanged. /// /// # Parameters /// - `target_table_name`: The name of the new table to create @@ -2787,6 +2803,14 @@ mod tests { "unexpected clone error: {err:?}" ); + let preserved = db + .clone_table(target, "missing-source.lance") + .exist_ok(true) + .execute() + .await + .unwrap(); + assert_eq!(preserved.name(), target); + // Reopen both tables to verify that the failed clone left their data and versions intact. for (name, expected_data) in [("source", source_data), ("target", target_data)] { let table = db.open_table(name).execute().await.unwrap(); @@ -2804,8 +2828,9 @@ mod tests { } } + #[rstest::rstest] #[tokio::test] - async fn test_clone_table() { + async fn test_clone_table(#[values(false, true)] exist_ok: bool) { let tmp_dir = tempdir().unwrap(); let uri = tmp_dir.path().to_str().unwrap(); let db = connect(uri).execute().await.unwrap(); @@ -2830,6 +2855,7 @@ mod tests { // Clone the table let cloned_table = db .clone_table("cloned_table", source_uri) + .exist_ok(exist_ok) .execute() .await .unwrap(); diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 39e361a39..638ae0dd5 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -163,6 +163,8 @@ impl CreateTableRequest { /// sharing the same data, deletion, and index files. #[derive(Clone, Debug)] pub struct CloneTableRequest { + /// Preserve an existing target rather than returning a duplicate-name error. + pub exist_ok: bool, /// The name of the target table to create pub target_table_name: String, /// The namespace path for the target table. Empty list represents root namespace. @@ -184,6 +186,7 @@ pub struct CloneTableRequest { impl CloneTableRequest { pub fn new(target_table_name: String, source_uri: String) -> Self { Self { + exist_ok: false, target_table_name, target_namespace_path: vec![], source_uri, diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index d5749f974..765c8af3d 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1119,40 +1119,47 @@ impl Database for ListingDatabase { }; let target_uri = self.table_uri(&request.target_table_name)?; - match DatasetBuilder::from_uri(&target_uri) + let exists = match DatasetBuilder::from_uri(&target_uri) .with_read_params(read_params.clone()) .load() .await { - Ok(_) => { + Ok(_) if !request.exist_ok => { return Err(Error::TableAlreadyExists { name: request.target_table_name, }); } - Err(lance::Error::DatasetNotFound { .. }) => {} + Ok(_) => true, + Err(lance::Error::DatasetNotFound { .. }) => false, Err(err) => return Err(err.into()), + }; + + if !exists { + let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri) + .with_read_params(read_params.clone()) + .load() + .await + .map_err(|e| -> Error { e.into() })?; + + let version_ref = match (request.source_version, request.source_tag) { + (Some(v), None) => Ok(Ref::Version(None, Some(v))), + (None, Some(tag)) => Ok(Ref::Tag(tag)), + (None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))), + _ => Err(Error::InvalidInput { + message: "Cannot specify both source_version and source_tag".to_string(), + }), + }?; + + match source_dataset + .shallow_clone(&target_uri, version_ref, Some(storage_params)) + .await + { + Ok(_) => {} + Err(lance::Error::DatasetAlreadyExists { .. }) if request.exist_ok => {} + Err(error) => return Err(error.into()), + } } - let mut source_dataset = DatasetBuilder::from_uri(&request.source_uri) - .with_read_params(read_params.clone()) - .load() - .await - .map_err(|e| -> Error { e.into() })?; - - let version_ref = match (request.source_version, request.source_tag) { - (Some(v), None) => Ok(Ref::Version(None, Some(v))), - (None, Some(tag)) => Ok(Ref::Tag(tag)), - (None, None) => Ok(Ref::Version(None, Some(source_dataset.version().version))), - _ => Err(Error::InvalidInput { - message: "Cannot specify both source_version and source_tag".to_string(), - }), - }?; - - source_dataset - .shallow_clone(&target_uri, version_ref, Some(storage_params)) - .await - .map_err(|e| -> Error { e.into() })?; - let cloned_table = NativeTable::open_with_params( &target_uri, &request.target_table_name, @@ -2252,6 +2259,7 @@ mod tests { // Clone the table let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned_table".to_string(), target_namespace_path: vec![], source_uri: source_uri.clone(), @@ -2313,6 +2321,7 @@ mod tests { // Clone the table let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned_with_data".to_string(), target_namespace_path: vec![], source_uri, @@ -2375,6 +2384,7 @@ mod tests { // Clone should work with storage options let cloned = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned".to_string(), target_namespace_path: vec![], source_uri, @@ -2412,6 +2422,7 @@ mod tests { // Try deep clone (should fail) let result = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned".to_string(), target_namespace_path: vec![], source_uri, @@ -2453,6 +2464,7 @@ mod tests { // Try clone with namespace (should fail for listing database) let result = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned".to_string(), target_namespace_path: vec!["namespace".to_string()], // Non-empty namespace source_uri, @@ -2494,6 +2506,7 @@ mod tests { // Try clone with invalid target name let result = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "invalid/name".to_string(), // Invalid name with slash target_namespace_path: vec![], source_uri, @@ -2514,6 +2527,7 @@ mod tests { // Try to clone from non-existent source let result = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned".to_string(), target_namespace_path: vec![], source_uri: "/nonexistent/table.lance".to_string(), @@ -2551,6 +2565,7 @@ mod tests { // Try clone with both version and tag (should fail) let result = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned".to_string(), target_namespace_path: vec![], source_uri, @@ -2625,6 +2640,7 @@ mod tests { // Clone from the initial version (should have only 2 rows) let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned_from_version".to_string(), target_namespace_path: vec![], source_uri, @@ -2704,6 +2720,7 @@ mod tests { // Clone from the tag (should have only 2 rows) let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned_from_tag".to_string(), target_namespace_path: vec![], source_uri, @@ -2756,6 +2773,7 @@ mod tests { // Clone the table let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "independent_clone".to_string(), target_namespace_path: vec![], source_uri, @@ -2849,6 +2867,7 @@ mod tests { // Clone without specifying version or tag (should get latest) let cloned_table = db .clone_table(CloneTableRequest { + exist_ok: false, target_table_name: "cloned_latest".to_string(), target_namespace_path: vec![], source_uri, diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index a05bb07b7..300cd6b09 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -1612,10 +1612,13 @@ impl Database for RemoteDatabase { is_shallow: Some(request.is_shallow), }; - let req = self + let mut req = self .client .post(&format!("/v1/table/{}/clone", table_identifier.clone())) .json(&remote_request); + if request.exist_ok { + req = req.query(&[("exist_ok", true)]); + } let (request_id, rsp) = self.client.send(req).await?; @@ -3651,9 +3654,10 @@ mod tests { } } + #[rstest::rstest] #[tokio::test] - async fn test_clone_table() { - let conn = Connection::new_with_handler(|request| { + async fn test_clone_table(#[values(false, true)] exist_ok: bool) { + let conn = Connection::new_with_handler(move |request| { assert_eq!(request.method(), &reqwest::Method::POST); assert_eq!(request.url().path(), "/v1/table/cloned_table/clone"); assert_eq!( @@ -3665,12 +3669,18 @@ mod tests { let body: serde_json::Value = serde_json::from_slice(body).unwrap(); assert_eq!(body["source_location"], "s3://bucket/source_table"); assert_eq!(body["is_shallow"], true); + if exist_ok { + assert_eq!(request.url().query(), Some("exist_ok=true")); + } else { + assert!(request.url().query().is_none()); + } http::Response::builder().status(200).body("").unwrap() }); let table = conn .clone_table("cloned_table", "s3://bucket/source_table") + .exist_ok(exist_ok) .execute() .await .unwrap();