From e98d8ac6857fa058fa07e005e0796c2a9a940d12 Mon Sep 17 00:00:00 2001 From: Drew Gallardo Date: Fri, 21 Aug 2026 23:37:12 -0700 Subject: [PATCH 1/2] feat!: rename branch merge to cherry_pick (#3986) This PR is a **breaking** rename of #3686. merge reads like git merge w/ three-way, replay history, combine two lines of work. That is not this API. This call takes one additive change on a branch and lands it on main. New column, including a blob column. Main's existing columns are not rewritten. If it cannot land, you get `status="failed"` and `diff.errors`, not a merge conflict to resolve. Cherry-pick is terminology that aligns more with that. ```python table = db.open_table("images") table.branches.create("exp") exp = table.branches.checkout("exp") exp.add_columns({"tag": "cast('draft' as string)"}) diff = table.branches.diff("exp") preview = table.branches.cherry_pick("exp", dry_run=True) result = table.branches.cherry_pick("exp") if result["status"] == "cherryPicked": print("landed at", result["mainVersionAfter"]) elif result["status"] == "failed": print(result["diff"]["errors"]) ``` ### Behavior - Remote / Enterprise only. Local still NotSupported. - HTTP 409 is not an exception. It is Ok with status="failed" and diff.errors (CherryPickError). - Unknown error / status codes still parse as Unknown. - Requests are not retried. 409 is final and carries the body. - Endpoint is POST /v1/table/{id}/branches/cherry_pick/. - merge_insert and Table.merge are unchanged. ### Testing - `cargo test -p lancedb --features remote diff_branch` - `cargo test -p lancedb --features remote cherry_pick` - `pytest python/python/tests/test_remote_db.py -k cherry_pick` - node `remote.test.ts` diffs / cherry-picks path --- docs/src/js/classes/Branches.md | 50 ++++++------ docs/src/js/globals.md | 6 +- docs/src/js/interfaces/BranchDiff.md | 24 ++---- .../{MergeBlocker.md => CherryPickError.md} | 6 +- docs/src/js/interfaces/CherryPickPreview.md | 17 +++++ ...rgeBranchResult.md => CherryPickResult.md} | 12 +-- docs/src/js/interfaces/MergePreview.md | 17 ----- nodejs/__test__/remote.test.ts | 26 +++---- nodejs/lancedb/index.ts | 6 +- nodejs/lancedb/table.ts | 37 +++++---- nodejs/src/table.rs | 6 +- python/python/lancedb/_lancedb.pyi | 2 +- python/python/lancedb/table.py | 22 +++--- python/python/tests/test_remote_db.py | 17 ++--- python/src/table.rs | 4 +- rust/lancedb/src/remote/table.rs | 76 ++++++++++--------- rust/lancedb/src/table.rs | 30 ++++---- .../table/{branch_merge.rs => cherry_pick.rs} | 27 ++++--- 18 files changed, 188 insertions(+), 197 deletions(-) rename docs/src/js/interfaces/{MergeBlocker.md => CherryPickError.md} (54%) create mode 100644 docs/src/js/interfaces/CherryPickPreview.md rename docs/src/js/interfaces/{MergeBranchResult.md => CherryPickResult.md} (60%) delete mode 100644 docs/src/js/interfaces/MergePreview.md rename rust/lancedb/src/table/{branch_merge.rs => cherry_pick.rs} (86%) diff --git a/docs/src/js/classes/Branches.md b/docs/src/js/classes/Branches.md index 5296d0d08..6680fbfee 100644 --- a/docs/src/js/classes/Branches.md +++ b/docs/src/js/classes/Branches.md @@ -37,6 +37,31 @@ latest and stays writable. *** +### cherryPick() + +```ts +cherryPick(fromBranch, dryRun): Promise +``` + +Cherry-pick a branch onto main. + +Set `dryRun` to `true` to preview. A failed cherry-pick resolves +with `status: "failed"` instead of throwing. + +#### Parameters + +* **fromBranch**: `string` + Branch to cherry-pick from. + +* **dryRun**: `boolean` = `false` + When true, only preview. Defaults to false. + +#### Returns + +`Promise`<[`CherryPickResult`](../interfaces/CherryPickResult.md)> + +*** + ### create() ```ts @@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata. #### Returns `Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>> - -*** - -### merge() - -```ts -merge(fromBranch, dryRun): Promise -``` - -Merge a branch into main. - -Set `dryRun` to `true` to preview the merge. A rejected merge resolves -with `status: "rejected"` instead of throwing. - -#### Parameters - -* **fromBranch**: `string` - Branch to merge from. - -* **dryRun**: `boolean` = `false` - When true, only preview the merge. Defaults to false. - -#### Returns - -`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)> diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 462907cfd..4deeed1bc 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -59,6 +59,9 @@ - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) - [BucketStats](interfaces/BucketStats.md) +- [CherryPickError](interfaces/CherryPickError.md) +- [CherryPickPreview](interfaces/CherryPickPreview.md) +- [CherryPickResult](interfaces/CherryPickResult.md) - [ClientConfig](interfaces/ClientConfig.md) - [ColumnAlteration](interfaces/ColumnAlteration.md) - [ColumnOrdering](interfaces/ColumnOrdering.md) @@ -99,9 +102,6 @@ - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) - [MemtableStats](interfaces/MemtableStats.md) -- [MergeBlocker](interfaces/MergeBlocker.md) -- [MergeBranchResult](interfaces/MergeBranchResult.md) -- [MergePreview](interfaces/MergePreview.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) - [OAuthConfig](interfaces/OAuthConfig.md) diff --git a/docs/src/js/interfaces/BranchDiff.md b/docs/src/js/interfaces/BranchDiff.md index 224be1991..b408ebe11 100644 --- a/docs/src/js/interfaces/BranchDiff.md +++ b/docs/src/js/interfaces/BranchDiff.md @@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[]; *** +### errors + +```ts +errors: CherryPickError[]; +``` + +*** + ### fromBranch ```ts @@ -66,22 +74,6 @@ mainVersion: number; *** -### mergeBlockers - -```ts -mergeBlockers: MergeBlocker[]; -``` - -*** - -### mergeable - -```ts -mergeable: boolean; -``` - -*** - ### parentVersion ```ts diff --git a/docs/src/js/interfaces/MergeBlocker.md b/docs/src/js/interfaces/CherryPickError.md similarity index 54% rename from docs/src/js/interfaces/MergeBlocker.md rename to docs/src/js/interfaces/CherryPickError.md index 6c8f84b37..84f8fe012 100644 --- a/docs/src/js/interfaces/MergeBlocker.md +++ b/docs/src/js/interfaces/CherryPickError.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBlocker +[@lancedb/lancedb](../globals.md) / CherryPickError -# Interface: MergeBlocker +# Interface: CherryPickError -A reason why a branch cannot currently be merged. +A reason why a cherry-pick cannot currently land. ## Properties diff --git a/docs/src/js/interfaces/CherryPickPreview.md b/docs/src/js/interfaces/CherryPickPreview.md new file mode 100644 index 000000000..9620068a2 --- /dev/null +++ b/docs/src/js/interfaces/CherryPickPreview.md @@ -0,0 +1,17 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / CherryPickPreview + +# Interface: CherryPickPreview + +Changes that would be, or were, promoted by a cherry-pick. + +## Properties + +### promotedColumns + +```ts +promotedColumns: string[]; +``` diff --git a/docs/src/js/interfaces/MergeBranchResult.md b/docs/src/js/interfaces/CherryPickResult.md similarity index 60% rename from docs/src/js/interfaces/MergeBranchResult.md rename to docs/src/js/interfaces/CherryPickResult.md index 61c2d0d33..c837ab143 100644 --- a/docs/src/js/interfaces/MergeBranchResult.md +++ b/docs/src/js/interfaces/CherryPickResult.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBranchResult +[@lancedb/lancedb](../globals.md) / CherryPickResult -# Interface: MergeBranchResult +# Interface: CherryPickResult -Result of previewing or attempting a branch merge. +Result of previewing or attempting a cherry-pick. ## Properties @@ -29,7 +29,7 @@ optional mainVersionAfter: number; ### preview ```ts -preview: MergePreview; +preview: CherryPickPreview; ``` *** @@ -38,9 +38,9 @@ preview: MergePreview; ```ts status: + | "failed" | "unknown" - | "rejected" | "ready" | "notImplemented" - | "merged"; + | "cherryPicked"; ``` diff --git a/docs/src/js/interfaces/MergePreview.md b/docs/src/js/interfaces/MergePreview.md deleted file mode 100644 index 0d9717289..000000000 --- a/docs/src/js/interfaces/MergePreview.md +++ /dev/null @@ -1,17 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / MergePreview - -# Interface: MergePreview - -Changes that would be, or were, promoted by a branch merge. - -## Properties - -### promotedColumns - -```ts -promotedColumns: string[]; -``` diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index e766b3d2a..01320d2b0 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -311,7 +311,7 @@ describe("remote connection", () => { expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); - it("diffs and merges remote branches", async () => { + it("diffs and cherry-picks remote branches", async () => { const sampleDiff = { fromBranch: "exp", parentVersion: 1, @@ -333,10 +333,9 @@ describe("remote connection", () => { changedColumns: [], addedIndexes: [], removedIndexes: [], - mergeable: true, - mergeBlockers: [], + errors: [], }; - const mergeBodies: Record[] = []; + const cherryPickBodies: Record[] = []; await withMockDatabase( (req, res) => { @@ -366,17 +365,16 @@ describe("remote connection", () => { .end(JSON.stringify(sampleDiff)); return; } - if (path.endsWith("/branches/merge/")) { - mergeBodies.push(body); + if (path.endsWith("/branches/cherry_pick/")) { + cherryPickBodies.push(body); const dryRun = body["dry_run"] === true; const response = { - status: dryRun ? "ready" : "rejected", + status: dryRun ? "ready" : "failed", diff: dryRun ? sampleDiff : { ...sampleDiff, - mergeable: false, - mergeBlockers: [ + errors: [ { code: "baseMoved", message: "main has advanced" }, ], }, @@ -398,19 +396,19 @@ describe("remote connection", () => { await expect(branches.diff("exp")).resolves.toEqual(sampleDiff); - const rejected = await branches.merge("exp"); - expect(rejected.status).toBe("rejected"); - expect(rejected.diff.mergeBlockers).toEqual([ + const failed = await branches.cherryPick("exp"); + expect(failed.status).toBe("failed"); + expect(failed.diff.errors).toEqual([ { code: "baseMoved", message: "main has advanced" }, ]); - const preview = await branches.merge("exp", true); + const preview = await branches.cherryPick("exp", true); expect(preview.status).toBe("ready"); expect(preview.preview.promotedColumns).toEqual(["tag"]); }, ); - expect(mergeBodies).toEqual([ + expect(cherryPickBodies).toEqual([ // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format { from_branch: "exp", dry_run: false }, // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 6a5bfe3b4..aa9c7ef13 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -135,10 +135,10 @@ export { BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, - MergeBlocker, + CherryPickError, BranchDiff, - MergePreview, - MergeBranchResult, + CherryPickPreview, + CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 964c2cea3..4e323ec64 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -1557,8 +1557,8 @@ export interface BranchRowCountSummary { deltaAvailable: boolean; } -/** A reason why a branch cannot currently be merged. */ -export interface MergeBlocker { +/** A reason why a cherry-pick cannot currently land. */ +export interface CherryPickError { code: string; message: string; } @@ -1578,20 +1578,19 @@ export interface BranchDiff { changedColumns: BranchColumnChange[]; addedIndexes: BranchIndexSummary[]; removedIndexes: BranchIndexSummary[]; - mergeable: boolean; - mergeBlockers: MergeBlocker[]; + errors: CherryPickError[]; } -/** Changes that would be, or were, promoted by a branch merge. */ -export interface MergePreview { +/** Changes that would be, or were, promoted by a cherry-pick. */ +export interface CherryPickPreview { promotedColumns: string[]; } -/** Result of previewing or attempting a branch merge. */ -export interface MergeBranchResult { - status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown"; +/** Result of previewing or attempting a cherry-pick. */ +export interface CherryPickResult { + status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown"; diff: BranchDiff; - preview: MergePreview; + preview: CherryPickPreview; mainVersionAfter?: number; } @@ -1654,21 +1653,21 @@ export class Branches { } /** - * Merge a branch into main. + * Cherry-pick a branch onto main. * - * Set `dryRun` to `true` to preview the merge. A rejected merge resolves - * with `status: "rejected"` instead of throwing. + * Set `dryRun` to `true` to preview. A failed cherry-pick resolves + * with `status: "failed"` instead of throwing. * - * @param fromBranch Branch to merge from. - * @param dryRun When true, only preview the merge. Defaults to false. + * @param fromBranch Branch to cherry-pick from. + * @param dryRun When true, only preview. Defaults to false. */ - async merge( + async cherryPick( fromBranch: string, dryRun: boolean = false, - ): Promise { - return (await this.#inner.merge( + ): Promise { + return (await this.#inner.cherryPick( fromBranch, dryRun, - )) as unknown as MergeBranchResult; + )) as unknown as CherryPickResult; } } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index b15491202..694b6a704 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -1605,18 +1605,18 @@ impl Branches { } #[napi(ts_return_type = "Promise>")] - pub async fn merge( + pub async fn cherry_pick( &self, from_branch: String, dry_run: Option, ) -> napi::Result { let result = self .inner - .merge_branch(&from_branch, dry_run.unwrap_or(false)) + .cherry_pick(&from_branch, dry_run.unwrap_or(false)) .await .default_error()?; serde_json::to_value(result).map_err(|err| { - napi::Error::from_reason(format!("failed to serialize branch merge result: {err}")) + napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}")) }) } } diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index b23d79c85..648469579 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -432,7 +432,7 @@ class Branches: async def checkout(self, name: str, version: Optional[int] = None) -> Table: ... async def delete(self, name: str) -> None: ... async def diff(self, from_branch: str) -> Dict[str, Any]: ... - async def merge( + async def cherry_pick( self, from_branch: str, dry_run: bool = False ) -> Dict[str, Any]: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 913ab5289..0c84b4036 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -6801,21 +6801,21 @@ class Branches: """Diff a branch against main.""" return LOOP.run(self._table.branches.diff(from_branch)) - def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. Parameters ---------- from_branch: str - Branch to merge from. + Branch to cherry-pick from. dry_run: bool, default False - When True, only preview. When False, attempt the merge. + When True, only preview. When False, attempt the cherry-pick. Notes ----- - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return LOOP.run(self._table.branches.merge(from_branch, dry_run)) + return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run)) def _wrap( self, async_table: "AsyncTable", version: Optional[int] = None @@ -6951,9 +6951,11 @@ class AsyncBranches: """Diff a branch against main.""" return await self._table.branches.diff(from_branch) - async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + async def cherry_pick( + self, from_branch: str, dry_run: bool = False + ) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return await self._table.branches.merge(from_branch, dry_run) + return await self._table.branches.cherry_pick(from_branch, dry_run) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 13ffc4415..08f5550eb 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -242,8 +242,8 @@ def test_remote_table_branches_sync(): table.branches.delete("exp") -def test_remote_table_branch_merge_defaults_to_execute(): - merge_bodies = [] +def test_remote_table_cherry_pick_defaults_to_execute(): + cherry_pick_bodies = [] diff = { "fromBranch": "exp", "parentVersion": 1, @@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute(): "changedColumns": [], "addedIndexes": [], "removedIndexes": [], - "mergeable": True, - "mergeBlockers": [], + "errors": [], } def handler(request): @@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute(): else: content_len = int(request.headers.get("Content-Length")) request_body = json.loads(request.rfile.read(content_len)) - merge_bodies.append(request_body) + cherry_pick_bodies.append(request_body) dry_run = request_body["dry_run"] status = 200 if dry_run else 409 body = { - "status": "ready" if dry_run else "rejected", + "status": "ready" if dry_run else "failed", "diff": diff, "preview": {"promotedColumns": []}, } @@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute(): with mock_lancedb_connection(handler) as db: branches = db.open_table("test").branches - assert branches.merge("exp")["status"] == "rejected" - assert branches.merge("exp", dry_run=True)["status"] == "ready" + assert branches.cherry_pick("exp")["status"] == "failed" + assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready" - assert merge_bodies == [ + assert cherry_pick_bodies == [ {"from_branch": "exp", "dry_run": False}, {"from_branch": "exp", "dry_run": True}, ] diff --git a/python/src/table.rs b/python/src/table.rs index a3a7c9d68..5cdcc3653 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -1940,7 +1940,7 @@ impl Branches { } #[pyo3(signature = (from_branch, dry_run=false))] - pub fn merge( + pub fn cherry_pick( self_: PyRef<'_, Self>, from_branch: String, dry_run: bool, @@ -1948,7 +1948,7 @@ impl Branches { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { let result = inner - .merge_branch(&from_branch, dry_run) + .cherry_pick(&from_branch, dry_run) .await .infer_error()?; Python::attach(|py| struct_to_wire_py(py, &result)) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 6c446907d..fb7278db9 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob; use crate::table::AddColumnsResult; use crate::table::AddResult; use crate::table::BranchDiff; +use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; use crate::table::LsmStats; use crate::table::LsmWriteSpec; -use crate::table::MergeBranchResult; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; @@ -2031,7 +2031,7 @@ impl BaseTable for RemoteTable { async fn diff_branch(&self, from_branch: &str) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self @@ -2058,20 +2058,23 @@ impl BaseTable for RemoteTable { }) } - async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result { + async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self .client - .post(&format!("/v1/table/{}/branches/merge/", self.identifier)) + .post(&format!( + "/v1/table/{}/branches/cherry_pick/", + self.identifier + )) .json(&serde_json::json!({ "from_branch": from_branch, "dry_run": dry_run, })); - // No retry. 409 rejected merge is final and carries a body. + // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error. let (request_id, response) = self.send(request, false).await?; let status = response.status(); if status == StatusCode::NOT_FOUND { @@ -2080,11 +2083,11 @@ impl BaseTable for RemoteTable { source: format!("branch '{}' does not exist", from_branch).into(), }); } - // 200 and 409 both carry MergeBranchResult. + // 200 and 409 both carry CherryPickResult. if status != StatusCode::OK && status != StatusCode::CONFLICT { let body = response.text().await.unwrap_or_default(); return Err(Error::Http { - source: format!("unexpected status {status} from merge_branch: {body}").into(), + source: format!("unexpected status {status} from cherry_pick: {body}").into(), request_id, status_code: Some(status), }); @@ -2092,7 +2095,7 @@ impl BaseTable for RemoteTable { let body = response.text().await.err_to_http(request_id.clone())?; serde_json::from_str(&body).map_err(|err| Error::Http { source: format!( - "Failed to parse merge_branch response: {}, body: {}", + "Failed to parse cherry_pick response: {}, body: {}", err, body ) .into(), @@ -10513,8 +10516,7 @@ mod tests { "changedColumns":[], "addedIndexes":[], "removedIndexes":[], - "mergeable":true, - "mergeBlockers":[] + "errors":[] }"# } @@ -10532,15 +10534,18 @@ mod tests { }); let diff = table.diff_branch("exp").await.unwrap(); assert_eq!(diff.from_branch, "exp"); - assert!(diff.mergeable); + assert!(diff.errors.is_empty()); assert_eq!(diff.added_columns.len(), 1); assert_eq!(diff.added_columns[0].name, "tag"); } #[tokio::test] - async fn test_merge_branch_dry_run() { + async fn test_cherry_pick_dry_run() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["from_branch"], "exp"); assert_eq!(body["dry_run"], true); @@ -10550,27 +10555,29 @@ mod tests { ); http::Response::builder().status(200).body(resp).unwrap() }); - let result = table.merge_branch("exp", true).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Ready); + let result = table.cherry_pick("exp", true).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Ready); assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]); assert!(result.main_version_after.is_none()); } #[tokio::test] - async fn test_merge_branch_rejected_returns_ok_with_body() { + async fn test_cherry_pick_failed_returns_ok_with_body() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["dry_run"], false); let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "baseMoved", "message": "main has advanced" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "promotedColumns": [] } }); @@ -10579,24 +10586,23 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); - assert!(!result.diff.mergeable); - assert_eq!(result.diff.merge_blockers.len(), 1); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); + assert!(!result.diff.errors.is_empty()); + assert_eq!(result.diff.errors.len(), 1); } #[tokio::test] - async fn test_merge_branch_unknown_blocker_code_parses() { + async fn test_cherry_pick_unknown_error_code_parses() { let table = Table::new_with_handler("my_table", |_| { let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "multipleCommits", "message": "branch has more than one data commit" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "operation": "append", "rowsAdded": 2 } }); @@ -10605,24 +10611,24 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); assert_eq!( - result.diff.merge_blockers[0].code, - crate::table::MergeBlockerCode::Unknown + result.diff.errors[0].code, + crate::table::CherryPickErrorCode::Unknown ); assert!(result.preview.promoted_columns.is_empty()); } #[tokio::test] - async fn test_merge_branch_unexpected_2xx_is_error() { + async fn test_cherry_pick_unexpected_2xx_is_error() { let table = Table::new_with_handler("my_table", |_| { http::Response::builder() .status(204) .body(String::new()) .unwrap() }); - let err = table.merge_branch("exp", false).await.unwrap_err(); + let err = table.cherry_pick("exp", false).await.unwrap_err(); match err { Error::Http { status_code: Some(code), diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index ca9ed8f26..c2c12fff5 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder; pub mod add_columns; mod add_data; -pub mod branch_merge; pub mod checkpoint; +pub mod cherry_pick; pub mod computed_columns; mod create_index; pub mod datafusion; @@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder; #[cfg(feature = "remote")] pub(crate) use add_data::PreprocessingOutput; pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior}; -pub use branch_merge::{ - BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode, - MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary, +pub use cherry_pick::{ + BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult, + CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary, }; pub use chrono::Duration; pub use computed_columns::{ @@ -832,14 +832,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Diff a branch against main. Remote only. async fn diff_branch(&self, _from_branch: &str) -> Result { Err(Error::NotSupported { - message: "diff_branch is only supported on remote tables".into(), + message: "Branch diffs are only supported on Enterprise tables.".into(), }) } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result { + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result { Err(Error::NotSupported { - message: "merge_branch is only supported on remote tables".into(), + message: "Cherry-picking branches is only supported on Enterprise tables.".into(), }) } /// The branch this handle is scoped to, or `None` for `main`. @@ -2263,14 +2263,10 @@ impl Table { self.inner.diff_branch(from_branch).await } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - pub async fn merge_branch( - &self, - from_branch: &str, - dry_run: bool, - ) -> Result { - self.inner.merge_branch(from_branch, dry_run).await + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { + self.inner.cherry_pick(from_branch, dry_run).await } /// The branch this handle is scoped to, or `None` for `main`. diff --git a/rust/lancedb/src/table/branch_merge.rs b/rust/lancedb/src/table/cherry_pick.rs similarity index 86% rename from rust/lancedb/src/table/branch_merge.rs rename to rust/lancedb/src/table/cherry_pick.rs index ad81ab64c..93bc76637 100644 --- a/rust/lancedb/src/table/branch_merge.rs +++ b/rust/lancedb/src/table/cherry_pick.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Types for remote branch diff / merge against main. +//! Types for remote branch diff / cherry-pick onto main. use serde::{Deserialize, Serialize}; @@ -44,13 +44,13 @@ pub struct RowCountSummary { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBlockerCode { +pub enum CherryPickErrorCode { BaseMoved, RowCountMismatch, RowsChanged, ColumnRemoved, ColumnChanged, - NoMergeableChanges, + NothingToApply, NoColumnChanges, InputColumnDependency, ParentNotMain, @@ -60,8 +60,8 @@ pub enum MergeBlockerCode { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBlocker { - pub code: MergeBlockerCode, +pub struct CherryPickError { + pub code: CherryPickErrorCode, pub message: String, } @@ -81,34 +81,33 @@ pub struct BranchDiff { pub changed_columns: Vec, pub added_indexes: Vec, pub removed_indexes: Vec, - pub mergeable: bool, - pub merge_blockers: Vec, + pub errors: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergePreview { +pub struct CherryPickPreview { #[serde(default)] pub promoted_columns: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBranchStatus { +pub enum CherryPickStatus { Ready, - Rejected, + Failed, NotImplemented, - Merged, + CherryPicked, #[serde(other)] Unknown, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBranchResult { - pub status: MergeBranchStatus, +pub struct CherryPickResult { + pub status: CherryPickStatus, pub diff: BranchDiff, - pub preview: MergePreview, + pub preview: CherryPickPreview, #[serde(default, skip_serializing_if = "Option::is_none")] pub main_version_after: Option, } From 68749ecfa38f966bec6e650bc41c3f6b8e498818 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Fri, 21 Aug 2026 23:48:43 -0700 Subject: [PATCH 2/2] feat(nodejs): materialized view bindings (#3935) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes materialized views to TypeScript: createMaterializedView, openMaterializedView and listMaterializedViews on Connection, and a MaterializedView handle carrying the parsed definition and refresh({full, sourceVersion}), which returns the typed refresh result. select accepts column names, [alias, expression] pairs, or a record of the same; the definition reads back off the stored schema, so a reopened handle needs no side channel. Remote connections surface the core's not-supported error up front. The napi crate needed the same recursion-limit raise as the core crate: the refresh future's type graph overflows the default trait-recursion depth. Stack created with GitHub Stacks CLI • Give Feedback 💬 --- docs/src/js/classes/Connection.md | 80 ++++++++- docs/src/js/classes/MaterializedView.md | 101 +++++++++++ docs/src/js/globals.md | 4 + .../interfaces/MaterializedViewDefinition.md | 59 +++++++ .../RefreshMaterializedViewResult.md | 41 +++++ .../js/type-aliases/MaterializedViewSelect.md | 14 ++ nodejs/__test__/embedding.test.ts | 48 ++++++ nodejs/__test__/materialized_view.test.ts | 147 ++++++++++++++++ nodejs/__test__/remote.test.ts | 19 +++ nodejs/__test__/table.test.ts | 2 +- nodejs/lancedb/connection.ts | 70 ++++++++ nodejs/lancedb/index.ts | 6 + nodejs/lancedb/materialized_view.ts | 161 ++++++++++++++++++ nodejs/lancedb/table.ts | 20 +++ nodejs/src/connection.rs | 52 ++++++ nodejs/src/lib.rs | 4 + nodejs/src/table.rs | 45 +++++ 17 files changed, 867 insertions(+), 6 deletions(-) create mode 100644 docs/src/js/classes/MaterializedView.md create mode 100644 docs/src/js/interfaces/MaterializedViewDefinition.md create mode 100644 docs/src/js/interfaces/RefreshMaterializedViewResult.md create mode 100644 docs/src/js/type-aliases/MaterializedViewSelect.md create mode 100644 nodejs/__test__/materialized_view.test.ts create mode 100644 nodejs/lancedb/materialized_view.ts diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index e4cbc1e96..92cfd2568 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -169,6 +169,45 @@ Creates a new empty Table *** +### createMaterializedView() + +```ts +abstract createMaterializedView( + name, + source, + options?): Promise +``` + +Define a materialized view named `name` over the table `source`. + +The view is created empty, with the query recorded in its schema +metadata; `view.refresh()` computes the rows. The view is a normal +table: it can be queried, indexed and searched, and it appears in +`tableNames`. The source table must have stable row ids (create it with +the `newTableEnableStableRowIds` storage option); they keep the view's +provenance valid across source compactions and cannot be enabled after +a table exists. Local databases only. + +#### Parameters + +* **name**: `string` + +* **source**: `string` + +* **options?** + +* **options.limit?**: `number` + +* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md) + +* **options.where?**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### createNamespace() ```ts @@ -499,6 +538,22 @@ List server-side jobs across the database's tables. *** +### listMaterializedViews() + +```ts +abstract listMaterializedViews(): Promise +``` + +The names of the materialized views in this database. + +Found by reading every table's schema, so this costs an open per table. + +#### Returns + +`Promise`<`string`[]> + +*** + ### listNamespaces() ```ts @@ -529,6 +584,26 @@ Child namespace names and *** +### openMaterializedView() + +```ts +abstract openMaterializedView(name): Promise +``` + +Open the materialized view named `name`. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### openTable() ```ts @@ -538,18 +613,13 @@ abstract openTable( options?): Promise ``` -Open a table in the database. - #### Parameters * **name**: `string` - The name of the table * **namespacePath?**: `string`[] - The namespace path of the table (defaults to root namespace) * **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)> - Additional options #### Returns diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md new file mode 100644 index 000000000..e6ff66142 --- /dev/null +++ b/docs/src/js/classes/MaterializedView.md @@ -0,0 +1,101 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedView + +# Class: MaterializedView + +A handle on a materialized view: its table plus its definition. + +Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or +[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table -- +queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table) +-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh). + +## Constructors + +### new MaterializedView() + +```ts +new MaterializedView(table): MaterializedView +``` + +#### Parameters + +* **table**: [`Table`](Table.md) + +#### Returns + +[`MaterializedView`](MaterializedView.md) + +## Accessors + +### name + +```ts +get name(): string +``` + +#### Returns + +`string` + +## Methods + +### definition() + +```ts +definition(): Promise +``` + +The query that defines the view, read from its stored schema. + +#### Returns + +`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)> + +*** + +### refresh() + +```ts +refresh(options?): Promise +``` + +Recompute the view from its source. + +The refresh is incremental when the source's changes can be reconciled +into the view -- rows added, changed or removed since the last one -- +and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` +refreshes to that source version instead of the latest. + +Concurrent refreshes of one view do not duplicate its rows. Two that +plan the same source rows conflict on commit, and the loser throws +rather than writing them a second time. + +#### Parameters + +* **options?** + +* **options.full?**: `boolean` + +* **options.sourceVersion?**: `number` + +#### Returns + +`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)> + +*** + +### table() + +```ts +table(): Table +``` + +The view, as the table it is. + +#### Returns + +[`Table`](Table.md) diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 4deeed1bc..6a8f644eb 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -28,6 +28,7 @@ - [Job](classes/Job.md) - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md) - [MatchQuery](classes/MatchQuery.md) +- [MaterializedView](classes/MaterializedView.md) - [MergeInsertBuilder](classes/MergeInsertBuilder.md) - [MultiMatchQuery](classes/MultiMatchQuery.md) - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md) @@ -101,6 +102,7 @@ - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) +- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) - [MemtableStats](interfaces/MemtableStats.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) @@ -110,6 +112,7 @@ - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) - [RefreshColumnResult](interfaces/RefreshColumnResult.md) +- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md) - [RemovalStats](interfaces/RemovalStats.md) - [RenameTableOptions](interfaces/RenameTableOptions.md) - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) @@ -142,6 +145,7 @@ - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) +- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md) - [MultiVector](type-aliases/MultiVector.md) - [RecordBatchLike](type-aliases/RecordBatchLike.md) - [SchemaLike](type-aliases/SchemaLike.md) diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md new file mode 100644 index 000000000..741bbba31 --- /dev/null +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -0,0 +1,59 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition + +# Interface: MaterializedViewDefinition + +The query that defines a materialized view. + +## Properties + +### filter? + +```ts +optional filter: string; +``` + +SQL predicate selecting the source rows the view holds. + +*** + +### inputs + +```ts +inputs: string[]; +``` + +Source columns the projections and filter read. + +*** + +### limit? + +```ts +optional limit: number; +``` + +Cap on the number of rows the view holds. + +*** + +### projections + +```ts +projections: [string, string][]; +``` + +`[output column, SQL expression]` pairs, in view schema order. + +*** + +### sourceTable + +```ts +sourceTable: string; +``` + +Name of the source table, in the same database as the view. diff --git a/docs/src/js/interfaces/RefreshMaterializedViewResult.md b/docs/src/js/interfaces/RefreshMaterializedViewResult.md new file mode 100644 index 000000000..cb7100cd8 --- /dev/null +++ b/docs/src/js/interfaces/RefreshMaterializedViewResult.md @@ -0,0 +1,41 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult + +# Interface: RefreshMaterializedViewResult + +## Properties + +### mode + +```ts +mode: string; +``` + +How the view was brought up to date: "rebuild", "incremental" or "no_op". + +*** + +### rowsWritten + +```ts +rowsWritten: number; +``` + +*** + +### sourceVersion + +```ts +sourceVersion: number; +``` + +*** + +### version + +```ts +version: number; +``` diff --git a/docs/src/js/type-aliases/MaterializedViewSelect.md b/docs/src/js/type-aliases/MaterializedViewSelect.md new file mode 100644 index 000000000..7b246e945 --- /dev/null +++ b/docs/src/js/type-aliases/MaterializedViewSelect.md @@ -0,0 +1,14 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewSelect + +# Type Alias: MaterializedViewSelect + +```ts +type MaterializedViewSelect: (string | [string, string])[] | Record; +``` + +The view's columns: column names, `[alias, SQL expression]` pairs, or a +record of the same. A bare name projects itself. diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index 06184751e..2a8494e0f 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -487,4 +487,52 @@ describe("embedding functions", () => { expect(stringSchema3).toEqual(stringExpectedSchema); }, ); + test("parses one function writing several vector columns", async () => { + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float32(); + } + async computeQueryEmbeddings(_data: string) { + return [1, 2, 3]; + } + async computeSourceEmbeddings(data: string[]) { + return Array.from({ length: data.length }).fill([ + 1, 2, 3, + ]) as number[][]; + } + } + const registry = getRegistry(); + registry.register("multi_output_mock")(MockEmbeddingFunction); + + // A materialized view can project one source vector column under two + // names, so a table's configuration names the same function twice. + const parsed = await registry.parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_a", + model: {}, + }, + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + + expect( + [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(), + ).toEqual(["vector_a", "vector_b"]); + }); }); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts new file mode 100644 index 000000000..2e7b2ec4d --- /dev/null +++ b/nodejs/__test__/materialized_view.test.ts @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as tmp from "tmp"; + +import { Connection, connect } from "../lancedb"; +import { + DEFINITION_META_KEY, + definitionFromMetadata, +} from "../lancedb/materialized_view"; + +describe("materialized views", () => { + let tmpDir: tmp.DirResult; + let db: Connection; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + db = await connect(tmpDir.name); + await db.createTable( + "people", + [ + { name: "ada", age: 36 }, + { name: "kid", age: 7 }, + { name: "grace", age: 85 }, + ], + { storageOptions: { newTableEnableStableRowIds: "true" } }, + ); + }); + afterEach(() => tmpDir.removeCallback()); + + it("rejects a stored limit a number cannot carry", () => { + const big = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":9007199254740993}', + ], + ]); + expect(() => definitionFromMetadata(big, "v")).toThrow( + /too large to represent exactly/, + ); + + const safe = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":42}', + ], + ]); + expect(definitionFromMetadata(safe, "v").limit).toBe(42); + }); + + it("creates, refreshes and queries a view", async () => { + const view = await db.createMaterializedView("adults", "people", { + select: ["name", ["shout", "upper(name)"]], + where: "age >= 18", + }); + expect(view.name).toBe("adults"); + expect(await view.table().countRows()).toBe(0); + + const result = await view.refresh(); + expect(result.mode).toBe("rebuild"); + expect(Number(result.rowsWritten)).toBe(2); + + const rows = await view.table().query().toArray(); + expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); + }); + + it("round-trips the definition", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + const view = await db.openMaterializedView("adults"); + const definition = await view.definition(); + expect(definition.sourceTable).toBe("people"); + expect(definition.filter).toBe("age >= 18"); + expect(definition.projections).toEqual([ + ["name", "`name`"], + ["age", "`age`"], + ]); + expect(definition.inputs).toEqual(["age", "name"]); + }); + + it("refreshes incrementally after an append", async () => { + const view = await db.createMaterializedView("copy", "people"); + await view.refresh(); + + const people = await db.openTable("people"); + await people.add([{ name: "alan", age: 41 }]); + const result = await view.refresh(); + expect(result.mode).toBe("incremental"); + expect(Number(result.rowsWritten)).toBe(1); + expect(await view.table().countRows()).toBe(4); + + expect((await view.refresh()).mode).toBe("no_op"); + }); + + it("lists views and rejects non-views", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + expect(await db.listMaterializedViews()).toEqual(["adults"]); + await expect(db.openMaterializedView("people")).rejects.toThrow( + "not a materialized view", + ); + }); + + it("rejects an invalid expression at create time", async () => { + await expect( + db.createMaterializedView("bad", "people", { + select: [["x", "missing + 1"]], + }), + ).rejects.toThrow("missing"); + }); + + it("rejects invalid numeric options before creating anything", async () => { + for (const limit of [-5, 1.5, Infinity, NaN]) { + await expect( + db.createMaterializedView("bad", "people", { limit }), + ).rejects.toThrow("non-negative integer"); + } + expect(await db.listMaterializedViews()).toEqual([]); + + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }); + + it("quotes bare select names", async () => { + await db.createTable("odd_names", [{ "order item": "widget" }], { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const view = await db.createMaterializedView("quoted", "odd_names", { + select: ["order item"], + }); + const result = await view.refresh(); + expect(Number(result.rowsWritten)).toBe(1); + }); + + it("requires stable row ids on the source", async () => { + await db.createTable("plain", [{ x: 1 }]); + await expect(db.createMaterializedView("v", "plain")).rejects.toThrow( + "stable row ids", + ); + }); +}); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 01320d2b0..c51cbbbb7 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -75,6 +75,25 @@ async function withMockDatabase( } describe("remote connection", () => { + it("refuses materialized views before issuing any request", async () => { + const paths: string[] = []; + await withMockDatabase( + (req, res) => { + paths.push(req.url ?? ""); + res.writeHead(404).end(); + }, + async (db) => { + await expect(db.openMaterializedView("secret_table")).rejects.toThrow( + /only on local databases/, + ); + await expect(db.listMaterializedViews()).rejects.toThrow( + /only on local databases/, + ); + expect(paths).toEqual([]); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 80c50f1ac..0f6ed3615 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2953,7 +2953,7 @@ describe("column name options", () => { .limit(10) .toArray(); expect(results2.length).toBe(10); - }); + }, 30_000); }); describe("when creating an empty table", () => { diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index a81dc0442..e5528f8c6 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -16,6 +16,12 @@ import { makeEmptyTable, } from "./arrow"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; +import { + MaterializedView, + MaterializedViewSelect, + normalizeSelect, + validateNonNegativeInteger, +} from "./materialized_view"; import { Connection as LanceDbConnection } from "./native"; import type { CreateNamespaceResponse, @@ -247,6 +253,41 @@ export abstract class Connection { * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace) * @param {Partial} options - Additional options */ + /** + * Define a materialized view named `name` over the table `source`. + * + * The view is created empty, with the query recorded in its schema + * metadata; `view.refresh()` computes the rows. The view is a normal + * table: it can be queried, indexed and searched, and it appears in + * `tableNames`. The source table must have stable row ids (create it with + * the `newTableEnableStableRowIds` storage option); they keep the view's + * provenance valid across source compactions and cannot be enabled after + * a table exists. Local databases only. + */ + abstract createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise; + + /** + * Open the materialized view named `name`. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract openMaterializedView(name: string): Promise; + + /** + * The names of the materialized views in this database. + * + * Found by reading every table's schema, so this costs an open per table. + */ + abstract listMaterializedViews(): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -531,6 +572,35 @@ export class LocalConnection extends Connection { ); } + async createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise { + validateNonNegativeInteger(options?.limit, "limit"); + const innerTable = await this.inner.createMaterializedView( + name, + source, + normalizeSelect(options?.select), + options?.where, + options?.limit, + ); + return new MaterializedView(new LocalTable(innerTable)); + } + + async openMaterializedView(name: string): Promise { + const innerTable = await this.inner.openMaterializedView(name); + return new MaterializedView(new LocalTable(innerTable)); + } + + async listMaterializedViews(): Promise { + return await this.inner.listMaterializedViews(); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index aa9c7ef13..da13b434f 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices"; import type { FtsToken } from "./table"; // Re-export native header provider for use with connectWithHeaderProvider +export { + MaterializedView, + MaterializedViewDefinition, + MaterializedViewSelect, +} from "./materialized_view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the @@ -51,6 +56,7 @@ export { AddResult, AddColumnsResult, RefreshColumnResult, + RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts new file mode 100644 index 000000000..b26dee59e --- /dev/null +++ b/nodejs/lancedb/materialized_view.ts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { RefreshMaterializedViewResult } from "./native"; +import { Table } from "./table"; + +/** Schema metadata key holding a materialized view's definition. */ +export const DEFINITION_META_KEY = "mv.definition"; + +/** The query that defines a materialized view. */ +export interface MaterializedViewDefinition { + /** Name of the source table, in the same database as the view. */ + sourceTable: string; + /** `[output column, SQL expression]` pairs, in view schema order. */ + projections: [string, string][]; + /** SQL predicate selecting the source rows the view holds. */ + filter?: string; + /** Cap on the number of rows the view holds. */ + limit?: number; + /** Source columns the projections and filter read. */ + inputs: string[]; +} + +/** + * The view's columns: column names, `[alias, SQL expression]` pairs, or a + * record of the same. A bare name projects itself. + */ +export type MaterializedViewSelect = + | (string | [string, string])[] + | Record; + +/** + * @internal Reject a numeric option N-API would otherwise silently coerce: + * `Infinity` reaches Rust as 0, `1.5` as 1. + */ +export function validateNonNegativeInteger( + value: number | undefined, + name: string, +): void { + if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) { + throw new Error(`${name} must be a non-negative integer`); + } +} + +/** @internal Quote a column name as a Lance SQL identifier (backticks). */ +function quoteIdentifier(name: string): string { + return "`" + name.replace(/`/g, "``") + "`"; +} + +/** + * @internal Normalize a select argument into `[alias, expression]` pairs. + * A bare name projects itself and is quoted, so any valid column name works; + * pair and record entries are kept verbatim because their right side is an + * expression. + */ +export function normalizeSelect( + select?: MaterializedViewSelect, +): [string, string][] | undefined { + if (select === undefined) { + return undefined; + } + if (Array.isArray(select)) { + return select.map((item) => + typeof item === "string" ? [item, quoteIdentifier(item)] : item, + ); + } + return Object.entries(select); +} + +/** @internal Parse a definition off a table's stored schema metadata. */ +export function definitionFromMetadata( + metadata: Map, + name: string, +): MaterializedViewDefinition { + const raw = metadata.get(DEFINITION_META_KEY); + if (raw === undefined) { + throw new Error(`Table '${name}' is not a materialized view`); + } + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + const value: any = JSON.parse(raw); + if (value.kind !== "select") { + throw new Error( + `materialized view '${name}' is defined by '${value.kind}', which this ` + + "version of lancedb cannot refresh", + ); + } + const limit = value.limit ?? undefined; + // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe + // integer and every rounded one does not, so this rejects precisely the + // values a number cannot carry. + if (limit !== undefined && !Number.isSafeInteger(limit)) { + throw new Error( + `materialized view '${name}' has a stored limit too large to represent exactly`, + ); + } + return { + sourceTable: value.source_table, + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + projections: (value.projections ?? []).map((p: any) => [ + p.output, + p.expression, + ]), + filter: value.filter ?? undefined, + limit, + inputs: value.inputs ?? [], + }; +} + +/** + * A handle on a materialized view: its table plus its definition. + * + * Obtained from {@link Connection#createMaterializedView} or + * {@link Connection#openMaterializedView}. The view is a normal table -- + * queries, indexes and search all apply through {@link MaterializedView#table} + * -- whose contents are maintained by {@link MaterializedView#refresh}. + */ +export class MaterializedView { + private readonly inner: Table; + + constructor(table: Table) { + this.inner = table; + } + + get name(): string { + return this.inner.name; + } + + /** The view, as the table it is. */ + table(): Table { + return this.inner; + } + + /** The query that defines the view, read from its stored schema. */ + async definition(): Promise { + const schema = await this.inner.schema(); + return definitionFromMetadata(schema.metadata, this.name); + } + + /** + * Recompute the view from its source. + * + * The refresh is incremental when the source's changes can be reconciled + * into the view -- rows added, changed or removed since the last one -- + * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` + * refreshes to that source version instead of the latest. + * + * Concurrent refreshes of one view do not duplicate its rows. Two that + * plan the same source rows conflict on commit, and the loser throws + * rather than writing them a second time. + */ + async refresh(options?: { + full?: boolean; + sourceVersion?: number; + }): Promise { + validateNonNegativeInteger(options?.sourceVersion, "sourceVersion"); + return await this.inner.refreshMaterializedView( + options?.full, + options?.sourceVersion, + ); + } +} diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 4e323ec64..28603cca9 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,6 +35,7 @@ import { Branches as NativeBranches, OptimizeStats, RefreshColumnResult, + RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, @@ -602,6 +603,18 @@ export abstract class Table { */ abstract refreshColumnAsync(column: string): Promise; + /** + * Recompute this table's contents from its materialized-view definition. + * + * Plumbing for {@link MaterializedView.refresh}, which is the way to call + * it: rejects tables that carry no view definition. Local tables only. + * @ignore + */ + abstract refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1264,6 +1277,13 @@ export class LocalTable extends Table { return await this.inner.refreshColumnAsync(column); } + async refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise { + return await this.inner.refreshMaterializedView(full, sourceVersion); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index c9f5e10ea..44eb68f32 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -266,6 +266,58 @@ impl Connection { Ok(Table::new(tbl)) } + #[napi(catch_unwind)] + pub async fn create_materialized_view( + &self, + name: String, + source: String, + projections: Option>>, + filter: Option, + limit: Option, + ) -> napi::Result
{ + let mut builder = self.get_inner()?.create_materialized_view(name, source); + if let Some(projections) = projections { + let mut pairs = Vec::with_capacity(projections.len()); + for pair in projections { + let [output, expression]: [String; 2] = pair.try_into().map_err(|_| { + napi::Error::from_reason("each projection must be an [output, expression] pair") + })?; + pairs.push((output, expression)); + } + builder = builder.select(pairs); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + let limit = u64::try_from(limit) + .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?; + builder = builder.limit(limit); + } + let view = builder.execute().await.default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn open_materialized_view(&self, name: String) -> napi::Result
{ + let view = self + .get_inner()? + .open_materialized_view(&name) + .await + .default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn list_materialized_views(&self) -> napi::Result> { + let views = self + .get_inner()? + .list_materialized_views() + .await + .default_error()?; + Ok(views.into_iter().map(|v| v.name).collect()) + } + #[napi(catch_unwind)] pub async fn open_table( &self, diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 312b675bd..1110f6203 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +// The materialized-view refresh future deepens the type graph past the +// default trait-recursion depth; same raise as the core crate applies. +#![recursion_limit = "256"] + use std::collections::HashMap; use env_logger::Env; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 694b6a704..9d60b2056 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -381,6 +381,26 @@ impl Table { Ok(crate::job::Job::new(job)) } + #[napi(catch_unwind)] + pub async fn refresh_materialized_view( + &self, + full: Option, + source_version: Option, + ) -> napi::Result { + let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone()) + .await + .default_error()?; + let mut builder = view.refresh().full(full.unwrap_or(false)); + if let Some(version) = source_version { + let version = u64::try_from(version).map_err(|_| { + napi::Error::from_reason("sourceVersion must be a non-negative integer") + })?; + builder = builder.source_version(version); + } + let result = builder.execute().await.default_error()?; + Ok(result.into()) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, @@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult { pub version: i64, } +#[napi(object)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date: "rebuild", "incremental" or "no_op". + pub mode: String, + pub rows_written: i64, + pub source_version: i64, + pub version: i64, +} + +impl From for RefreshMaterializedViewResult { + fn from(value: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match value.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: value.rows_written as i64, + source_version: value.source_version as i64, + version: value.version as i64, + } + } +} + impl From for RefreshColumnResult { fn from(value: lancedb::table::RefreshColumnResult) -> Self { Self {