diff --git a/.bumpversion.toml b/.bumpversion.toml index b1d2bf1dc..86a5064f7 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.3" +current_version = "0.38.0-beta.8" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 73f5923fe..8d0058569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -4888,8 +4888,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4911,7 +4911,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4925,7 +4925,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -4934,8 +4934,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrayref", "crunchy", @@ -4945,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -4983,8 +4983,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5013,8 +5013,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "proc-macro2", "quote", @@ -5041,8 +5041,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5075,8 +5075,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-arith", "arrow-array", @@ -5107,8 +5107,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arc-swap", "arrow", @@ -5172,8 +5172,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5195,8 +5195,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5222,7 +5222,11 @@ dependencies = [ "pin-project", "prost", "rand 0.9.5", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", "serde", + "serde_json", "tempfile", "tokio", "tracing", @@ -5232,8 +5236,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5247,8 +5251,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "async-trait", @@ -5260,8 +5264,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-ipc", @@ -5314,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-buffer", @@ -5329,8 +5333,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow", "arrow-array", @@ -5370,8 +5374,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "arrow-array", "arrow-schema", @@ -5384,8 +5388,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "11.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65" +version = "11.0.0-beta.22" +source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20" dependencies = [ "frostem", "icu_segmenter", @@ -5398,7 +5402,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5490,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5515,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" dependencies = [ "arrow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 9aeccd76f..716b14d7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a8869d319..28de11c30 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.3 + 0.38.0-beta.8 ``` diff --git a/docs/src/js/classes/Branches.md b/docs/src/js/classes/Branches.md index 5296d0d08..6680fbfee 100644 --- a/docs/src/js/classes/Branches.md +++ b/docs/src/js/classes/Branches.md @@ -37,6 +37,31 @@ latest and stays writable. *** +### cherryPick() + +```ts +cherryPick(fromBranch, dryRun): Promise +``` + +Cherry-pick a branch onto main. + +Set `dryRun` to `true` to preview. A failed cherry-pick resolves +with `status: "failed"` instead of throwing. + +#### Parameters + +* **fromBranch**: `string` + Branch to cherry-pick from. + +* **dryRun**: `boolean` = `false` + When true, only preview. Defaults to false. + +#### Returns + +`Promise`<[`CherryPickResult`](../interfaces/CherryPickResult.md)> + +*** + ### create() ```ts @@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata. #### Returns `Promise`<`Record`<`string`, [`BranchContents`](BranchContents.md)>> - -*** - -### merge() - -```ts -merge(fromBranch, dryRun): Promise -``` - -Merge a branch into main. - -Set `dryRun` to `true` to preview the merge. A rejected merge resolves -with `status: "rejected"` instead of throwing. - -#### Parameters - -* **fromBranch**: `string` - Branch to merge from. - -* **dryRun**: `boolean` = `false` - When true, only preview the merge. Defaults to false. - -#### Returns - -`Promise`<[`MergeBranchResult`](../interfaces/MergeBranchResult.md)> diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index e4cbc1e96..1c5abd89f 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -169,6 +169,45 @@ Creates a new empty Table *** +### createMaterializedView() + +```ts +abstract createMaterializedView( + name, + source, + options?): Promise +``` + +Define a materialized view named `name` over the table `source`. + +The view is created empty, with the query recorded in its schema +metadata; `view.refresh()` computes the rows. The view is a normal +table: it can be queried, indexed and searched, and it appears in +`tableNames`. The source table must have stable row ids (create it with +the `newTableEnableStableRowIds` storage option); they keep the view's +provenance valid across source compactions and cannot be enabled after +a table exists. Local databases only. + +#### Parameters + +* **name**: `string` + +* **source**: `string` + +* **options?** + +* **options.limit?**: `number` + +* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md) + +* **options.where?**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### createNamespace() ```ts @@ -499,6 +538,22 @@ List server-side jobs across the database's tables. *** +### listMaterializedViews() + +```ts +abstract listMaterializedViews(): Promise +``` + +The names of the materialized views in this database. + +Found by reading every table's schema, so this costs an open per table. + +#### Returns + +`Promise`<`string`[]> + +*** + ### listNamespaces() ```ts @@ -529,6 +584,90 @@ Child namespace names and *** +### listTables() + +#### listTables(options) + +```ts +abstract listTables(options?): Promise +``` + +List a page of the tables in this database. + +To retrieve the tables after the page, pass the `pageToken` the response +carries back in. A page can be shorter than `limit` without being the last +one, so walk until a response carries no page token: + +```ts +const names = []; +let pageToken = undefined; +do { + const page = await conn.listTables({ pageToken, limit: 100 }); + names.push(...page.tables); + pageToken = page.pageToken; +} while (pageToken); +``` + +##### Parameters + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +#### listTables(namespacePath, options) + +```ts +abstract listTables(namespacePath?, options?): Promise +``` + +List a page of the tables in this database. + +##### Parameters + +* **namespacePath?**: `string`[] + The namespace path to list tables from + (defaults to root namespace) + +* **options?**: `Partial`<[`ListTablesOptions`](../interfaces/ListTablesOptions.md)> + Pagination options + (`pageToken`, `limit`). + +##### Returns + +`Promise`<[`ListTablesResponse`](../interfaces/ListTablesResponse.md)> + +A page of table names and an + optional token for the tables after it. + +*** + +### openMaterializedView() + +```ts +abstract openMaterializedView(name): Promise +``` + +Open the materialized view named `name`. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +#### Returns + +`Promise`<[`MaterializedView`](MaterializedView.md)> + +*** + ### openTable() ```ts @@ -538,18 +677,13 @@ abstract openTable( options?): Promise ``` -Open a table in the database. - #### Parameters * **name**: `string` - The name of the table * **namespacePath?**: `string`[] - The namespace path of the table (defaults to root namespace) * **options?**: `Partial`<[`OpenTableOptions`](../interfaces/OpenTableOptions.md)> - Additional options #### Returns @@ -590,7 +724,7 @@ a "not supported" error. *** -### tableNames() +### ~~tableNames()~~ #### tableNames(options) @@ -612,6 +746,10 @@ Tables will be returned in lexicographical order. `Promise`<`string`[]> +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. + #### tableNames(namespacePath, options) ```ts @@ -634,3 +772,7 @@ Tables will be returned in lexicographical order. ##### Returns `Promise`<`string`[]> + +##### Deprecated + +Use [Connection.listTables](Connection.md#listtables) instead. diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md new file mode 100644 index 000000000..e6ff66142 --- /dev/null +++ b/docs/src/js/classes/MaterializedView.md @@ -0,0 +1,101 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedView + +# Class: MaterializedView + +A handle on a materialized view: its table plus its definition. + +Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or +[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table -- +queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table) +-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh). + +## Constructors + +### new MaterializedView() + +```ts +new MaterializedView(table): MaterializedView +``` + +#### Parameters + +* **table**: [`Table`](Table.md) + +#### Returns + +[`MaterializedView`](MaterializedView.md) + +## Accessors + +### name + +```ts +get name(): string +``` + +#### Returns + +`string` + +## Methods + +### definition() + +```ts +definition(): Promise +``` + +The query that defines the view, read from its stored schema. + +#### Returns + +`Promise`<[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)> + +*** + +### refresh() + +```ts +refresh(options?): Promise +``` + +Recompute the view from its source. + +The refresh is incremental when the source's changes can be reconciled +into the view -- rows added, changed or removed since the last one -- +and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` +refreshes to that source version instead of the latest. + +Concurrent refreshes of one view do not duplicate its rows. Two that +plan the same source rows conflict on commit, and the loser throws +rather than writing them a second time. + +#### Parameters + +* **options?** + +* **options.full?**: `boolean` + +* **options.sourceVersion?**: `number` + +#### Returns + +`Promise`<[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)> + +*** + +### table() + +```ts +table(): Table +``` + +The view, as the table it is. + +#### Returns + +[`Table`](Table.md) diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 462907cfd..e0635ab65 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -28,6 +28,7 @@ - [Job](classes/Job.md) - [MakeArrowTableOptions](classes/MakeArrowTableOptions.md) - [MatchQuery](classes/MatchQuery.md) +- [MaterializedView](classes/MaterializedView.md) - [MergeInsertBuilder](classes/MergeInsertBuilder.md) - [MultiMatchQuery](classes/MultiMatchQuery.md) - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md) @@ -59,6 +60,9 @@ - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) - [BucketStats](interfaces/BucketStats.md) +- [CherryPickError](interfaces/CherryPickError.md) +- [CherryPickPreview](interfaces/CherryPickPreview.md) +- [CherryPickResult](interfaces/CherryPickResult.md) - [ClientConfig](interfaces/ClientConfig.md) - [ColumnAlteration](interfaces/ColumnAlteration.md) - [ColumnOrdering](interfaces/ColumnOrdering.md) @@ -96,12 +100,12 @@ - [JobInfo](interfaces/JobInfo.md) - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md) - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) +- [ListTablesOptions](interfaces/ListTablesOptions.md) +- [ListTablesResponse](interfaces/ListTablesResponse.md) - [LsmStats](interfaces/LsmStats.md) - [LsmWriteSpec](interfaces/LsmWriteSpec.md) +- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md) - [MemtableStats](interfaces/MemtableStats.md) -- [MergeBlocker](interfaces/MergeBlocker.md) -- [MergeBranchResult](interfaces/MergeBranchResult.md) -- [MergePreview](interfaces/MergePreview.md) - [MergeResult](interfaces/MergeResult.md) - [NativeOAuthConfig](interfaces/NativeOAuthConfig.md) - [OAuthConfig](interfaces/OAuthConfig.md) @@ -110,6 +114,7 @@ - [OptimizeStats](interfaces/OptimizeStats.md) - [QueryExecutionOptions](interfaces/QueryExecutionOptions.md) - [RefreshColumnResult](interfaces/RefreshColumnResult.md) +- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md) - [RemovalStats](interfaces/RemovalStats.md) - [RenameTableOptions](interfaces/RenameTableOptions.md) - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) @@ -142,6 +147,7 @@ - [FieldLike](type-aliases/FieldLike.md) - [IntoSql](type-aliases/IntoSql.md) - [IntoVector](type-aliases/IntoVector.md) +- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md) - [MultiVector](type-aliases/MultiVector.md) - [RecordBatchLike](type-aliases/RecordBatchLike.md) - [SchemaLike](type-aliases/SchemaLike.md) diff --git a/docs/src/js/interfaces/BranchDiff.md b/docs/src/js/interfaces/BranchDiff.md index 224be1991..b408ebe11 100644 --- a/docs/src/js/interfaces/BranchDiff.md +++ b/docs/src/js/interfaces/BranchDiff.md @@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[]; *** +### errors + +```ts +errors: CherryPickError[]; +``` + +*** + ### fromBranch ```ts @@ -66,22 +74,6 @@ mainVersion: number; *** -### mergeBlockers - -```ts -mergeBlockers: MergeBlocker[]; -``` - -*** - -### mergeable - -```ts -mergeable: boolean; -``` - -*** - ### parentVersion ```ts diff --git a/docs/src/js/interfaces/MergeBlocker.md b/docs/src/js/interfaces/CherryPickError.md similarity index 54% rename from docs/src/js/interfaces/MergeBlocker.md rename to docs/src/js/interfaces/CherryPickError.md index 6c8f84b37..84f8fe012 100644 --- a/docs/src/js/interfaces/MergeBlocker.md +++ b/docs/src/js/interfaces/CherryPickError.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBlocker +[@lancedb/lancedb](../globals.md) / CherryPickError -# Interface: MergeBlocker +# Interface: CherryPickError -A reason why a branch cannot currently be merged. +A reason why a cherry-pick cannot currently land. ## Properties diff --git a/docs/src/js/interfaces/CherryPickPreview.md b/docs/src/js/interfaces/CherryPickPreview.md new file mode 100644 index 000000000..9620068a2 --- /dev/null +++ b/docs/src/js/interfaces/CherryPickPreview.md @@ -0,0 +1,17 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / CherryPickPreview + +# Interface: CherryPickPreview + +Changes that would be, or were, promoted by a cherry-pick. + +## Properties + +### promotedColumns + +```ts +promotedColumns: string[]; +``` diff --git a/docs/src/js/interfaces/MergeBranchResult.md b/docs/src/js/interfaces/CherryPickResult.md similarity index 60% rename from docs/src/js/interfaces/MergeBranchResult.md rename to docs/src/js/interfaces/CherryPickResult.md index 61c2d0d33..c837ab143 100644 --- a/docs/src/js/interfaces/MergeBranchResult.md +++ b/docs/src/js/interfaces/CherryPickResult.md @@ -2,11 +2,11 @@ *** -[@lancedb/lancedb](../globals.md) / MergeBranchResult +[@lancedb/lancedb](../globals.md) / CherryPickResult -# Interface: MergeBranchResult +# Interface: CherryPickResult -Result of previewing or attempting a branch merge. +Result of previewing or attempting a cherry-pick. ## Properties @@ -29,7 +29,7 @@ optional mainVersionAfter: number; ### preview ```ts -preview: MergePreview; +preview: CherryPickPreview; ``` *** @@ -38,9 +38,9 @@ preview: MergePreview; ```ts status: + | "failed" | "unknown" - | "rejected" | "ready" | "notImplemented" - | "merged"; + | "cherryPicked"; ``` diff --git a/docs/src/js/interfaces/ListTablesOptions.md b/docs/src/js/interfaces/ListTablesOptions.md new file mode 100644 index 000000000..52ace47e5 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesOptions.md @@ -0,0 +1,34 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesOptions + +# Interface: ListTablesOptions + +## Properties + +### limit? + +```ts +optional limit: number; +``` + +An upper bound on how many tables to return. + +A page may hold fewer than this and still not be the last one, so keep +going while the response carries a page token rather than while pages are +full. + +*** + +### pageToken? + +```ts +optional pageToken: string; +``` + +Token from a previous response, to resume listing where it left off. + +The token is opaque: it carries whatever the database needs to resume, and +callers should not construct or interpret one. diff --git a/docs/src/js/interfaces/ListTablesResponse.md b/docs/src/js/interfaces/ListTablesResponse.md new file mode 100644 index 000000000..76cac2b23 --- /dev/null +++ b/docs/src/js/interfaces/ListTablesResponse.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListTablesResponse + +# Interface: ListTablesResponse + +## Properties + +### pageToken? + +```ts +optional pageToken: string; +``` + +*** + +### tables + +```ts +tables: string[]; +``` diff --git a/docs/src/js/interfaces/MaterializedViewDefinition.md b/docs/src/js/interfaces/MaterializedViewDefinition.md new file mode 100644 index 000000000..741bbba31 --- /dev/null +++ b/docs/src/js/interfaces/MaterializedViewDefinition.md @@ -0,0 +1,59 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition + +# Interface: MaterializedViewDefinition + +The query that defines a materialized view. + +## Properties + +### filter? + +```ts +optional filter: string; +``` + +SQL predicate selecting the source rows the view holds. + +*** + +### inputs + +```ts +inputs: string[]; +``` + +Source columns the projections and filter read. + +*** + +### limit? + +```ts +optional limit: number; +``` + +Cap on the number of rows the view holds. + +*** + +### projections + +```ts +projections: [string, string][]; +``` + +`[output column, SQL expression]` pairs, in view schema order. + +*** + +### sourceTable + +```ts +sourceTable: string; +``` + +Name of the source table, in the same database as the view. diff --git a/docs/src/js/interfaces/MergePreview.md b/docs/src/js/interfaces/MergePreview.md deleted file mode 100644 index 0d9717289..000000000 --- a/docs/src/js/interfaces/MergePreview.md +++ /dev/null @@ -1,17 +0,0 @@ -[**@lancedb/lancedb**](../README.md) • **Docs** - -*** - -[@lancedb/lancedb](../globals.md) / MergePreview - -# Interface: MergePreview - -Changes that would be, or were, promoted by a branch merge. - -## Properties - -### promotedColumns - -```ts -promotedColumns: string[]; -``` diff --git a/docs/src/js/interfaces/RefreshMaterializedViewResult.md b/docs/src/js/interfaces/RefreshMaterializedViewResult.md new file mode 100644 index 000000000..cb7100cd8 --- /dev/null +++ b/docs/src/js/interfaces/RefreshMaterializedViewResult.md @@ -0,0 +1,41 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult + +# Interface: RefreshMaterializedViewResult + +## Properties + +### mode + +```ts +mode: string; +``` + +How the view was brought up to date: "rebuild", "incremental" or "no_op". + +*** + +### rowsWritten + +```ts +rowsWritten: number; +``` + +*** + +### sourceVersion + +```ts +sourceVersion: number; +``` + +*** + +### version + +```ts +version: number; +``` diff --git a/docs/src/js/interfaces/TableNamesOptions.md b/docs/src/js/interfaces/TableNamesOptions.md index 45fa3d1b0..9254e9fa7 100644 --- a/docs/src/js/interfaces/TableNamesOptions.md +++ b/docs/src/js/interfaces/TableNamesOptions.md @@ -4,11 +4,16 @@ [@lancedb/lancedb](../globals.md) / TableNamesOptions -# Interface: TableNamesOptions +# Interface: ~~TableNamesOptions~~ + +## Deprecated + +Use [ListTablesOptions](ListTablesOptions.md) with [Connection.listTables](../classes/Connection.md#listtables) +instead. ## Properties -### limit? +### ~~limit?~~ ```ts optional limit: number; @@ -18,7 +23,7 @@ An optional limit to the number of results to return. *** -### startAfter? +### ~~startAfter?~~ ```ts optional startAfter: string; diff --git a/docs/src/js/type-aliases/MaterializedViewSelect.md b/docs/src/js/type-aliases/MaterializedViewSelect.md new file mode 100644 index 000000000..7b246e945 --- /dev/null +++ b/docs/src/js/type-aliases/MaterializedViewSelect.md @@ -0,0 +1,14 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / MaterializedViewSelect + +# Type Alias: MaterializedViewSelect + +```ts +type MaterializedViewSelect: (string | [string, string])[] | Record; +``` + +The view's columns: column names, `[alias, SQL expression]` pairs, or a +record of the same. A bare name projects itself. diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 70b2a7207..3cbeee6f0 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -102,6 +102,12 @@ listing a storage directory. ::: lancedb.job.AsyncJob +## Materialized Views (Synchronous) + +::: lancedb.materialized_view.MaterializedView + +::: lancedb.materialized_view.MaterializedViewDefinition + ## Expressions Type-safe expression builder for filters and projections. Use these instead @@ -255,6 +261,8 @@ instead of being materialized with the rest of the row. ::: lancedb.streaming.StreamingDataset +::: lancedb.streaming.StreamingDataLoader + ::: lancedb.permutation.permutation_builder ::: lancedb.permutation.PermutationBuilder @@ -295,6 +303,10 @@ Table hold your actual data as a collection of records / rows. ::: lancedb.table.AsyncBranches +## Materialized Views (Asynchronous) + +::: lancedb.materialized_view.AsyncMaterializedView + ## Indices (Asynchronous) Indices can be created on a table to speed up queries. This section diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 9de14d1f9..6f4b0744b 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.8 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 978f7c7c7..ac6196a45 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.3 + 0.38.0-beta.8 pom ${project.artifactId} LanceDB Java SDK Parent POM @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 11.0.0-beta.18 + 11.0.0-beta.22 false 2.30.0 1.7 diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index cba7012d1..e85da66ec 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 160f4d5ef..c5bbbf169 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -515,6 +515,137 @@ describe.each([arrow15, arrow16, arrow17, arrow18])( ); }); + it("will allow matching inferred types across records", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: 2 }]), + ).not.toThrow(); + }); + + it("will reject mismatched inferred types across records", function () { + expect(() => makeArrowTable([{ value: 1 }, { value: "two" }])).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Utf8 for field value at row 1. Consider providing an explicit schema.", + ); + }); + + it("will ignore generated dictionary IDs when comparing inferred types", function () { + const table = makeArrowTable([{ str: "a" }, { str: "b" }], { + dictionaryEncodeStrings: true, + }); + + expect(table.getChild("str")?.toJSON()).toEqual(["a", "b"]); + }); + + it("will preserve null values without treating them as type mismatches", function () { + for (const records of [ + [{ vector: [1, 2, 3] }, { vector: null }], + [{ vector: null }, { vector: [1, 2, 3] }], + ]) { + const table = makeArrowTable(records); + + expect(table.numRows).toBe(2); + expect(table.getChild("vector")?.nullCount).toBe(1); + } + }); + + it("will preserve empty variable-size lists", function () { + for (const records of [ + [{ items: [1] }, { items: [] }], + [{ items: [] }, { items: [1] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + }); + + it("will propagate deferred evidence through nested lists", function () { + for (const records of [ + [{ items: [1] }, { items: [null] }], + [{ items: [null] }, { items: [1] }], + [{ items: [null, 1] }, { items: [2, null] }], + ]) { + const table = makeArrowTable(records); + expect( + table + .getChild("items") + ?.toJSON() + .map((value) => value.toJSON()), + ).toEqual(records.map((record) => record.items)); + } + + const nestedRecords = [{ items: [[1]] }, { items: [[null]] }]; + const nestedTable = makeArrowTable(nestedRecords); + expect( + nestedTable + .getChild("items") + ?.toJSON() + .map((value) => + value + .toJSON() + .map((nestedValue: { toJSON: () => unknown[] }) => + nestedValue.toJSON(), + ), + ), + ).toEqual(nestedRecords.map((record) => record.items)); + }); + + it("will reject incompatible deferred evidence within a list", function () { + for (const items of [ + [[], 1], + [1, []], + [[null], 1], + [1, [null]], + ]) { + expect(() => makeArrowTable([{ items }])).toThrow( + "Failed to infer data type for field items at row 0.", + ); + } + }); + + it("will reject empty fixed-size lists", function () { + expect(() => + makeArrowTable([{ vector: [1, 2, 3] }, { vector: [] }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type FixedSizeList[3] but found List[0] for field vector at row 1.", + ); + }); + + it("will reject inferred leaf and branch shape changes", function () { + expect(() => + makeArrowTable([{ value: 1 }, { value: { nested: 2 } }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Float64 but found Struct for field value at row 1.", + ); + expect(() => + makeArrowTable([{ value: { nested: 1 } }, { value: 2 }]), + ).toThrow( + "Failed to infer schema for data. Previously inferred type Struct but found Float64 for field value at row 1.", + ); + }); + + it("will allow null values around inferred struct values", function () { + for (const { records, nullIndex } of [ + { + records: [{ value: null }, { value: { nested: 2 } }], + nullIndex: 0, + }, + { + records: [{ value: { nested: 1 } }, { value: null }], + nullIndex: 1, + }, + ]) { + const table = makeArrowTable(records); + const values = table.getChild("value"); + + expect(values?.nullCount).toBe(1); + expect(values?.get(nullIndex)).toBeNull(); + } + }); + it("will allow a schema to be provided", async function () { await checkTableCreation( async (records, _, schema) => diff --git a/nodejs/__test__/connection.test.ts b/nodejs/__test__/connection.test.ts index af471b478..816f8c3de 100644 --- a/nodejs/__test__/connection.test.ts +++ b/nodejs/__test__/connection.test.ts @@ -4,7 +4,13 @@ import { readdirSync } from "fs"; import { Field, Float64, Schema } from "apache-arrow"; import * as tmp from "tmp"; -import { Connection, Table, connect, connectNamespace } from "../lancedb"; +import { + Connection, + ListTablesResponse, + Table, + connect, + connectNamespace, +} from "../lancedb"; import { LocalTable } from "../lancedb/table"; describe("when connecting", () => { @@ -47,6 +53,7 @@ describe("given a connection", () => { await db.close(); expect(db.isOpen()).toBe(false); await expect(db.tableNames()).rejects.toThrow("Connection is closed"); + await expect(db.listTables()).rejects.toThrow("Connection is closed"); await expect(db.renameTable("a", "b")).rejects.toThrow( "Connection is closed", ); @@ -129,6 +136,66 @@ describe("given a connection", () => { expect(tables).toEqual(["b", "c"]); }); + it("should respect limit and page token when listing tables", async () => { + const db = await connect(tmpDir.name); + + await db.createTable("b", [{ id: 1 }]); + await db.createTable("a", [{ id: 1 }]); + await db.createTable("c", [{ id: 1 }]); + + const all = await db.listTables(); + expect(all.tables).toEqual(["a", "b", "c"]); + expect(all.pageToken).toBeUndefined(); + + const first = await db.listTables({ limit: 1 }); + expect(first.tables).toEqual(["a"]); + expect(first.pageToken).toBeDefined(); + + const second = await db.listTables({ + limit: 1, + pageToken: first.pageToken, + }); + expect(second.tables).toEqual(["b"]); + }); + + it("should visit every table exactly once when walking pages", async () => { + const db = await connect(tmpDir.name); + + const created = ["a", "b", "c", "d", "e"]; + for (const name of created) { + await db.createTable(name, [{ id: 1 }]); + } + + const seen: string[] = []; + let pageToken: string | undefined = undefined; + do { + const page: ListTablesResponse = await db.listTables({ + limit: 2, + pageToken, + }); + seen.push(...page.tables); + pageToken = page.pageToken; + } while (pageToken); + + expect(seen).toEqual(created); + }); + + it("should list tables in a namespace", async () => { + const db = await connect(tmpDir.name, { + // biome-ignore lint/style/useNamingConvention: opaque backend property key, must match Rust + namespaceClientProperties: { manifest_enabled: "true" }, + }); + await db.createNamespace(["child"]); + await db.createTable("nested", [{ id: 1 }], ["child"]); + + await expect(db.listTables(["child"])).resolves.toEqual( + expect.objectContaining({ tables: ["nested"] }), + ); + await expect(db.listTables()).resolves.toEqual( + expect.objectContaining({ tables: [] }), + ); + }); + it("should create tables in v2 mode", async () => { const db = await connect(tmpDir.name); const data = [...Array(10000).keys()].map((i) => ({ id: i })); diff --git a/nodejs/__test__/embedding.test.ts b/nodejs/__test__/embedding.test.ts index 06184751e..2a8494e0f 100644 --- a/nodejs/__test__/embedding.test.ts +++ b/nodejs/__test__/embedding.test.ts @@ -487,4 +487,52 @@ describe("embedding functions", () => { expect(stringSchema3).toEqual(stringExpectedSchema); }, ); + test("parses one function writing several vector columns", async () => { + class MockEmbeddingFunction extends EmbeddingFunction { + ndims() { + return 3; + } + embeddingDataType(): Float { + return new Float32(); + } + async computeQueryEmbeddings(_data: string) { + return [1, 2, 3]; + } + async computeSourceEmbeddings(data: string[]) { + return Array.from({ length: data.length }).fill([ + 1, 2, 3, + ]) as number[][]; + } + } + const registry = getRegistry(); + registry.register("multi_output_mock")(MockEmbeddingFunction); + + // A materialized view can project one source vector column under two + // names, so a table's configuration names the same function twice. + const parsed = await registry.parseFunctions( + new Map([ + [ + "embedding_functions", + JSON.stringify([ + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_a", + model: {}, + }, + { + name: "multi_output_mock", + sourceColumn: "text", + vectorColumn: "vector_b", + model: {}, + }, + ]), + ], + ]), + ); + + expect( + [...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(), + ).toEqual(["vector_a", "vector_b"]); + }); }); diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts new file mode 100644 index 000000000..2e7b2ec4d --- /dev/null +++ b/nodejs/__test__/materialized_view.test.ts @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as tmp from "tmp"; + +import { Connection, connect } from "../lancedb"; +import { + DEFINITION_META_KEY, + definitionFromMetadata, +} from "../lancedb/materialized_view"; + +describe("materialized views", () => { + let tmpDir: tmp.DirResult; + let db: Connection; + + beforeEach(async () => { + tmpDir = tmp.dirSync({ unsafeCleanup: true }); + db = await connect(tmpDir.name); + await db.createTable( + "people", + [ + { name: "ada", age: 36 }, + { name: "kid", age: 7 }, + { name: "grace", age: 85 }, + ], + { storageOptions: { newTableEnableStableRowIds: "true" } }, + ); + }); + afterEach(() => tmpDir.removeCallback()); + + it("rejects a stored limit a number cannot carry", () => { + const big = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":9007199254740993}', + ], + ]); + expect(() => definitionFromMetadata(big, "v")).toThrow( + /too large to represent exactly/, + ); + + const safe = new Map([ + [ + DEFINITION_META_KEY, + '{"kind":"select","source_table":"people","limit":42}', + ], + ]); + expect(definitionFromMetadata(safe, "v").limit).toBe(42); + }); + + it("creates, refreshes and queries a view", async () => { + const view = await db.createMaterializedView("adults", "people", { + select: ["name", ["shout", "upper(name)"]], + where: "age >= 18", + }); + expect(view.name).toBe("adults"); + expect(await view.table().countRows()).toBe(0); + + const result = await view.refresh(); + expect(result.mode).toBe("rebuild"); + expect(Number(result.rowsWritten)).toBe(2); + + const rows = await view.table().query().toArray(); + expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); + }); + + it("round-trips the definition", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + const view = await db.openMaterializedView("adults"); + const definition = await view.definition(); + expect(definition.sourceTable).toBe("people"); + expect(definition.filter).toBe("age >= 18"); + expect(definition.projections).toEqual([ + ["name", "`name`"], + ["age", "`age`"], + ]); + expect(definition.inputs).toEqual(["age", "name"]); + }); + + it("refreshes incrementally after an append", async () => { + const view = await db.createMaterializedView("copy", "people"); + await view.refresh(); + + const people = await db.openTable("people"); + await people.add([{ name: "alan", age: 41 }]); + const result = await view.refresh(); + expect(result.mode).toBe("incremental"); + expect(Number(result.rowsWritten)).toBe(1); + expect(await view.table().countRows()).toBe(4); + + expect((await view.refresh()).mode).toBe("no_op"); + }); + + it("lists views and rejects non-views", async () => { + await db.createMaterializedView("adults", "people", { + where: "age >= 18", + }); + expect(await db.listMaterializedViews()).toEqual(["adults"]); + await expect(db.openMaterializedView("people")).rejects.toThrow( + "not a materialized view", + ); + }); + + it("rejects an invalid expression at create time", async () => { + await expect( + db.createMaterializedView("bad", "people", { + select: [["x", "missing + 1"]], + }), + ).rejects.toThrow("missing"); + }); + + it("rejects invalid numeric options before creating anything", async () => { + for (const limit of [-5, 1.5, Infinity, NaN]) { + await expect( + db.createMaterializedView("bad", "people", { limit }), + ).rejects.toThrow("non-negative integer"); + } + expect(await db.listMaterializedViews()).toEqual([]); + + const view = await db.createMaterializedView("copy", "people"); + for (const sourceVersion of [-1, 1.5, Infinity, NaN]) { + await expect(view.refresh({ sourceVersion })).rejects.toThrow( + "non-negative integer", + ); + } + }); + + it("quotes bare select names", async () => { + await db.createTable("odd_names", [{ "order item": "widget" }], { + storageOptions: { newTableEnableStableRowIds: "true" }, + }); + const view = await db.createMaterializedView("quoted", "odd_names", { + select: ["order item"], + }); + const result = await view.refresh(); + expect(Number(result.rowsWritten)).toBe(1); + }); + + it("requires stable row ids on the source", async () => { + await db.createTable("plain", [{ x: 1 }]); + await expect(db.createMaterializedView("v", "plain")).rejects.toThrow( + "stable row ids", + ); + }); +}); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index e766b3d2a..c51cbbbb7 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -75,6 +75,25 @@ async function withMockDatabase( } describe("remote connection", () => { + it("refuses materialized views before issuing any request", async () => { + const paths: string[] = []; + await withMockDatabase( + (req, res) => { + paths.push(req.url ?? ""); + res.writeHead(404).end(); + }, + async (db) => { + await expect(db.openMaterializedView("secret_table")).rejects.toThrow( + /only on local databases/, + ); + await expect(db.listMaterializedViews()).rejects.toThrow( + /only on local databases/, + ); + expect(paths).toEqual([]); + }, + ); + }); + it("should accept partial connection options", async () => { await connect("db://test", { apiKey: "fake", @@ -311,7 +330,7 @@ describe("remote connection", () => { expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]); }); - it("diffs and merges remote branches", async () => { + it("diffs and cherry-picks remote branches", async () => { const sampleDiff = { fromBranch: "exp", parentVersion: 1, @@ -333,10 +352,9 @@ describe("remote connection", () => { changedColumns: [], addedIndexes: [], removedIndexes: [], - mergeable: true, - mergeBlockers: [], + errors: [], }; - const mergeBodies: Record[] = []; + const cherryPickBodies: Record[] = []; await withMockDatabase( (req, res) => { @@ -366,17 +384,16 @@ describe("remote connection", () => { .end(JSON.stringify(sampleDiff)); return; } - if (path.endsWith("/branches/merge/")) { - mergeBodies.push(body); + if (path.endsWith("/branches/cherry_pick/")) { + cherryPickBodies.push(body); const dryRun = body["dry_run"] === true; const response = { - status: dryRun ? "ready" : "rejected", + status: dryRun ? "ready" : "failed", diff: dryRun ? sampleDiff : { ...sampleDiff, - mergeable: false, - mergeBlockers: [ + errors: [ { code: "baseMoved", message: "main has advanced" }, ], }, @@ -398,19 +415,19 @@ describe("remote connection", () => { await expect(branches.diff("exp")).resolves.toEqual(sampleDiff); - const rejected = await branches.merge("exp"); - expect(rejected.status).toBe("rejected"); - expect(rejected.diff.mergeBlockers).toEqual([ + const failed = await branches.cherryPick("exp"); + expect(failed.status).toBe("failed"); + expect(failed.diff.errors).toEqual([ { code: "baseMoved", message: "main has advanced" }, ]); - const preview = await branches.merge("exp", true); + const preview = await branches.cherryPick("exp", true); expect(preview.status).toBe("ready"); expect(preview.preview.promotedColumns).toEqual(["tag"]); }, ); - expect(mergeBodies).toEqual([ + expect(cherryPickBodies).toEqual([ // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format { from_branch: "exp", dry_run: false }, // biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 80c50f1ac..0f6ed3615 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2953,7 +2953,7 @@ describe("column name options", () => { .limit(10) .toArray(); expect(results2.length).toBe(10); - }); + }, 30_000); }); describe("when creating an empty table", () => { diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 8b388d593..b52ab50ef 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -5,7 +5,6 @@ import { Data as ArrowData, Table as ArrowTable, Binary, - Bool, BufferType, DataType, DateUnit, @@ -18,12 +17,7 @@ import { FixedSizeList, Float, Float32, - Float64, Int, - Int8, - Int16, - Int32, - Int64, LargeBinary, List, Null, @@ -36,17 +30,16 @@ import { Struct, Timestamp, Type, - Uint8, - Uint16, - Uint32, Utf8, Vector, makeVector as arrowMakeVector, + util as arrowUtil, vectorFromArray as badVectorFromArray, makeBuilder, makeData, } from "apache-arrow"; import { Buffers } from "apache-arrow/data"; +import { typedArrayToArrowType } from "./arrow_type"; import { type EmbeddingFunction } from "./embedding/embedding_function"; import { EmbeddingFunctionConfig, @@ -59,14 +52,7 @@ import { sanitizeTable, sanitizeType, } from "./sanitize"; - -/** - * Check if a field name indicates a vector column. - */ -function nameSuggestsVectorColumn(fieldName: string): boolean { - const nameLower = fieldName.toLowerCase(); - return nameLower.includes("vector") || nameLower.includes("embedding"); -} +import { inferSchema } from "./schema"; export * from "apache-arrow"; export type SchemaLike = @@ -459,110 +445,6 @@ export function makeArrowTable( return new ArrowTable(inferredSchema, finalColumns); } -function inferSchema( - data: Array>, - schema: Schema | undefined, - opts: MakeArrowTableOptions, -): Schema { - // We will collect all fields we see in the data. - const pathTree = new PathTree(); - - for (const [rowI, row] of data.entries()) { - for (const [path, value] of rowPathsAndValues(row)) { - if (!pathTree.has(path)) { - // First time seeing this field. - if (schema !== undefined) { - const field = getFieldForPath(schema, path); - if (field === undefined) { - throw new Error( - `Found field not in schema: ${path.join(".")} at row ${rowI}`, - ); - } else { - pathTree.set(path, field.type); - } - } else { - const inferredType = inferType(value, path, opts); - if (inferredType === undefined) { - throw new Error(`Failed to infer data type for field ${path.join( - ".", - )} at row ${rowI}. \ - Consider providing an explicit schema.`); - } - pathTree.set(path, inferredType); - } - } else if (schema === undefined) { - const currentType = pathTree.get(path); - const newType = inferType(value, path, opts); - if (currentType !== newType) { - new Error(`Failed to infer schema for data. Previously inferred type \ - ${currentType} but found ${newType} at row ${rowI}. Consider \ - providing an explicit schema.`); - } - } - } - } - - if (schema === undefined) { - function fieldsFromPathTree(pathTree: PathTree): Field[] { - const fields = []; - for (const [name, value] of pathTree.map.entries()) { - if (value instanceof PathTree) { - const children = fieldsFromPathTree(value); - fields.push(new Field(name, new Struct(children), true)); - } else { - fields.push(new Field(name, value, true)); - } - } - return fields; - } - const fields = fieldsFromPathTree(pathTree); - return new Schema(fields); - } else { - function takeMatchingFields( - fields: Field[], - pathTree: PathTree, - ): Field[] { - const outFields = []; - for (const field of fields) { - if (pathTree.map.has(field.name)) { - const value = pathTree.get([field.name]); - if (value instanceof PathTree) { - const struct = field.type as Struct; - const children = takeMatchingFields(struct.children, value); - outFields.push( - new Field(field.name, new Struct(children), field.nullable), - ); - } else { - outFields.push( - new Field(field.name, value as DataType, field.nullable), - ); - } - } - } - return outFields; - } - const fields = takeMatchingFields(schema.fields, pathTree); - return new Schema(fields); - } -} - -function* rowPathsAndValues( - row: Record, - basePath: string[] = [], -): Generator<[string[], unknown]> { - for (const [key, value] of Object.entries(row)) { - if (isObject(value)) { - yield* rowPathsAndValues(value, [...basePath, key]); - } else { - // Skip undefined values - they should be treated the same as missing fields - // for embedding function purposes - if (value !== undefined) { - yield [[...basePath, key], value]; - } - } - } -} - function isObject(value: unknown): value is Record { return ( typeof value === "object" && @@ -577,146 +459,19 @@ function isObject(value: unknown): value is Record { ); } -function getFieldForPath(schema: Schema, path: string[]): Field | undefined { - let current: Field | Schema = schema; +function valueAtPath(datum: Record, path: string[]): unknown { + let current: unknown = datum; for (const key of path) { - if (current instanceof Schema) { - const field: Field | undefined = current.fields.find( - (f) => f.name === key, - ); - if (field === undefined) { - return undefined; - } - current = field; - } else if (current instanceof Field && DataType.isStruct(current.type)) { - const struct: Struct = current.type; - const field = struct.children.find((f) => f.name === key); - if (field === undefined) { - return undefined; - } - current = field; + if (current == null) { + return null; + } + if (isObject(current) && (Object.hasOwn(current, key) || key in current)) { + current = current[key]; } else { return undefined; } } - if (current instanceof Field) { - return current; - } else { - return undefined; - } -} - -/** - * Try to infer which Arrow type to use for a given value. - * - * May return undefined if the type cannot be inferred. - */ -function inferType( - value: unknown, - path: string[], - opts: MakeArrowTableOptions, -): DataType | undefined { - if (typeof value === "bigint") { - return new Int64(); - } else if (typeof value === "number") { - // Even if it's an integer, it's safer to assume Float64. Users can - // always provide an explicit schema or use BigInt if they mean integer. - return new Float64(); - } else if (typeof value === "string") { - if (opts.dictionaryEncodeStrings) { - return new Dictionary(new Utf8(), new Int32()); - } else { - return new Utf8(); - } - } else if (typeof value === "boolean") { - return new Bool(); - } else if (value instanceof Buffer) { - return new Binary(); - } else if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { - const info = typedArrayToArrowType(value); - if (info !== undefined) { - const child = new Field("item", info.elementType, true); - return new FixedSizeList(info.length, child); - } - return undefined; - } else if (Array.isArray(value)) { - if (value.length === 0) { - return undefined; // Without any values we can't infer the type - } - if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) { - const floatType = sanitizeType(opts.vectorColumns[path[0]].type); - return new FixedSizeList( - value.length, - new Field("item", floatType, true), - ); - } - const valueType = inferType(value[0], path, opts); - if (valueType === undefined) { - return undefined; - } - // Try to automatically detect embedding columns. - if (nameSuggestsVectorColumn(path[path.length - 1])) { - // Check if value is a Uint8Array for integer vector type determination - if (value instanceof Uint8Array) { - // For integer vectors, we default to Uint8 (matching Python implementation) - const child = new Field("item", new Uint8(), true); - return new FixedSizeList(value.length, child); - } else { - // For float vectors, we default to Float32 - const child = new Field("item", new Float32(), true); - return new FixedSizeList(value.length, child); - } - } else { - const child = new Field("item", valueType, true); - return new List(child); - } - } else { - // TODO: timestamp - return undefined; - } -} - -class PathTree { - map: Map>; - - constructor(entries?: [string[], V][]) { - this.map = new Map(); - if (entries !== undefined) { - for (const [path, value] of entries) { - this.set(path, value); - } - } - } - has(path: string[]): boolean { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return false; - } - ref = ref.map.get(part) as PathTree; - } - return true; - } - get(path: string[]): V | undefined { - let ref: PathTree = this; - for (const part of path) { - if (!(ref instanceof PathTree) || !ref.map.has(part)) { - return undefined; - } - ref = ref.map.get(part) as PathTree; - } - return ref as V; - } - set(path: string[], value: V): void { - let ref: PathTree = this; - for (const part of path.slice(0, path.length - 1)) { - if (!ref.map.has(part)) { - ref.map.set(part, new PathTree()); - } - ref = ref.map.get(part) as PathTree; - } - ref.map.set(path[path.length - 1], value); - } + return current; } function transposeData( @@ -724,37 +479,26 @@ function transposeData( field: Field, path: string[] = [], ): Vector { + const valuesPath = [...path, field.name]; + const values = data.map((datum) => valueAtPath(datum, valuesPath)); if (field.type instanceof Struct) { const childFields = field.type.children; - const fullPath = [...path, field.name]; const childVectors = childFields.map((child) => { - return transposeData(data, child, fullPath); + return transposeData(data, child, valuesPath); }); + const nullCount = values.filter((value) => value === null).length; const structData = makeData({ type: field.type, + length: values.length, + nullCount, + nullBitmap: + nullCount > 0 + ? arrowUtil.packBools(values.map((value) => value !== null)) + : undefined, children: childVectors as unknown as ArrowData[], }); return arrowMakeVector(structData); } else { - const valuesPath = [...path, field.name]; - const values = data.map((datum) => { - let current: unknown = datum; - for (const key of valuesPath) { - if (current == null) { - return null; - } - - if ( - isObject(current) && - (Object.hasOwn(current, key) || key in current) - ) { - current = current[key]; - } else { - return null; - } - } - return current; - }); return makeVector(values, field.type, undefined, field.nullable); } } @@ -797,32 +541,6 @@ function makeListVector(lists: unknown[][]): Vector { return listBuilder.finish().toVector(); } -/** - * Map a JS TypedArray instance to the corresponding Arrow element DataType - * and its length. Returns undefined if the value is not a recognized TypedArray. - */ -function typedArrayToArrowType( - value: ArrayBufferView, -): { elementType: DataType; length: number } | undefined { - if (value instanceof Float32Array) - return { elementType: new Float32(), length: value.length }; - if (value instanceof Float64Array) - return { elementType: new Float64(), length: value.length }; - if (value instanceof Uint8Array) - return { elementType: new Uint8(), length: value.length }; - if (value instanceof Uint16Array) - return { elementType: new Uint16(), length: value.length }; - if (value instanceof Uint32Array) - return { elementType: new Uint32(), length: value.length }; - if (value instanceof Int8Array) - return { elementType: new Int8(), length: value.length }; - if (value instanceof Int16Array) - return { elementType: new Int16(), length: value.length }; - if (value instanceof Int32Array) - return { elementType: new Int32(), length: value.length }; - return undefined; -} - /** Helper function to convert an Array of JS values to an Arrow Vector */ function makeVector( values: unknown[], @@ -1462,8 +1180,12 @@ export function ensureNestedFieldsExist( completeRow[field.name] = row[field.name]; } } else { - // Field is missing from the data - set to null - completeRow[field.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeRow[field.name] = + field.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, field.type as Struct) + : null; } } @@ -1498,8 +1220,12 @@ function ensureStructFieldsExist( completeStruct[childField.name] = data[childField.name]; } } else { - // Field is missing - set to null - completeStruct[childField.name] = null; + // Keep a missing struct valid while filling each of its children with + // null. This is distinct from an explicitly null struct value. + completeStruct[childField.name] = + childField.type.constructor.name === "Struct" + ? ensureStructFieldsExist({}, childField.type as Struct) + : null; } } diff --git a/nodejs/lancedb/arrow_type.ts b/nodejs/lancedb/arrow_type.ts new file mode 100644 index 000000000..35da346ef --- /dev/null +++ b/nodejs/lancedb/arrow_type.ts @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + type DataType, + Float32, + Float64, + Int8, + Int16, + Int32, + Uint8, + Uint16, + Uint32, +} from "apache-arrow"; + +/** + * Map a JS TypedArray instance to the corresponding Arrow element type and + * length. Returns undefined when the view is not a supported TypedArray. + */ +export function typedArrayToArrowType( + value: ArrayBufferView, +): { elementType: DataType; length: number } | undefined { + if (value instanceof Float32Array) + return { elementType: new Float32(), length: value.length }; + if (value instanceof Float64Array) + return { elementType: new Float64(), length: value.length }; + if (value instanceof Uint8Array) + return { elementType: new Uint8(), length: value.length }; + if (value instanceof Uint16Array) + return { elementType: new Uint16(), length: value.length }; + if (value instanceof Uint32Array) + return { elementType: new Uint32(), length: value.length }; + if (value instanceof Int8Array) + return { elementType: new Int8(), length: value.length }; + if (value instanceof Int16Array) + return { elementType: new Int16(), length: value.length }; + if (value instanceof Int32Array) + return { elementType: new Int32(), length: value.length }; + return undefined; +} diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index a81dc0442..263a338ab 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -16,6 +16,12 @@ import { makeEmptyTable, } from "./arrow"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; +import { + MaterializedView, + MaterializedViewSelect, + normalizeSelect, + validateNonNegativeInteger, +} from "./materialized_view"; import { Connection as LanceDbConnection } from "./native"; import type { CreateNamespaceResponse, @@ -25,12 +31,14 @@ import type { JobDescription, JobInfo, ListNamespacesResponse, + ListTablesResponse, } from "./native"; export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, + ListTablesResponse, }; import { sanitizeTable } from "./sanitize"; import { LocalTable, Table } from "./table"; @@ -128,6 +136,10 @@ export interface OpenTableOptions { indexCacheSize?: number; } +/** + * @deprecated Use {@link ListTablesOptions} with {@link Connection.listTables} + * instead. + */ export interface TableNamesOptions { /** * If present, only return names that come lexicographically after the @@ -141,6 +153,24 @@ export interface TableNamesOptions { limit?: number; } +export interface ListTablesOptions { + /** + * Token from a previous response, to resume listing where it left off. + * + * The token is opaque: it carries whatever the database needs to resume, and + * callers should not construct or interpret one. + */ + pageToken?: string; + /** + * An upper bound on how many tables to return. + * + * A page may hold fewer than this and still not be the last one, so keep + * going while the response carries a page token rather than while pages are + * full. + */ + limit?: number; +} + export interface ListNamespacesOptions { /** Token from a previous response for pagination. */ pageToken?: string; @@ -225,6 +255,7 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point (backwards compatibility) * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames(options?: Partial): Promise; /** @@ -235,18 +266,94 @@ export abstract class Connection { * @param {Partial} options - options to control the * paging / start point * + * @deprecated Use {@link Connection.listTables} instead. */ abstract tableNames( namespacePath?: string[], options?: Partial, ): Promise; + /** + * List a page of the tables in this database. + * + * To retrieve the tables after the page, pass the `pageToken` the response + * carries back in. A page can be shorter than `limit` without being the last + * one, so walk until a response carries no page token: + * + * ```ts + * const names = []; + * let pageToken = undefined; + * do { + * const page = await conn.listTables({ pageToken, limit: 100 }); + * names.push(...page.tables); + * pageToken = page.pageToken; + * } while (pageToken); + * ``` + * + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + options?: Partial, + ): Promise; + /** + * List a page of the tables in this database. + * + * @param {string[]} namespacePath - The namespace path to list tables from + * (defaults to root namespace) + * @param {Partial} options - Pagination options + * (`pageToken`, `limit`). + * @returns {Promise} A page of table names and an + * optional token for the tables after it. + */ + abstract listTables( + namespacePath?: string[], + options?: Partial, + ): Promise; + /** * Open a table in the database. * @param {string} name - The name of the table * @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace) * @param {Partial} options - Additional options */ + /** + * Define a materialized view named `name` over the table `source`. + * + * The view is created empty, with the query recorded in its schema + * metadata; `view.refresh()` computes the rows. The view is a normal + * table: it can be queried, indexed and searched, and it appears in + * `tableNames`. The source table must have stable row ids (create it with + * the `newTableEnableStableRowIds` storage option); they keep the view's + * provenance valid across source compactions and cannot be enabled after + * a table exists. Local databases only. + */ + abstract createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise; + + /** + * Open the materialized view named `name`. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract openMaterializedView(name: string): Promise; + + /** + * The names of the materialized views in this database. + * + * Found by reading every table's schema, so this costs an open per table. + */ + abstract listMaterializedViews(): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -531,6 +638,54 @@ export class LocalConnection extends Connection { ); } + async createMaterializedView( + name: string, + source: string, + options?: { + select?: MaterializedViewSelect; + where?: string; + limit?: number; + }, + ): Promise { + validateNonNegativeInteger(options?.limit, "limit"); + const innerTable = await this.inner.createMaterializedView( + name, + source, + normalizeSelect(options?.select), + options?.where, + options?.limit, + ); + return new MaterializedView(new LocalTable(innerTable)); + } + + async openMaterializedView(name: string): Promise { + const innerTable = await this.inner.openMaterializedView(name); + return new MaterializedView(new LocalTable(innerTable)); + } + + async listMaterializedViews(): Promise { + return await this.inner.listMaterializedViews(); + } + + async listTables( + namespacePathOrOptions?: string[] | Partial, + options?: Partial, + ): Promise { + // Detect if first argument is namespacePath array or options object + const namespacePath = Array.isArray(namespacePathOrOptions) + ? namespacePathOrOptions + : undefined; + const listTablesOptions = Array.isArray(namespacePathOrOptions) + ? options + : namespacePathOrOptions; + + return this.inner.listTables( + namespacePath ?? [], + listTablesOptions?.pageToken, + listTablesOptions?.limit, + ); + } + async openTable( name: string, namespacePath?: string[], diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 6a5bfe3b4..ebc8cda8d 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices"; import type { FtsToken } from "./table"; // Re-export native header provider for use with connectWithHeaderProvider +export { + MaterializedView, + MaterializedViewDefinition, + MaterializedViewSelect, +} from "./materialized_view"; export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js"; // OpenTelemetry metrics bridge. Only the high-level entry point is public; the @@ -51,6 +56,7 @@ export { AddResult, AddColumnsResult, RefreshColumnResult, + RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, @@ -75,11 +81,13 @@ export { Connection, CreateTableOptions, TableNamesOptions, + ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, + ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, @@ -135,10 +143,10 @@ export { BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, - MergeBlocker, + CherryPickError, BranchDiff, - MergePreview, - MergeBranchResult, + CherryPickPreview, + CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts new file mode 100644 index 000000000..b26dee59e --- /dev/null +++ b/nodejs/lancedb/materialized_view.ts @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { RefreshMaterializedViewResult } from "./native"; +import { Table } from "./table"; + +/** Schema metadata key holding a materialized view's definition. */ +export const DEFINITION_META_KEY = "mv.definition"; + +/** The query that defines a materialized view. */ +export interface MaterializedViewDefinition { + /** Name of the source table, in the same database as the view. */ + sourceTable: string; + /** `[output column, SQL expression]` pairs, in view schema order. */ + projections: [string, string][]; + /** SQL predicate selecting the source rows the view holds. */ + filter?: string; + /** Cap on the number of rows the view holds. */ + limit?: number; + /** Source columns the projections and filter read. */ + inputs: string[]; +} + +/** + * The view's columns: column names, `[alias, SQL expression]` pairs, or a + * record of the same. A bare name projects itself. + */ +export type MaterializedViewSelect = + | (string | [string, string])[] + | Record; + +/** + * @internal Reject a numeric option N-API would otherwise silently coerce: + * `Infinity` reaches Rust as 0, `1.5` as 1. + */ +export function validateNonNegativeInteger( + value: number | undefined, + name: string, +): void { + if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) { + throw new Error(`${name} must be a non-negative integer`); + } +} + +/** @internal Quote a column name as a Lance SQL identifier (backticks). */ +function quoteIdentifier(name: string): string { + return "`" + name.replace(/`/g, "``") + "`"; +} + +/** + * @internal Normalize a select argument into `[alias, expression]` pairs. + * A bare name projects itself and is quoted, so any valid column name works; + * pair and record entries are kept verbatim because their right side is an + * expression. + */ +export function normalizeSelect( + select?: MaterializedViewSelect, +): [string, string][] | undefined { + if (select === undefined) { + return undefined; + } + if (Array.isArray(select)) { + return select.map((item) => + typeof item === "string" ? [item, quoteIdentifier(item)] : item, + ); + } + return Object.entries(select); +} + +/** @internal Parse a definition off a table's stored schema metadata. */ +export function definitionFromMetadata( + metadata: Map, + name: string, +): MaterializedViewDefinition { + const raw = metadata.get(DEFINITION_META_KEY); + if (raw === undefined) { + throw new Error(`Table '${name}' is not a materialized view`); + } + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + const value: any = JSON.parse(raw); + if (value.kind !== "select") { + throw new Error( + `materialized view '${name}' is defined by '${value.kind}', which this ` + + "version of lancedb cannot refresh", + ); + } + const limit = value.limit ?? undefined; + // JSON.parse rounds integers past 2^53; every exact u64 parses to a safe + // integer and every rounded one does not, so this rejects precisely the + // values a number cannot carry. + if (limit !== undefined && !Number.isSafeInteger(limit)) { + throw new Error( + `materialized view '${name}' has a stored limit too large to represent exactly`, + ); + } + return { + sourceTable: value.source_table, + // biome-ignore lint/suspicious/noExplicitAny: raw JSON + projections: (value.projections ?? []).map((p: any) => [ + p.output, + p.expression, + ]), + filter: value.filter ?? undefined, + limit, + inputs: value.inputs ?? [], + }; +} + +/** + * A handle on a materialized view: its table plus its definition. + * + * Obtained from {@link Connection#createMaterializedView} or + * {@link Connection#openMaterializedView}. The view is a normal table -- + * queries, indexes and search all apply through {@link MaterializedView#table} + * -- whose contents are maintained by {@link MaterializedView#refresh}. + */ +export class MaterializedView { + private readonly inner: Table; + + constructor(table: Table) { + this.inner = table; + } + + get name(): string { + return this.inner.name; + } + + /** The view, as the table it is. */ + table(): Table { + return this.inner; + } + + /** The query that defines the view, read from its stored schema. */ + async definition(): Promise { + const schema = await this.inner.schema(); + return definitionFromMetadata(schema.metadata, this.name); + } + + /** + * Recompute the view from its source. + * + * The refresh is incremental when the source's changes can be reconciled + * into the view -- rows added, changed or removed since the last one -- + * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion` + * refreshes to that source version instead of the latest. + * + * Concurrent refreshes of one view do not duplicate its rows. Two that + * plan the same source rows conflict on commit, and the loser throws + * rather than writing them a second time. + */ + async refresh(options?: { + full?: boolean; + sourceVersion?: number; + }): Promise { + validateNonNegativeInteger(options?.sourceVersion, "sourceVersion"); + return await this.inner.refreshMaterializedView( + options?.full, + options?.sourceVersion, + ); + } +} diff --git a/nodejs/lancedb/schema.ts b/nodejs/lancedb/schema.ts new file mode 100644 index 000000000..e4749ef37 --- /dev/null +++ b/nodejs/lancedb/schema.ts @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { + Binary, + Bool, + DataType, + Dictionary, + Field, + FixedSizeList, + Float32, + Float64, + Int32, + Int64, + List, + Schema, + Struct, + Utf8, + util as arrowUtil, +} from "apache-arrow"; +import { typedArrayToArrowType } from "./arrow_type"; +import { sanitizeType } from "./sanitize"; + +type InferenceOptions = { + dictionaryEncodeStrings: boolean; + vectorColumns: Record; +}; + +/** + * Infer the Arrow schema represented by a set of records. + * + * This is the intentionally small interface to schema inference. The stateful + * details of combining partial type evidence are encapsulated below so callers + * only need to provide records, an optional schema, and inference options. + */ +export function inferSchema( + data: Array>, + schema: Schema | undefined, + options: InferenceOptions, +): Schema { + return new SchemaInferrer(schema, options).infer(data); +} + +class SchemaInferrer { + private readonly fields = new FieldTree(); + + constructor( + private readonly providedSchema: Schema | undefined, + private readonly options: InferenceOptions, + ) {} + + infer(data: Array>): Schema { + for (const [row, record] of data.entries()) { + for (const [path, value] of recordPathsAndValues(record)) { + this.observe(path, value, row); + } + } + + return this.providedSchema === undefined + ? new Schema(fieldsFromTree(this.fields)) + : new Schema(matchingFields(this.providedSchema.fields, this.fields)); + } + + private observe(path: string[], value: unknown, row: number): void { + const current = this.fields.get(path); + if (current === undefined) { + this.addField(path, value, row); + } else if (this.providedSchema === undefined) { + this.updateInferredField(path, value, row, current); + } + } + + private addField(path: string[], value: unknown, row: number): void { + if (this.providedSchema !== undefined) { + this.addSchemaField(this.providedSchema, path, row); + return; + } + + const evidence = + this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row); + if (evidence === undefined) { + throw typeInferenceError(path, row); + } + + const conflict = this.fields.set( + path, + evidence, + (existing) => + existing instanceof DeferredTypeEvidence && existing.isOnlyNulls(), + ); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private addSchemaField(schema: Schema, path: string[], row: number): void { + const field = fieldAtPath(schema, path); + if (field === undefined) { + throw new Error( + `Found field not in schema: ${path.join(".")} at row ${row}`, + ); + } + + const conflict = this.fields.set(path, field.type); + if (conflict !== undefined) { + throw branchConflictError(conflict, row, "Struct"); + } + } + + private updateInferredField( + path: string[], + value: unknown, + row: number, + current: FieldNode, + ): void { + const newType = this.inferType(value, path); + const deferred = DeferredTypeEvidence.from(value, row); + + if (current instanceof FieldTree) { + if (deferred?.isOnlyNulls()) { + return; + } + throw schemaInferenceError( + path, + row, + "Struct", + describeEvidence(newType ?? deferred), + ); + } + + if (current instanceof DeferredTypeEvidence) { + this.resolveDeferredField(path, row, current, newType, deferred); + return; + } + + if (newType !== undefined) { + if (!inferredTypesEqual(current, newType)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(newType), + ); + } + return; + } + + if (deferred === undefined || !deferred.matches(current)) { + throw schemaInferenceError( + path, + row, + describeEvidence(current), + describeEvidence(deferred), + ); + } + } + + private resolveDeferredField( + path: string[], + row: number, + current: DeferredTypeEvidence, + newType: DataType | undefined, + deferred: DeferredTypeEvidence | undefined, + ): void { + if (newType !== undefined) { + if (!current.matches(newType)) { + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + this.fields.set(path, newType); + return; + } + + if (deferred !== undefined) { + this.fields.set(path, current.merge(deferred)); + return; + } + + throw schemaInferenceError( + path, + row, + current.describe(), + describeEvidence(newType), + ); + } + + private inferType(value: unknown, path: string[]): DataType | undefined { + if (typeof value === "bigint") { + return new Int64(); + } + if (typeof value === "number") { + return new Float64(); + } + if (typeof value === "string") { + return this.options.dictionaryEncodeStrings + ? new Dictionary(new Utf8(), new Int32()) + : new Utf8(); + } + if (typeof value === "boolean") { + return new Bool(); + } + if (value instanceof Buffer) { + return new Binary(); + } + if (ArrayBuffer.isView(value) && !(value instanceof DataView)) { + const typedArray = typedArrayToArrowType(value); + return typedArray === undefined + ? undefined + : new FixedSizeList( + typedArray.length, + new Field("item", typedArray.elementType, true), + ); + } + if (!Array.isArray(value) || value.length === 0) { + return undefined; + } + + const configuredVector = + path.length === 1 ? this.options.vectorColumns[path[0]] : undefined; + if (configuredVector !== undefined) { + return new FixedSizeList( + value.length, + new Field("item", sanitizeType(configuredVector.type), true), + ); + } + + const itemType = this.inferArrayItemType(value, path); + if (itemType === undefined) { + return undefined; + } + + return nameSuggestsVectorColumn(path[path.length - 1]) + ? new FixedSizeList(value.length, new Field("item", new Float32(), true)) + : new List(new Field("item", itemType, true)); + } + + private inferArrayItemType( + values: unknown[], + path: string[], + ): DataType | undefined { + let itemType: DataType | undefined; + const deferredItems: unknown[] = []; + + for (const value of values) { + const candidate = this.inferType(value, path); + if (candidate === undefined) { + if (!isDeferredValue(value)) { + return undefined; + } + deferredItems.push(value); + } else if (itemType === undefined) { + itemType = candidate; + } else if (!inferredTypesEqual(itemType, candidate)) { + return undefined; + } + } + + if (itemType === undefined) { + return undefined; + } + return deferredItems.every((value) => + deferredValueMatchesType(value, itemType), + ) + ? itemType + : undefined; + } +} + +/** Nulls and empty/all-null lists that do not determine a type by themselves. */ +class DeferredTypeEvidence { + private constructor( + private readonly values: Array<{ value: unknown; row: number }>, + ) {} + + static from(value: unknown, row: number): DeferredTypeEvidence | undefined { + return isDeferredValue(value) + ? new DeferredTypeEvidence([{ value, row }]) + : undefined; + } + + isOnlyNulls(): boolean { + return this.values.every(({ value }) => value == null); + } + + matches(type: DataType): boolean { + return this.values.every(({ value }) => + deferredValueMatchesType(value, type), + ); + } + + merge(other: DeferredTypeEvidence): DeferredTypeEvidence { + return new DeferredTypeEvidence([...this.values, ...other.values]); + } + + describe(): string { + const list = this.values.find(({ value }) => Array.isArray(value)); + return list === undefined + ? "null" + : `List[${(list.value as unknown[]).length}]`; + } + + firstRow(): number { + return this.values[0].row; + } +} + +type FieldNode = DataType | DeferredTypeEvidence | FieldTree; +type LeafNode = Exclude; +type FieldConflict = { path: string[]; value: FieldNode }; + +/** Nested field state, kept separate from Arrow's eventual Struct types. */ +class FieldTree { + private readonly children = new Map(); + + get(path: string[]): FieldNode | undefined { + let current: FieldNode = this; + for (const part of path) { + if (!(current instanceof FieldTree)) { + return undefined; + } + const child = current.children.get(part); + if (child === undefined) { + return undefined; + } + current = child; + } + return current; + } + + set( + path: string[], + value: LeafNode, + canReplaceLeaf: (value: LeafNode) => boolean = () => false, + ): FieldConflict | undefined { + let branch: FieldTree = this; + for (const [index, part] of path.slice(0, -1).entries()) { + const child = branch.children.get(part); + if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) { + const nextBranch = new FieldTree(); + branch.children.set(part, nextBranch); + branch = nextBranch; + } else if (child instanceof FieldTree) { + branch = child; + } else { + return { path: path.slice(0, index + 1), value: child }; + } + } + + const name = path[path.length - 1]; + const current = branch.children.get(name); + if (current instanceof FieldTree) { + return { path, value: current }; + } + branch.children.set(name, value); + return undefined; + } + + entries(): IterableIterator<[string, FieldNode]> { + return this.children.entries(); + } + + has(name: string): boolean { + return this.children.has(name); + } +} + +function isLeaf(value: FieldNode): value is LeafNode { + return !(value instanceof FieldTree); +} + +function fieldsFromTree(tree: FieldTree, path: string[] = []): Field[] { + const fields: Field[] = []; + for (const [name, value] of tree.entries()) { + if (value instanceof FieldTree) { + fields.push( + new Field( + name, + new Struct(fieldsFromTree(value, [...path, name])), + true, + ), + ); + } else if (value instanceof DeferredTypeEvidence) { + throw typeInferenceError([...path, name], value.firstRow()); + } else { + fields.push(new Field(name, value, true)); + } + } + return fields; +} + +function matchingFields(fields: Field[], tree: FieldTree): Field[] { + const matches: Field[] = []; + for (const field of fields) { + if (!tree.has(field.name)) { + continue; + } + const value = tree.get([field.name]); + if (value instanceof FieldTree) { + const struct = field.type as Struct; + matches.push( + new Field( + field.name, + new Struct(matchingFields(struct.children, value)), + field.nullable, + ), + ); + } else { + matches.push(new Field(field.name, value as DataType, field.nullable)); + } + } + return matches; +} + +function* recordPathsAndValues( + record: Record, + path: string[] = [], +): Generator<[string[], unknown]> { + for (const [name, value] of Object.entries(record)) { + if (isRecord(value)) { + yield* recordPathsAndValues(value, [...path, name]); + } else if (value !== undefined) { + yield [[...path, name], value]; + } + } +} + +function isRecord(value: unknown): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !(value instanceof RegExp) && + !(value instanceof Date) && + !(value instanceof Set) && + !(value instanceof Map) && + !(value instanceof Buffer) && + !ArrayBuffer.isView(value) + ); +} + +function fieldAtPath(schema: Schema, path: string[]): Field | undefined { + let fields = schema.fields; + let field: Field | undefined; + for (const [index, name] of path.entries()) { + field = fields.find((candidate) => candidate.name === name); + if (field === undefined || index === path.length - 1) { + return field; + } + if (!DataType.isStruct(field.type)) { + return undefined; + } + fields = field.type.children; + } + return field; +} + +function isDeferredValue(value: unknown): boolean { + return ( + value == null || (Array.isArray(value) && value.every(isDeferredValue)) + ); +} + +function deferredValueMatchesType(value: unknown, type: DataType): boolean { + if (value == null) { + return true; + } + if (!Array.isArray(value)) { + return false; + } + if (DataType.isList(type)) { + return value.every((item) => + deferredValueMatchesType(item, type.valueType), + ); + } + if (DataType.isFixedSizeList(type)) { + return ( + value.length === type.listSize && + value.every((item) => deferredValueMatchesType(item, type.valueType)) + ); + } + return false; +} + +function inferredTypesEqual(current: DataType, candidate: DataType): boolean { + if (DataType.isDictionary(current)) { + return ( + DataType.isDictionary(candidate) && + current.isOrdered === candidate.isOrdered && + inferredTypesEqual(current.indices, candidate.indices) && + inferredTypesEqual(current.dictionary, candidate.dictionary) + ); + } + if (DataType.isList(current)) { + return ( + DataType.isList(candidate) && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + if (DataType.isFixedSizeList(current)) { + return ( + DataType.isFixedSizeList(candidate) && + current.listSize === candidate.listSize && + current.valueField.name === candidate.valueField.name && + current.valueField.nullable === candidate.valueField.nullable && + inferredTypesEqual(current.valueType, candidate.valueType) + ); + } + return arrowUtil.compareTypes(current, candidate); +} + +function describeEvidence( + evidence: DataType | DeferredTypeEvidence | undefined, +): string { + if (evidence === undefined) { + return "an unsupported value"; + } + return evidence instanceof DeferredTypeEvidence + ? evidence.describe() + : evidence.toString(); +} + +function branchConflictError( + conflict: FieldConflict, + row: number, + candidate: string, +): Error { + return schemaInferenceError( + conflict.path, + row, + conflict.value instanceof FieldTree + ? "Struct" + : describeEvidence(conflict.value), + candidate, + ); +} + +function schemaInferenceError( + path: string[], + row: number, + currentType: string, + newType: string, +): Error { + return new Error( + `Failed to infer schema for data. Previously inferred type ${currentType} ` + + `but found ${newType} for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function typeInferenceError(path: string[], row: number): Error { + return new Error( + `Failed to infer data type for field ${path.join(".")} at row ${row}. ` + + "Consider providing an explicit schema.", + ); +} + +function nameSuggestsVectorColumn(name: string): boolean { + const normalized = name.toLowerCase(); + return normalized.includes("vector") || normalized.includes("embedding"); +} diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 964c2cea3..28603cca9 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -35,6 +35,7 @@ import { Branches as NativeBranches, OptimizeStats, RefreshColumnResult, + RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, @@ -602,6 +603,18 @@ export abstract class Table { */ abstract refreshColumnAsync(column: string): Promise; + /** + * Recompute this table's contents from its materialized-view definition. + * + * Plumbing for {@link MaterializedView.refresh}, which is the way to call + * it: rejects tables that carry no view definition. Local tables only. + * @ignore + */ + abstract refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1264,6 +1277,13 @@ export class LocalTable extends Table { return await this.inner.refreshColumnAsync(column); } + async refreshMaterializedView( + full?: boolean, + sourceVersion?: number, + ): Promise { + return await this.inner.refreshMaterializedView(full, sourceVersion); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { @@ -1557,8 +1577,8 @@ export interface BranchRowCountSummary { deltaAvailable: boolean; } -/** A reason why a branch cannot currently be merged. */ -export interface MergeBlocker { +/** A reason why a cherry-pick cannot currently land. */ +export interface CherryPickError { code: string; message: string; } @@ -1578,20 +1598,19 @@ export interface BranchDiff { changedColumns: BranchColumnChange[]; addedIndexes: BranchIndexSummary[]; removedIndexes: BranchIndexSummary[]; - mergeable: boolean; - mergeBlockers: MergeBlocker[]; + errors: CherryPickError[]; } -/** Changes that would be, or were, promoted by a branch merge. */ -export interface MergePreview { +/** Changes that would be, or were, promoted by a cherry-pick. */ +export interface CherryPickPreview { promotedColumns: string[]; } -/** Result of previewing or attempting a branch merge. */ -export interface MergeBranchResult { - status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown"; +/** Result of previewing or attempting a cherry-pick. */ +export interface CherryPickResult { + status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown"; diff: BranchDiff; - preview: MergePreview; + preview: CherryPickPreview; mainVersionAfter?: number; } @@ -1654,21 +1673,21 @@ export class Branches { } /** - * Merge a branch into main. + * Cherry-pick a branch onto main. * - * Set `dryRun` to `true` to preview the merge. A rejected merge resolves - * with `status: "rejected"` instead of throwing. + * Set `dryRun` to `true` to preview. A failed cherry-pick resolves + * with `status: "failed"` instead of throwing. * - * @param fromBranch Branch to merge from. - * @param dryRun When true, only preview the merge. Defaults to false. + * @param fromBranch Branch to cherry-pick from. + * @param dryRun When true, only preview. Defaults to false. */ - async merge( + async cherryPick( fromBranch: string, dryRun: boolean = false, - ): Promise { - return (await this.#inner.merge( + ): Promise { + return (await this.#inner.cherryPick( fromBranch, dryRun, - )) as unknown as MergeBranchResult; + )) as unknown as CherryPickResult; } } diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 3c76481fc..f920c9e91 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 7a1a3af03..ad6926039 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 7175233af..ccd39f18e 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index add5a1da2..50446b597 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 72a82bf6d..d4da82dc7 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index c69beeb8a..dddc3a470 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index d21fc1eb3..23ac122f7 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 1c368dab5..21deb8dd8 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index 7f72eb8b0..96f4bafd5 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.3", + "version": "0.38.0-beta.8", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index c9f5e10ea..5cf676256 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -17,6 +17,7 @@ use lancedb::connection::{ConnectBuilder, Connection as LanceDBConnection, conne use lance_namespace::models::{ CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest, + ListTablesRequest, }; use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; @@ -36,6 +37,12 @@ pub struct ListNamespacesResponse { pub page_token: Option, } +#[napi(object)] +pub struct ListTablesResponse { + pub tables: Vec, + pub page_token: Option, +} + #[napi(object)] pub struct CreateNamespaceResponse { pub properties: Option>, @@ -206,6 +213,33 @@ impl Connection { op.execute().await.default_error() } + /// List a page of tables in the database. + #[napi(catch_unwind)] + pub async fn list_tables( + &self, + namespace_path: Option>, + page_token: Option, + limit: Option, + ) -> napi::Result { + let request = ListTablesRequest { + // The root namespace is an empty path, not an absent one: a namespace-backed + // database rejects a request that names no namespace. + id: Some(namespace_path.unwrap_or_default()), + page_token, + limit: limit.map(|limit| i32::try_from(limit).unwrap_or(i32::MAX)), + ..Default::default() + }; + let response = self + .get_inner()? + .list_tables(request) + .await + .default_error()?; + Ok(ListTablesResponse { + tables: response.tables, + page_token: response.page_token, + }) + } + /// Create table from a Apache Arrow IPC (file) buffer. /// /// Parameters: @@ -266,6 +300,58 @@ impl Connection { Ok(Table::new(tbl)) } + #[napi(catch_unwind)] + pub async fn create_materialized_view( + &self, + name: String, + source: String, + projections: Option>>, + filter: Option, + limit: Option, + ) -> napi::Result
{ + let mut builder = self.get_inner()?.create_materialized_view(name, source); + if let Some(projections) = projections { + let mut pairs = Vec::with_capacity(projections.len()); + for pair in projections { + let [output, expression]: [String; 2] = pair.try_into().map_err(|_| { + napi::Error::from_reason("each projection must be an [output, expression] pair") + })?; + pairs.push((output, expression)); + } + builder = builder.select(pairs); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + let limit = u64::try_from(limit) + .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?; + builder = builder.limit(limit); + } + let view = builder.execute().await.default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn open_materialized_view(&self, name: String) -> napi::Result
{ + let view = self + .get_inner()? + .open_materialized_view(&name) + .await + .default_error()?; + Ok(Table::new(view.table().clone())) + } + + #[napi(catch_unwind)] + pub async fn list_materialized_views(&self) -> napi::Result> { + let views = self + .get_inner()? + .list_materialized_views() + .await + .default_error()?; + Ok(views.into_iter().map(|v| v.name).collect()) + } + #[napi(catch_unwind)] pub async fn open_table( &self, diff --git a/nodejs/src/job.rs b/nodejs/src/job.rs index 6aaeee174..14013fd27 100644 --- a/nodejs/src/job.rs +++ b/nodejs/src/job.rs @@ -14,9 +14,12 @@ pub struct Job { } impl Job { - pub(crate) fn new(inner: lancedb::Job) -> Self { + pub(crate) fn new(inner: lancedb::Job) -> Self + where + T: Clone + Send + Sync + 'static, + { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|_| ())), } } } diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 312b675bd..1110f6203 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -1,6 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +// The materialized-view refresh future deepens the type graph past the +// default trait-recursion depth; same raise as the core crate applies. +#![recursion_limit = "256"] + use std::collections::HashMap; use env_logger::Env; diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index b15491202..9d60b2056 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -381,6 +381,26 @@ impl Table { Ok(crate::job::Job::new(job)) } + #[napi(catch_unwind)] + pub async fn refresh_materialized_view( + &self, + full: Option, + source_version: Option, + ) -> napi::Result { + let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone()) + .await + .default_error()?; + let mut builder = view.refresh().full(full.unwrap_or(false)); + if let Some(version) = source_version { + let version = u64::try_from(version).map_err(|_| { + napi::Error::from_reason("sourceVersion must be a non-negative integer") + })?; + builder = builder.source_version(version); + } + let result = builder.execute().await.default_error()?; + Ok(result.into()) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, @@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult { pub version: i64, } +#[napi(object)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date: "rebuild", "incremental" or "no_op". + pub mode: String, + pub rows_written: i64, + pub source_version: i64, + pub version: i64, +} + +impl From for RefreshMaterializedViewResult { + fn from(value: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match value.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: value.rows_written as i64, + source_version: value.source_version as i64, + version: value.version as i64, + } + } +} + impl From for RefreshColumnResult { fn from(value: lancedb::table::RefreshColumnResult) -> Self { Self { @@ -1605,18 +1650,18 @@ impl Branches { } #[napi(ts_return_type = "Promise>")] - pub async fn merge( + pub async fn cherry_pick( &self, from_branch: String, dry_run: Option, ) -> napi::Result { let result = self .inner - .merge_branch(&from_branch, dry_run.unwrap_or(false)) + .cherry_pick(&from_branch, dry_run.unwrap_or(false)) .await .default_error()?; serde_json::to_value(result).map_err(|err| { - napi::Error::from_reason(format!("failed to serialize branch merge result: {err}")) + napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}")) }) } } diff --git a/python/Cargo.toml b/python/Cargo.toml index 6181f704f..bffac2ef1 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" publish = false edition.workspace = true description = "Python bindings for LanceDB" @@ -26,7 +26,9 @@ lance-namespace-impls.workspace = true lance-io.workspace = true env_logger.workspace = true log.workspace = true -pyo3 = { version = "0.28", features = ["extension-module", "abi3-py310", "chrono"] } +# Maturin enables extension-module mode for Python builds. Keeping it out of +# Cargo features lets Rust unit tests link against libpython. +pyo3 = { version = "0.28", features = ["abi3-py310", "chrono"] } chrono.workspace = true pyo3-async-runtimes = { version = "0.28", features = [ "attributes", @@ -41,10 +43,7 @@ tokio.workspace = true libc = "0.2" [build-dependencies] -pyo3-build-config = { version = "0.28", features = [ - "extension-module", - "abi3-py310", -] } +pyo3-build-config = { version = "0.28", features = ["abi3-py310"] } [features] default = ["remote", "lancedb/aws", "lancedb/gcs", "lancedb/azure", "lancedb/dynamodb", "lancedb/oss", "lancedb/huggingface", "lancedb/cos", "lancedb/goosefs", "lancedb/metrics-otel"] diff --git a/python/README.md b/python/README.md index 550698500..3a81c486a 100644 --- a/python/README.md +++ b/python/README.md @@ -38,6 +38,25 @@ Stable releases are created about every 2 weeks. For the latest features and bug pip install --pre --extra-index-url https://pypi.fury.io/lancedb/ lancedb ``` +### Threading in CPU-limited containers + +LanceDB uses separate pools for compute work and storage I/O. On a container with +two visible CPUs, current releases intentionally use one compute worker by default; +no manual configuration is needed. If every query logs an I/O core reservation +warning on a two-CPU container, upgrade from LanceDB 0.21.1 or earlier. + +The two commonly tuned environment variables control different resources: + +- `LANCE_CPU_THREADS` overrides the number of compute workers. One worker is the + appropriate setting for a two-CPU container when an explicit override is needed. +- `LANCE_IO_THREADS` controls concurrent storage operations, not reserved CPU + cores. Its default can be greater than the number of CPUs because I/O workers + spend much of their time waiting for storage. + +Keep the defaults unless measurements show that the workload benefits from an +override. See the [Lance threading model](https://lance.org/guide/performance/#threading-model) +for the current defaults and tuning guidance. + ## Usage ### Basic Example diff --git a/python/pyproject.toml b/python/pyproject.toml index ae42172c0..fad3b1001 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -103,7 +103,7 @@ python-source = "python" module-name = "lancedb._lancedb" [build-system] -requires = ["maturin>=1.4"] +requires = ["maturin>=1.9.4"] build-backend = "maturin" [tool.ruff.lint] diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 0ceda4558..df8950686 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -29,9 +29,15 @@ from .functions import ( FunctionRegistrationRequest as FunctionRegistrationRequest, FunctionVersion as FunctionVersion, PythonRuntimeSpec as PythonRuntimeSpec, + RefreshColumnResult as RefreshColumnResult, UdfDefinition as UdfDefinition, udf as udf, ) +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + MaterializedViewDefinition, +) from .table import AsyncTable, Table from .types import BaseTokenizerType from ._lancedb import Session @@ -506,6 +512,9 @@ async def connect_async( __all__ = [ + "AsyncMaterializedView", + "MaterializedView", + "MaterializedViewDefinition", "connect", "connect_async", "tokenize", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 59537f45a..1e314ede8 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -147,7 +147,7 @@ class Connection(object): limit: Optional[int], ) -> list[str]: ... # Deprecated: Use list_tables instead def job(self, job_id: str) -> Job: ... - async def create_function_async(self, request_json: str) -> FunctionJob: ... + async def create_function_async(self, request_json: str) -> Job: ... async def get_function(self, name: str, version: str) -> str: ... async def list_jobs(self) -> List[JobInfo]: ... async def get_job(self, job_id: str) -> Optional[JobDescription]: ... @@ -197,6 +197,15 @@ class Connection(object): cur_namespace_path: Optional[List[str]] = None, new_namespace_path: Optional[List[str]] = None, ) -> None: ... + async def create_materialized_view( + self, + name: str, + source: str, + projections: Optional[List[Tuple[str, str]]] = None, + filter: Optional[str] = None, + limit: Optional[int] = None, + ) -> Table: ... + async def list_materialized_views(self) -> List[str]: ... async def drop_table( self, name: str, namespace_path: Optional[List[str]] = None ) -> None: ... @@ -225,14 +234,7 @@ class Job: @property def id(self) -> Optional[str]: ... async def status(self) -> str: ... - async def wait(self) -> None: ... - async def cancel(self) -> None: ... - -class FunctionJob: - @property - def id(self) -> Optional[str]: ... - async def status(self) -> str: ... - async def wait(self) -> str: ... + async def wait(self) -> Optional[str]: ... async def cancel(self) -> None: ... class JobInfo: @@ -355,6 +357,9 @@ class Table: ) -> AddColumnsResult: ... async def refresh_column(self, column: str) -> RefreshColumnResult: ... async def refresh_column_async(self, column: str) -> Job: ... + async def refresh_materialized_view( + self, full: bool = False, source_version: Optional[int] = None + ) -> RefreshMaterializedViewResult: ... async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ... async def alter_columns( self, columns: list[dict[str, Any]] @@ -420,7 +425,7 @@ class Branches: async def checkout(self, name: str, version: Optional[int] = None) -> Table: ... async def delete(self, name: str) -> None: ... async def diff(self, from_branch: str) -> Dict[str, Any]: ... - async def merge( + async def cherry_pick( self, from_branch: str, dry_run: bool = False ) -> Dict[str, Any]: ... @@ -704,6 +709,12 @@ class RefreshColumnResult: rows_filled: int version: int +class RefreshMaterializedViewResult: + mode: str + rows_written: int + source_version: int + version: int + class AlterColumnsResult: version: int diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index af18b6944..51b8d9993 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -46,7 +46,13 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError from . import __version__ from ._lancedb import connect as lancedb_connect # type: ignore from .functions import FunctionVersion, UdfDefinition -from .job import AsyncJob, Job, _function_job +from .job import AsyncJob, Job, _typed_job +from .materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, + normalize_select, +) from .table import ( AsyncTable, LanceTable, @@ -510,6 +516,70 @@ class DBConnection(EnforceOverrides): """ raise NotImplementedError + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + + The view is created empty, with the query recorded in its schema + metadata; ``view.refresh()`` computes the rows. The view is a normal + table: it can be queried, indexed and searched, and it appears in + ``table_names``. Local databases only. + + The source table must have stable row ids (create it with the + ``new_table_enable_stable_row_ids`` storage option): they keep the + view's provenance valid across source compactions, and cannot be + enabled after a table exists. + + Parameters + ---------- + name: str + The name of the view. + source: str + The name of the source table, in this database. + select: list or dict, optional + The view's columns: column names, ``(alias, SQL expression)`` + pairs, or a dict of the same. Omitting it selects every source + column, expanded against the source schema at creation time. + where: str, optional + SQL predicate; only matching source rows appear in the view. + limit: int, optional + Cap the view at this many rows, in materialization order. + + Returns + ------- + MaterializedView + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. @@ -1136,6 +1206,58 @@ class LanceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + + Examples + -------- + >>> import lancedb + >>> db = lancedb.connect( + ... "./.lancedb", + ... storage_options={"new_table_enable_stable_row_ids": "true"}, + ... ) + >>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}] + >>> table = db.create_table("people", data) + >>> view = db.create_materialized_view( + ... "adults", + ... "people", + ... select=["name", ("shout", "upper(name)")], + ... where="age >= 18", + ... ) + >>> result = view.refresh() + >>> result.rows_written + 1 + """ + LOOP.run( + self._conn.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ) + return MaterializedView(self.open_table(name)) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database.""" + return LOOP.run(self._conn.list_materialized_views()) + def clone_table( self, target_table_name: str, @@ -1906,6 +2028,50 @@ class AsyncConnection(object): await tbl.checkout(version) return tbl + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> AsyncMaterializedView: + """Define a materialized view named ``name`` over the table ``source``. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + inner = await self._inner.create_materialized_view( + name, + source, + projections=normalize_select(select), + filter=where, + limit=limit, + ) + return AsyncMaterializedView(AsyncTable(inner)) + + async def open_materialized_view(self, name: str) -> AsyncMaterializedView: + """Open the materialized view named ``name``. + + Raises ``ValueError`` if the table exists but is not a materialized + view. + """ + if self.uri.startswith("db://"): + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in this database. + + Found by reading every table's schema, so this costs an open per + table. + """ + return await self._inner.list_materialized_views() + async def clone_table( self, target_table_name: str, @@ -2071,7 +2237,7 @@ class AsyncConnection(object): inner = await self._inner.create_function_async( definition.registration_request.to_canonical_json() ) - return _function_job(inner) + return _typed_job(inner, FunctionVersion.from_json) async def get_function(self, name: str, *, version: str) -> FunctionVersion: """Open one exact immutable Function version from the remote catalog.""" diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 9532ab8b9..33a67b413 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -1,19 +1,24 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors -"""Canonical values exchanged with LanceDB Enterprise Function services. +"""Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, environment bake, secret resolution, and execution are owned by Sophon. +``RefreshColumnResult`` is also the backend-neutral result of a local +expression-backed refresh job. """ from __future__ import annotations import ast +import builtins import base64 import functools import hashlib +import importlib import inspect +import symtable import json import math import re @@ -460,7 +465,11 @@ class FunctionBinding(_RemoteValue): class RefreshColumnResult(_RemoteValue): - """Terminal result of a remote Function-column refresh Job.""" + """Terminal result of an expression-backed or Function-backed refresh Job. + + Local jobs produce this value in process. LanceDB Cloud and Enterprise + decode the same value from the durable server-job terminal payload. + """ rows_assigned: _UInt64 rows_failed: _UInt64 @@ -483,59 +492,58 @@ _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") _SECRET_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_GRAMMAR_PRIMITIVES = ( + (pa.bool_(), "bool"), + (pa.int8(), "int8"), + (pa.int16(), "int16"), + (pa.int32(), "int32"), + (pa.int64(), "int64"), + (pa.uint8(), "uint8"), + (pa.uint16(), "uint16"), + (pa.uint32(), "uint32"), + (pa.uint64(), "uint64"), + (pa.float16(), "float16"), + (pa.float32(), "float32"), + (pa.float64(), "float64"), + (pa.string(), "utf8"), + (pa.binary(), "binary"), + (pa.date32(), "date32"), + (pa.date64(), "date64"), +) + + def _canonical_arrow_type(data_type: pa.DataType) -> str: - primitive_types = ( - (pa.bool_(), "bool"), - (pa.int8(), "int8"), - (pa.int16(), "int16"), - (pa.int32(), "int32"), - (pa.int64(), "int64"), - (pa.uint8(), "uint8"), - (pa.uint16(), "uint16"), - (pa.uint32(), "uint32"), - (pa.uint64(), "uint64"), - (pa.float16(), "float16"), - (pa.float32(), "float32"), - (pa.float64(), "float64"), - (pa.string(), "utf8"), - (pa.large_utf8(), "large_utf8"), - (pa.binary(), "binary"), - (pa.large_binary(), "large_binary"), - (pa.date32(), "date32"), - (pa.date64(), "date64"), - ) - for candidate, name in primitive_types: + """The server's V1 Function type grammar. Anything outside it is rejected + here rather than at registration.""" + for candidate, name in _GRAMMAR_PRIMITIVES: if data_type == candidate: return name - if pa.types.is_fixed_size_binary(data_type): - return f"fixed_size_binary[{data_type.byte_width}]" - if pa.types.is_list(data_type): - return f"list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_large_list(data_type): - return f"large_list<{_canonical_arrow_type(data_type.value_type)}>" - if pa.types.is_fixed_size_list(data_type): + if pa.types.is_list(data_type) or pa.types.is_large_list(data_type): + prefix = "list" if pa.types.is_list(data_type) else "large_list" + return f"{prefix}<{_canonical_list_item(data_type)}>" + if pa.types.is_fixed_size_list(data_type) and data_type.list_size > 0: return ( - f"fixed_size_list<{_canonical_arrow_type(data_type.value_type)}>" - f"[{data_type.list_size}]" + f"fixed_size_list<{_canonical_list_item(data_type)}, {data_type.list_size}>" ) - if pa.types.is_struct(data_type): - fields = ",".join( - f"{field.name}:{_canonical_arrow_type(field.type)}" for field in data_type - ) - return f"struct<{fields}>" - if pa.types.is_timestamp(data_type): - timezone = f",tz={data_type.tz}" if data_type.tz is not None else "" - return f"timestamp[{data_type.unit}{timezone}]" - if pa.types.is_time32(data_type) or pa.types.is_time64(data_type): - return f"time[{data_type.unit}]" - if pa.types.is_duration(data_type): - return f"duration[{data_type.unit}]" - if pa.types.is_decimal(data_type): - bit_width = data_type.bit_width - return f"decimal{bit_width}({data_type.precision},{data_type.scale})" raise TypeError(f"unsupported Arrow type for Function signature: {data_type}") +def _canonical_list_item(data_type: pa.DataType) -> str: + """The grammar names only the item type; it always means a non-nullable + child called `item`, so any other child metadata cannot be represented.""" + child = data_type.value_field + if child.name != "item" or child.nullable or child.metadata: + raise TypeError( + "unsupported Arrow type for Function signature: list items must be a " + f"non-nullable field named 'item', got {child}" + ) + return _canonical_arrow_type(child.type) + + +def _list_of(item: pa.DataType) -> pa.DataType: + return pa.list_(pa.field("item", item, nullable=False)) + + def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: nullable = False origin = get_origin(annotation) @@ -583,7 +591,7 @@ def _annotation_type(annotation: Any) -> tuple[pa.DataType, bool]: value_type, value_nullable = _annotation_type(arguments[0]) if value_nullable: raise TypeError("nullable Function list elements are not supported") - return pa.list_(value_type), nullable + return _list_of(value_type), nullable raise TypeError(f"unsupported Function annotation: {annotation!r}") @@ -730,6 +738,104 @@ def _literal_source(value: Any) -> str: ) +_DYNAMIC_NAMESPACE_ACCESS = frozenset( + {"globals", "locals", "vars", "eval", "exec", "compile", "__import__"} +) +# Modules that hand out namespaces (`sys.modules`, `builtins`, importers, +# introspection). The artifact's module namespace holds only the names it was +# packaged with, so reaching around it cannot be represented. +_NAMESPACE_MODULES = frozenset( + {"sys", "builtins", "importlib", "inspect", "gc", "ctypes", "types"} +) + + +def _namespace_acquisition( + definition: ast.FunctionDef, references: set[str] +) -> list[str]: + found = set(references & _DYNAMIC_NAMESPACE_ACCESS) + for node in ast.walk(definition): + if isinstance(node, ast.Import): + found.update( + alias.name + for alias in node.names + if alias.name.split(".")[0] in _NAMESPACE_MODULES + ) + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module.split(".")[0] in _NAMESPACE_MODULES: + found.add(node.module) + return sorted(found) + + +def _module_references(module_source: str) -> set[str]: + """Names any scope in `module_source` binds or loads at module scope. + Python's own scope analysis on the exact text that ships: free variables + belong to an enclosing scope inside the function, and postponed + annotations are not runtime loads.""" + + def visit(table: symtable.SymbolTable, found: set[str]) -> None: + for symbol in table.get_symbols(): + if symbol.is_global() and ( + symbol.is_referenced() or symbol.is_declared_global() + ): + found.add(symbol.get_name()) + for child in table.get_children(): + visit(child, found) + + found: set[str] = set() + for table in symtable.symtable(module_source, "", "exec").get_children(): + visit(table, found) + return found + + +def _global_source(name: str, value: Any) -> str: + """One module-level line that rebinds `name` to `value` in the artifact: + an import for modules and importable classes/functions, a literal otherwise.""" + if isinstance(value, types.ModuleType): + if value.__name__.split(".")[0] in _NAMESPACE_MODULES: + raise ValueError( + f"@udf cannot package dynamic namespace access: {value.__name__!r}" + ) + try: + imported = importlib.import_module(value.__name__) + except ImportError: + imported = None + if imported is not value: + raise TypeError( + f"Function source references module {name!r} that does not import " + f"as {value.__name__!r}" + ) + return f"import {value.__name__} as {name}" + module_name = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if ( + isinstance(module_name, str) + and isinstance(qualname, str) + and module_name != "__main__" + and "." not in qualname + and "<" not in qualname + ): + try: + imported = getattr(importlib.import_module(module_name), qualname) + except (ImportError, AttributeError): + imported = None + if imported is value: + return f"from {module_name} import {qualname} as {name}" + return f"{name} = {_literal_source(value)}" + + +def _is_recursive_reference(function: Callable[..., Any], name: str) -> bool: + """`name` inside the body means the function itself unless the module has + since bound it to something else.""" + if name != function.__name__: + return False + bound = function.__globals__.get(name, function) + if bound is function: + return True + # The decorator's own result is the one wrapper known to call `function` + # unchanged; any other binding may behave differently from a self-call. + return type(bound) is UdfDefinition and bound._function is function + + def _package_source(function: Callable[..., Any]) -> bytes: if not inspect.isfunction(function) or inspect.iscoroutinefunction(function): raise TypeError("@udf requires a synchronous Python function") @@ -754,23 +860,46 @@ def _package_source(function: Callable[..., Any]) -> bytes: closure = inspect.getclosurevars(function) if closure.nonlocals: raise ValueError("@udf cannot package functions that capture closure values") - if closure.unbound: - raise ValueError( - f"@udf source contains unresolved global names: {sorted(closure.unbound)!r}" - ) - globals_source = [] - for name, value in sorted(closure.globals.items()): - if isinstance(value, types.ModuleType): - globals_source.append(f"import {value.__name__} as {name}") - else: - globals_source.append(f"{name} = {_literal_source(value)}") - function_source = ast.unparse(definition) - parts = ["from __future__ import annotations"] + module_header = "from __future__ import annotations" + references = _module_references(f"{module_header}\n\n{function_source}\n") + dynamic = _namespace_acquisition(definition, references) + if dynamic: + raise ValueError(f"@udf cannot package dynamic namespace access: {dynamic!r}") + # Resolve every module-scope reference the way the interpreter would: the + # function's own globals first (a module global may shadow a builtin, and + # nested scopes are not visible to getclosurevars), then its builtins. + # The artifact runs under the standard builtins; only the exact mapping is + # provably equivalent (a subclass or copy can change lookups and hooks). + if function.__builtins__ is not vars(builtins): + raise ValueError("@udf cannot package a non-standard builtins environment") + globals_source = [] + unresolved = [] + for name in sorted(references): + if name == function.__name__: + if not _is_recursive_reference(function, name): + raise ValueError( + f"@udf cannot package {name!r}: the module binds that name to " + "another value, which the artifact's own definition would shadow" + ) + continue + if name in function.__globals__: + globals_source.append(_global_source(name, function.__globals__[name])) + elif hasattr(builtins, name): + pass + else: + unresolved.append(name) + if unresolved: + raise ValueError( + f"@udf source contains unresolved global names: {unresolved!r}" + ) + + parts = [module_header] if globals_source: parts.extend(["", *globals_source]) parts.extend(["", function_source, ""]) - return "\n".join(parts).encode("utf-8") + packaged = "\n".join(parts) + return packaged.encode("utf-8") class UdfDefinition: @@ -917,6 +1046,13 @@ def udf( python_version : str, optional Remote Python major/minor version. Defaults to the client version. + The packaged artifact is a snapshot: the function source plus exactly + the module-level names it references (modules as imports, importable + classes and functions as imports, literals inline). Code that reaches the + module namespace another way -- ``globals()``/``eval``, ``sys.modules``, + ``builtins`` -- is rejected where it can be seen and otherwise + unsupported; closures and a non-standard ``__builtins__`` are rejected. + Returns ------- UdfDefinition diff --git a/python/python/lancedb/index.py b/python/python/lancedb/index.py index aa7846892..d2b63baf6 100644 --- a/python/python/lancedb/index.py +++ b/python/python/lancedb/index.py @@ -163,6 +163,15 @@ class FTS: The number of documents per compressed posting block. Supported values are 128 and 256. A value of 256 uses the experimental FTS V3 format and may introduce breaking changes. + memory_limit : int, optional + The total memory limit in MiB for the local FTS build stage. The limit + is divided evenly among indexing workers. This build-only setting is + not persisted with the index and does not apply to remote tables. + num_workers : int, optional + The number of workers for a local FTS build. By default Lance uses + roughly half of the available CPU cores. The effective value is + limited by the available compute capacity. This build-only setting is + not persisted with the index and does not apply to remote tables. Notes ----- @@ -185,6 +194,8 @@ class FTS: prefix_only: bool = False block_size: int = 128 custom_stop_words: Optional[List[str]] = None + memory_limit: Optional[int] = None + num_workers: Optional[int] = None @dataclass diff --git a/python/python/lancedb/job.py b/python/python/lancedb/job.py index 7bd600a74..f688768cb 100644 --- a/python/python/lancedb/job.py +++ b/python/python/lancedb/job.py @@ -5,12 +5,11 @@ import asyncio from datetime import timedelta -from typing import Any, Generic, Optional, TypeVar, cast +from typing import Any, Callable, Generic, Optional, TypeVar, cast from lancedb.background_loop import LOOP from . import _lancedb -from .functions import FunctionVersion T = TypeVar("T") @@ -18,11 +17,18 @@ T = TypeVar("T") class AsyncJob(Generic[T]): """A handle to an operation that may still be running. - The operation may already be complete when the handle is created. + The operation may already be complete when the handle is created. ``T`` + is the endpoint's terminal result type; unit-result jobs resolve to + ``None``. """ - def __init__(self, inner: Optional[Any]): + def __init__( + self, + inner: Optional[Any], + result_decoder: Optional[Callable[[Any], T]] = None, + ): self._inner = inner + self._result_decoder = result_decoder @property def id(self) -> Optional[str]: @@ -50,17 +56,21 @@ class AsyncJob(Generic[T]): async def wait(self, timeout: Optional[timedelta] = None) -> T: """Wait until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ if self._inner is None: return cast(T, None) if timeout is None: - return cast(T, await self._inner.wait()) - return cast( - T, - await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()), - ) + result = await self._inner.wait() + else: + result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()) + if self._result_decoder is not None: + return self._result_decoder(result) + return cast(T, result) async def cancel(self): """Request cancellation. Cancelling a finished operation is a no-op.""" @@ -70,7 +80,7 @@ class AsyncJob(Generic[T]): class Job(Generic[T]): - """Synchronous counterpart of `AsyncJob`.""" + """Synchronous counterpart of `AsyncJob` with the same result type.""" def __init__(self, inner: Optional[AsyncJob[T]]): self._inner = inner @@ -96,6 +106,9 @@ class Job(Generic[T]): def wait(self, timeout: Optional[timedelta] = None) -> T: """Block until the operation reaches a terminal state. + Returns the endpoint's typed result, or ``None`` for a unit-result + job. + Raises `JobFailedError` if the operation failed, `JobCancelledError` if it was cancelled, and `TimeoutError` if `timeout` elapses first. """ @@ -110,23 +123,8 @@ class Job(Generic[T]): LOOP.run(self._inner.cancel()) -class _FunctionJobAdapter: - def __init__(self, inner: "_lancedb.FunctionJob"): - self._inner = inner - - @property - def id(self) -> Optional[str]: - return self._inner.id - - async def status(self) -> str: - return await self._inner.status() - - async def wait(self) -> FunctionVersion: - return FunctionVersion.from_json(await self._inner.wait()) - - async def cancel(self): - await self._inner.cancel() - - -def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]: - return AsyncJob(_FunctionJobAdapter(inner)) +def _typed_job( + inner: "_lancedb.Job", result_decoder: Callable[[str], T] +) -> AsyncJob[T]: + """Bind an internal JSON-producing job to its public result model.""" + return AsyncJob(inner, result_decoder) diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py new file mode 100644 index 000000000..5abb44dc0 --- /dev/null +++ b/python/python/lancedb/materialized_view.py @@ -0,0 +1,178 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Materialized views: tables defined by a query over a source table and +maintained by refresh. See ``DBConnection.create_materialized_view``.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union + +from .background_loop import LOOP + +if TYPE_CHECKING: + import pyarrow as pa + + from ._lancedb import RefreshMaterializedViewResult + from .table import AsyncTable, LanceTable + +DEFINITION_META_KEY = b"mv.definition" + +SelectArg = Union[ + str, + Sequence[Union[str, Tuple[str, str]]], + Dict[str, str], + None, +] + + +@dataclass +class MaterializedViewDefinition: + """The query that defines a materialized view.""" + + source_table: str + """Name of the source table, in the same database as the view.""" + projections: List[Tuple[str, str]] + """``(output column, SQL expression)`` pairs, in view schema order.""" + filter: Optional[str] = None + """SQL predicate selecting the source rows the view holds.""" + limit: Optional[int] = None + """Cap on the number of rows the view holds.""" + inputs: List[str] = field(default_factory=list) + """Source columns the projections and filter read.""" + + +def _definition_from_schema( + schema: "pa.Schema", name: str +) -> MaterializedViewDefinition: + metadata = schema.metadata or {} + raw = metadata.get(DEFINITION_META_KEY) + if raw is None: + raise ValueError(f"Table '{name}' is not a materialized view") + value = json.loads(raw) + kind = value.get("kind") + if kind != "select": + raise NotImplementedError( + f"materialized view '{name}' is defined by '{kind}', which this " + "version of lancedb cannot refresh" + ) + return MaterializedViewDefinition( + source_table=value["source_table"], + projections=[ + (p["output"], p["expression"]) for p in value.get("projections", []) + ], + filter=value.get("filter"), + limit=value.get("limit"), + inputs=value.get("inputs", []), + ) + + +def _quote_identifier(name: str) -> str: + """Quote a column name as a Lance SQL identifier (backticks).""" + escaped = name.replace("`", "``") + return f"`{escaped}`" + + +def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]: + """``select`` items may be a column name, an ``(alias, expression)`` pair, + or a dict of the same. A bare name projects itself and is quoted, so any + valid column name works; dict and pair entries are kept verbatim because + their right side is an expression. + + A lone string is one column, not a sequence of its characters.""" + if select is None: + return None + if isinstance(select, str): + select = [select] + if isinstance(select, dict): + return list(select.items()) + normalized = [] + for item in select: + if isinstance(item, str): + normalized.append((item, _quote_identifier(item))) + else: + alias, expression = item + normalized.append((alias, expression)) + return normalized + + +class AsyncMaterializedView: + """A handle on a materialized view: its table plus its definition. + + Obtained from ``AsyncConnection.create_materialized_view`` or + ``AsyncConnection.open_materialized_view``. + """ + + def __init__(self, table: "AsyncTable"): + self._table = table + + def __repr__(self) -> str: + return f"AsyncMaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "AsyncTable": + """The view, as the table it is. Queries, indexes and search all + apply; writes are not blocked, but a rebuild replaces them.""" + return self._table + + async def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(await self._table.schema(), self.name) + + async def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. + + The refresh is incremental when the source's changes can be + reconciled into the view -- rows added, changed or removed since the + last one -- and otherwise rebuilds. ``full=True`` forces a rebuild; + ``source_version`` refreshes to that source version instead of the + latest. + + Concurrent refreshes of one view do not duplicate its rows. Two that + plan the same source rows conflict on commit, and the loser raises + rather than writing them a second time. + """ + return await self._table._inner.refresh_materialized_view( + full=full, source_version=source_version + ) + + +class MaterializedView: + """Synchronous variant of + [AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView].""" + + def __init__(self, table: "LanceTable"): + self._table = table + self._async = AsyncMaterializedView(table._table) + + def __repr__(self) -> str: + return f"MaterializedView(name={self.name!r})" + + @property + def name(self) -> str: + return self._table.name + + @property + def table(self) -> "LanceTable": + """The view, as the table it is.""" + return self._table + + @property + def definition(self) -> MaterializedViewDefinition: + """The query that defines the view, read from its stored schema.""" + return _definition_from_schema(self._table.schema, self.name) + + def refresh( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "RefreshMaterializedViewResult": + """Recompute the view from its source. See + [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh].""" + return LOOP.run(self._async.refresh(full=full, source_version=source_version)) diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 0e60bd218..f2e553321 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -61,6 +61,11 @@ from lance_namespace import ( NamespaceExistsRequest, TableExistsRequest, ) +from lancedb.materialized_view import ( + AsyncMaterializedView, + MaterializedView, + SelectArg, +) from lancedb.table import AsyncTable, LanceTable, Table from lancedb.util import validate_table_name from lancedb.common import DATA @@ -619,6 +624,42 @@ class LanceNamespaceDBConnection(DBConnection): tbl.checkout(version) return tbl + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "MaterializedView": + """Define a materialized view over a table in the root namespace. + See + [DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view]. + """ + return MaterializedView( + self.open_table( + LOOP.run( + self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + ).name + ) + ) + + @override + def open_materialized_view(self, name: str) -> "MaterializedView": + """Open the materialized view named ``name``.""" + view = MaterializedView(self.open_table(name)) + view.definition + return view + + @override + def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return LOOP.run(self._inner.list_materialized_views()) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): if namespace_path is None: @@ -1141,6 +1182,33 @@ class AsyncLanceNamespaceDBConnection: route_pushdown_to_rust=self._route_pushdown_to_rust, ) + async def create_materialized_view( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> "AsyncMaterializedView": + """Define a materialized view over a table in the root namespace.""" + view = await self._inner.create_materialized_view( + name, source, select=select, where=where, limit=limit + ) + # Reopen through the namespace so the view's table carries the + # namespace client and pushdown configuration a bare inner table lacks. + return AsyncMaterializedView(await self.open_table(view.name)) + + async def open_materialized_view(self, name: str) -> "AsyncMaterializedView": + """Open the materialized view named ``name``.""" + view = AsyncMaterializedView(await self.open_table(name)) + await view.definition() + return view + + async def list_materialized_views(self) -> List[str]: + """The names of the materialized views in the root namespace.""" + return await self._inner.list_materialized_views() + async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the namespace.""" if namespace_path is None: diff --git a/python/python/lancedb/permutation.py b/python/python/lancedb/permutation.py index 5d7a685ef..8287ef2fe 100644 --- a/python/python/lancedb/permutation.py +++ b/python/python/lancedb/permutation.py @@ -391,6 +391,15 @@ def _table_to_pickle_state(table: Table) -> dict[str, Any]: } +def _drop_base_version(permutation_data: pa.Table) -> pa.Table: + """Strip the recorded base version so the reader leaves the base table unpinned.""" + metadata = dict(permutation_data.schema.metadata or {}) + if metadata.pop(b"base_version", None) is None: + return permutation_data + metadata.pop(b"base_branch", None) + return permutation_data.replace_schema_metadata(metadata) + + def _table_from_pickle_state(state: dict[str, Any]) -> Table: from . import connect @@ -679,11 +688,15 @@ class Permutation: from . import connect connection_factory = state["connection_factory"] + rebuilt_base = False if connection_factory is not None: base_table = connection_factory(state["base_table_name"]) elif "base_table_state" in state: - base_table = _table_from_pickle_state(state["base_table_state"]) + base_state = state["base_table_state"] + rebuilt_base = base_state["kind"] == "memory" + base_table = _table_from_pickle_state(base_state) elif "base_table_data" in state: + rebuilt_base = True # In-memory base table inlined into the pickle; rebuild the same # way we rebuild the in-memory permutation table. mem_db = connect("memory://") @@ -701,11 +714,14 @@ class Permutation: ) permutation_table: Optional[Table] = None - if state["permutation_data"] is not None: + permutation_data = state["permutation_data"] + if permutation_data is not None: + if rebuilt_base: + # The base table was materialized from Arrow, so it is a fresh + # single-version dataset and the recorded pin cannot resolve on it. + permutation_data = _drop_base_version(permutation_data) mem_db = connect("memory://") - permutation_table = mem_db.create_table( - "permutation", state["permutation_data"] - ) + permutation_table = mem_db.create_table("permutation", permutation_data) self.base_table = base_table self.permutation_table = permutation_table diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 822756a34..b228cfb5b 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -25,6 +25,7 @@ from ..common import DATA from ..db import DBConnection, LOOP from ..functions import FunctionVersion, UdfDefinition from ..job import AsyncJob, Job +from ..materialized_view import MaterializedView, SelectArg if TYPE_CHECKING: from .._lancedb import JobDescription, JobInfo @@ -648,6 +649,32 @@ class RemoteDBConnection(DBConnection): namespace_path=namespace_path, ) + @override + def create_materialized_view( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + ) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def open_materialized_view(self, name: str) -> MaterializedView: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + + @override + def list_materialized_views(self) -> List[str]: + raise NotImplementedError( + "materialized views are supported only on local databases" + ) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index b97f8f194..41394ed71 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -49,7 +49,7 @@ from lancedb.index import ( LabelList, ) from lancedb.job import Job -from lancedb.functions import FunctionApplication +from lancedb.functions import FunctionApplication, RefreshColumnResult from lancedb.remote.db import LOOP from lancedb.table import IndexConfigType, KNOWN_METRICS import pyarrow as pa @@ -972,7 +972,7 @@ class RemoteTable(Table): def refresh_column(self, column: str): return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]: return Job(LOOP.run(self._table.refresh_column_async(column))) def alter_columns( diff --git a/python/python/lancedb/streaming.py b/python/python/lancedb/streaming.py index b27e606a4..76b3702a1 100644 --- a/python/python/lancedb/streaming.py +++ b/python/python/lancedb/streaming.py @@ -19,22 +19,29 @@ above. """ import ctypes +import heapq import logging import os import random import threading import time +import warnings from collections import deque from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy from multiprocessing import RawArray -from typing import Any, Callable, Iterator, Optional, Union +from typing import Any, Callable, cast, Iterator, Literal, NamedTuple, Optional, Union -from torch.utils.data import IterableDataset, get_worker_info +import pyarrow as pa +import pyarrow.compute as pc +import torch +from torch.utils.data import DataLoader, IterableDataset, get_worker_info from .permutation import ( Permutation, Transforms, permutation_builder, + _drop_base_version, _table_from_pickle_state, _table_to_pickle_state, ) @@ -50,6 +57,155 @@ DEFAULT_READ_BATCH_SIZE = 64 DEFAULT_PREFETCH_BATCHES = 4 +class _WorkerSample(NamedTuple): + data: Any + dataset: "StreamingDataset" + + +class _WorkerBatch(NamedTuple): + data: Any + state: dict + + +class _ConsumerIteratorLease(NamedTuple): + owner_token: int + owner_thread: int + + +class _CheckpointCollate: + """Attach the worker's post-fetch state to a collated batch.""" + + def __init__(self, collate_fn: Callable): + self._collate_fn = collate_fn + + def __call__(self, samples): + try: + if isinstance(samples, list): + if not samples: + return _WorkerBatch(self._collate_fn(samples), {}) + worker_samples = samples + data = self._collate_fn([sample.data for sample in worker_samples]) + dataset = worker_samples[-1].dataset + else: + data = self._collate_fn(samples.data) + dataset = samples.dataset + except StopIteration as exc: + raise RuntimeError( + "collate_fn raised StopIteration before returning a batch" + ) from exc + return _WorkerBatch(data, dataset._checkpoint_snapshot()) + + +class _StreamingDatasetAdapter(IterableDataset): + """Yield private sample wrappers for :class:`StreamingDataLoader`.""" + + def __init__(self, dataset: "StreamingDataset"): + super().__init__() + self.dataset = dataset + + def __iter__(self): + for sample in self.dataset._iter(consumer_checkpoint_transport=True): + yield _WorkerSample(sample, self.dataset) + + def __getattr__(self, name): + dataset = self.__dict__.get("dataset") + if dataset is None: + raise AttributeError(name) + return getattr(dataset, name) + + +class _ConsumerCommitIterator: + def __init__( + self, + iterator, + dataset: "StreamingDataset", + *, + owner_token: int, + require_uniform: bool, + ): + self._iterator = iterator + self._dataset = dataset + self._owner_token = owner_token + self._require_uniform = require_uniform + self._released = False + self._terminal = False + + def __iter__(self): + return self + + def __next__(self): + if self._terminal: + raise StopIteration + try: + batch = next(self._iterator) + except StopIteration: + self._terminal = True + self._release() + raise + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + try: + if not isinstance(batch, _WorkerBatch): + raise RuntimeError( + "StreamingDataLoader did not receive worker checkpoint metadata" + ) + self._dataset._commit_worker_state( + batch.state, require_uniform=self._require_uniform + ) + return batch.data + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader batch failed before it was returned: {exc}" + ) + raise + + def _release(self) -> None: + if self.__dict__.get("_released", True): + return + self._released = True + dataset = self.__dict__.get("_dataset") + if dataset is not None: + dataset._release_consumer_iterator(self._owner_token) + + def _shutdown_workers(self): + if self.__dict__.get("_released", True): + return None + self._terminal = True + iterator = self.__dict__.get("_iterator") + shutdown = getattr(iterator, "_shutdown_workers", None) + try: + if shutdown is not None: + shutdown() + else: + fetcher = getattr(iterator, "_dataset_fetcher", None) + dataset_iterator = getattr(fetcher, "dataset_iter", None) + close = getattr(dataset_iterator, "close", None) + if close is None: + raise RuntimeError( + "StreamingDataLoader could not close its inner iterator" + ) + close() + except BaseException as exc: + self._dataset._invalidate_checkpoint( + f"a DataLoader iterator could not be shut down safely: {exc}" + ) + raise + else: + self._release() + + def __del__(self): + try: + self._shutdown_workers() + except BaseException: + pass + + def __getattr__(self, name): + return getattr(self._iterator, name) + + class StreamingDataset(IterableDataset): """An elastic, resumable PyTorch IterableDataset backed by a LanceDB table. @@ -61,7 +217,7 @@ class StreamingDataset(IterableDataset): Internally ``__iter__`` runs a two-stage pipeline: - - **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches`` + - **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth`` workers fetches raw ``RecordBatch`` objects from LanceDB in parallel across all splits and places them in a per-split raw-batch queue. - **Stage 2 (transform)**: a second thread pool with @@ -104,11 +260,11 @@ class StreamingDataset(IterableDataset): call. Larger values amortise per-request overhead (critical on object storage) at the cost of higher memory usage per split buffer. Defaults to ``DEFAULT_READ_BATCH_SIZE`` (64). - prefetch_batches: + io_queue_depth: Number of I/O batches to keep in flight per split. Higher values overlap storage latency with transform and training compute at the cost - of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES`` - (4). + of more memory and threads. Must be greater than zero. Defaults to + ``DEFAULT_PREFETCH_BATCHES`` (4). columns: Optional list of column names to read. When set, only those columns are fetched from storage; all others are omitted. ``None`` (the @@ -132,6 +288,39 @@ class StreamingDataset(IterableDataset): Maximum number of transforms to run concurrently. Must be greater than zero. When ``None`` (the default), uses ``os.cpu_count()`` or 1 when the CPU count is unavailable. + pack_sequences: + Sequence-packing mode: token lists from consecutive documents are + joined with ``eos_id`` and sliced into blocks of this many tokens. + Each item is then a dict of two ``(pack_sequences,)`` LongTensors — + ``input_ids`` and ``doc_ids`` (per-position document index within + the block, for block-diagonal masks or position-id resets). + * Packing happens independently per owned split and preserves per-split + resume state. + * When a split cannot fill a real block for a cycle but an owned sibling + still can, or when only a short tail remains at epoch end, the short buffer + is padded to ``pack_sequences`` with ``pad_id`` so every local cycle emits + one block per owned split. + * ``eos_id``, ``pad_id``, and ``columns`` naming a single integer-list + column are required; incompatible with ``transform``. + eos_id: + Separator token id between packed documents. Required with + ``pack_sequences``, ignored otherwise. + pad_id: + Padding token id used to complete blocks when a split runs out of + real tokens mid-cycle or at epoch end. Required with + ``pack_sequences``, ignored otherwise. It must be reserved for padding: + padding positions retain the preceding document's ``doc_id`` (or zero + in an all-padding block), so callers must mask them separately using + ``input_ids == pad_id``. + blocks_per_epoch: + Total number of packed blocks emitted globally per epoch. Required with + ``pack_sequences``. An integer must be divisible by ``num_splits``. + Every logical split emits exactly ``blocks_per_epoch / num_splits`` + blocks: exhausted splits emit padding, while tokens beyond the budget + are left out of the epoch. This fixed per-split budget keeps packed + iteration and checkpoints independent of rank topology. + Pass ``"auto"`` to estimate a corpus-level budget from a bounded sample + of token lists. The estimate may be inaccurate. on_transform_error: What to do when the transform raises an exception: @@ -175,6 +364,16 @@ class StreamingDataset(IterableDataset): Prefer the ``filter`` parameter when bad rows can be expressed as a SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before splits are built, so every guarantee is fully preserved. + transform_queue_depth: + Number of transform-result batches to buffer per split in the + post-transform queue before backpressure is applied to the transform + stage. When the combined count of in-flight transform futures and + already-buffered rows for a split reaches + ``transform_queue_depth * read_batch_size``, no new transforms are + submitted for that split until the consumer catches up. Useful for + capping peak memory when the consumer (e.g. a GPU training step) is + slower than the transform stage. Must be greater than zero. + ``None`` (the default) imposes no limit. worker_info_override: If set, used in place of ``torch.utils.data.get_worker_info()`` to determine the DataLoader worker assignment. Intended for unit tests @@ -194,17 +393,30 @@ class StreamingDataset(IterableDataset): rank: int = 0, world_size: int = 1, read_batch_size: int = DEFAULT_READ_BATCH_SIZE, - prefetch_batches: int = DEFAULT_PREFETCH_BATCHES, + io_queue_depth: int = DEFAULT_PREFETCH_BATCHES, columns: Optional[list[str]] = None, shuffle_clump_size: Optional[int] = None, filter: Optional[str] = None, transform: Optional[Callable] = None, transform_parallelism: Optional[int] = None, + pack_sequences: Optional[int] = None, + eos_id: Optional[int] = None, + pad_id: Optional[int] = None, + blocks_per_epoch: Optional[Union[int, Literal["auto"]]] = None, on_transform_error: Union[str, Callable[[Exception], bool]] = "raise", + transform_queue_depth: Optional[int] = None, connection_factory: Optional[Callable[[str], Any]] = None, worker_info_override=None, + # Deprecated; use io_queue_depth instead. + prefetch_batches: Optional[int] = None, ): super().__init__() + if prefetch_batches is not None: + logger.warning( + "prefetch_batches is deprecated and will be removed in a future " + "version; use io_queue_depth instead" + ) + io_queue_depth = prefetch_batches if num_splits is None: num_splits = world_size if shuffle_seed is None: @@ -214,8 +426,59 @@ class StreamingDataset(IterableDataset): f"num_splits ({num_splits}) must be divisible by " f"world_size ({world_size})" ) + if io_queue_depth <= 0: + raise ValueError("io_queue_depth must be greater than 0") if transform_parallelism is not None and transform_parallelism <= 0: raise ValueError("transform_parallelism must be greater than 0") + if pack_sequences is not None: + if pack_sequences <= 0: + raise ValueError("pack_sequences must be greater than 0") + if eos_id is None: + raise ValueError("eos_id is required when pack_sequences is set") + if pad_id is None: + raise ValueError("pad_id is required when pack_sequences is set") + if blocks_per_epoch is None: + raise ValueError( + "blocks_per_epoch is required when pack_sequences is set" + ) + if blocks_per_epoch != "auto": + if not isinstance(blocks_per_epoch, int) or isinstance( + blocks_per_epoch, bool + ): + raise ValueError( + "blocks_per_epoch must be a positive integer or 'auto'" + ) + if blocks_per_epoch <= 0: + raise ValueError("blocks_per_epoch must be greater than 0") + if blocks_per_epoch % num_splits != 0: + raise ValueError( + f"blocks_per_epoch ({blocks_per_epoch}) must be divisible by " + f"num_splits ({num_splits})" + ) + if transform is not None: + raise ValueError("transform cannot be combined with pack_sequences") + if columns is None or len(columns) != 1: + raise ValueError( + "pack_sequences requires columns to name exactly one " + "list-typed column of token ids" + ) + field = table.schema.field(columns[0]) + if not ( + pa.types.is_list(field.type) + or pa.types.is_large_list(field.type) + or pa.types.is_fixed_size_list(field.type) + ): + raise ValueError( + f"pack_sequences requires a list-typed token column; " + f"{columns[0]} has type {field.type}" + ) + if not pa.types.is_integer(field.type.value_type): + raise ValueError( + "pack_sequences requires a token column with integer values; " + f"{columns[0]} has value type {field.type.value_type}" + ) + elif blocks_per_epoch is not None: + raise ValueError("blocks_per_epoch requires pack_sequences") if on_transform_error not in ("raise", "skip", "warn") and not callable( on_transform_error ): @@ -223,6 +486,8 @@ class StreamingDataset(IterableDataset): "on_transform_error must be 'raise', 'skip', 'warn', or a " f"callable, got {on_transform_error!r}" ) + if transform_queue_depth is not None and transform_queue_depth <= 0: + raise ValueError("transform_queue_depth must be greater than 0") self._table = table self._num_splits = num_splits @@ -232,16 +497,26 @@ class StreamingDataset(IterableDataset): self._rank = rank self._world_size = world_size self._read_batch_size = read_batch_size - self._prefetch_batches = prefetch_batches + self._io_queue_depth = io_queue_depth self._columns = columns self._shuffle_clump_size = shuffle_clump_size self._filter = filter self._transform = transform self._transform_parallelism = transform_parallelism + self._pack_sequences = pack_sequences + self._eos_id = eos_id + self._pad_id = pad_id + self._blocks_per_epoch = blocks_per_epoch self._on_transform_error = on_transform_error + self._transform_queue_depth = transform_queue_depth self._connection_factory = connection_factory self._worker_info_override = worker_info_override + # Packing resume state: permutation positions and partial-block buffers. + self._pack_consumed: list[int] = [0] * num_splits + self._pack_buffers: dict[int, dict[str, list[int]]] = {} + self._pack_blocks_emitted: list[int] = [0] * num_splits + # Live references to pipeline state, set only while __iter__ is running # in the same process. Used by the observability properties when the # DataLoader runs with num_workers=0. @@ -260,6 +535,22 @@ class StreamingDataset(IterableDataset): # rows_skipped] self._worker_stats: RawArray = RawArray(ctypes.c_int64, 8) + # A standard multi-process DataLoader cannot report which prefetched + # batches were actually returned to its consumer. Workers set this + # shared flag so state_dict() can reject a stale parent checkpoint + # unless StreamingDataLoader installed the consumer-commit transport. + self._untracked_worker_iteration: RawArray = RawArray(ctypes.c_int64, 1) + + # Parent-side checkpoint lifecycle. A failed DataLoader task creates + # a permanent hole in that iterator's delivery stream, while a + # multi-worker checkpoint is safe to restore only after all splits + # reach the same logical step boundary. + self._checkpoint_invalid_reason: Optional[str] = None + self._consumer_checkpoint_requires_uniform = False + self._consumer_iterator_lock = threading.Lock() + self._consumer_iterator_generation = 0 + self._consumer_iterator_lease: Optional[_ConsumerIteratorLease] = None + # Cumulative bytes of Arrow buffer data fetched across all iterations. self._bytes_loaded: int = 0 # Cumulative seconds spent in LanceDB I/O and in transform functions. @@ -272,6 +563,10 @@ class StreamingDataset(IterableDataset): # step boundaries all splits have consumed this many samples, so a # single scalar captures the topology-independent checkpoint state. self._resume_offset: int = 0 + # Exact yielded-sample counts for splits this process has advanced. + # Missing entries use _resume_offset, which remains the lower-bound + # checkpoint inherited from an earlier uniform/global state. + self._resume_samples: dict[int, int] = {} # Permutation position each split has consumed through, keyed by # global split index. Equal to _resume_offset for every split unless # on_transform_error skipped rows, in which case skipped positions @@ -291,6 +586,9 @@ class StreamingDataset(IterableDataset): else: self._perm_table = builder.split_sequential(fixed=num_splits).execute() + if self._blocks_per_epoch == "auto": + self._blocks_per_epoch = self._estimate_blocks_per_epoch() + # Contiguous block of global split indices assigned to this rank. splits_per_rank = num_splits // world_size rank_start = rank * splits_per_rank @@ -298,6 +596,71 @@ class StreamingDataset(IterableDataset): range(rank_start, rank_start + splits_per_rank) ) + def _estimate_blocks_per_epoch(self) -> int: + """Estimate a fixed packed-block budget from a bounded token sample.""" + # TODO: Replace this fallback with Lance's dedicated exact token-count + # estimation API once it is available. + if self._pack_sequences is None or not self._columns: + raise RuntimeError( + "packing must be configured before estimating its budget" + ) + + pack_len = self._pack_sequences + token_column = self._columns[0] + sample_cap_per_split = max(1, 100_000 // self._num_splits) + sampled_tokens = 0 + total_sampled = 0 + total_rows = 0 + rng = random.Random(self._shuffle_seed) + + warnings.warn( + "blocks_per_epoch='auto' uses an approximate token-count sample; " + "pass an explicit value for exact epoch sizing", + ) + + for split in range(self._num_splits): + permutation = Permutation.from_tables( + self._table, self._perm_table, split=split + ) + permutation = permutation.select_columns([token_column]) + permutation = permutation.with_transform(Transforms.arrow2arrow) + split_rows = permutation.num_rows + if split_rows == 0: + raise ValueError( + "blocks_per_epoch='auto' cannot estimate an empty dataset" + ) + + # Sample roughly 1% from each logical split, with at least one row + # per split and a global target cap of 100,000 rows. + sample_rows = min( + split_rows, + max(1, min((split_rows + 99) // 100, sample_cap_per_split)), + ) + sample_offsets = sorted(rng.sample(range(split_rows), sample_rows)) + sample_batch_size = max(1, self._read_batch_size) + for start in range(0, sample_rows, sample_batch_size): + batch = permutation.__getitems__( + sample_offsets[start : start + sample_batch_size] + ) + lengths = pc.list_value_length(batch.column(0)) + if lengths.null_count: + raise ValueError("pack_sequences does not support null token lists") + sampled_tokens += int(pc.sum(lengths).as_py()) + + total_sampled += sample_rows + total_rows += split_rows + + # Pool the samples into one global average. Each document contributes + # one EOS token. + estimated_tokens = ( + (sampled_tokens + total_sampled) * total_rows // total_sampled + ) + blocks = estimated_tokens // pack_len + return max( + self._num_splits, + blocks - blocks % self._num_splits, + ) + def _resolve_my_splits(self) -> list[int]: """Return the split indices this instance should read in __iter__.""" torch_worker_info = get_worker_info() @@ -329,11 +692,45 @@ class StreamingDataset(IterableDataset): return self._rank_splits[start : start + splits_per_worker] def __iter__(self) -> Iterator[dict[str, Any]]: + return self._iter() + + def _iter( + self, *, consumer_checkpoint_transport: bool = False + ) -> Iterator[dict[str, Any]]: + owner_token = None + previous_lease = self._consumer_iterator_lease + if consumer_checkpoint_transport: + if not self._consumer_iterator_active: + raise RuntimeError( + "StreamingDataLoader worker transport requires an active " + "parent iterator reservation" + ) + else: + try: + owner_token = self._acquire_consumer_iterator() + except BaseException: + self._release_consumer_iterator_after_failed_acquire(previous_lease) + raise + try: + yield from self._iter_owned( + consumer_checkpoint_transport=consumer_checkpoint_transport + ) + finally: + if owner_token is not None: + self._release_consumer_iterator(owner_token) + + def _iter_owned( + self, *, consumer_checkpoint_transport: bool + ) -> Iterator[dict[str, Any]]: if self._raw_batches_ref is not None: raise RuntimeError( "StreamingDataset does not support concurrent iteration. " "Only one active iterator per dataset instance is allowed." ) + real_worker = get_worker_info() is not None + if real_worker and not consumer_checkpoint_transport: + self._untracked_worker_iteration[0] = 1 + my_splits = self._resolve_my_splits() if not my_splits: return @@ -341,6 +738,7 @@ class StreamingDataset(IterableDataset): # Set identity transform on each Permutation so __getitems__ returns # the raw RecordBatch. Stage 2 applies the real transform. permutations: list[Permutation] = [] + initial_samples: list[int] = [] initial_positions: list[int] = [] for split_idx in my_splits: perm = Permutation.from_tables( @@ -348,16 +746,23 @@ class StreamingDataset(IterableDataset): ) if self._columns is not None: perm = perm.select_columns(self._columns) - perm = perm.with_transform(lambda batch: batch) - start_pos = self._resume_positions.get(split_idx, self._resume_offset) + perm = perm.with_transform(Transforms.arrow2arrow) + sample_count = self._resume_samples.get(split_idx, self._resume_offset) + # Both modes resume from absolute permutation positions. Packing + # stores them separately because it also checkpoints partial blocks. + start_pos = ( + self._pack_consumed[split_idx] + if self._pack_sequences is not None + else self._resume_positions.get(split_idx, sample_count) + ) if start_pos > 0: perm = perm.with_skip(start_pos) + initial_samples.append(sample_count) initial_positions.append(start_pos) permutations.append(perm) n = len(permutations) split_sizes = [perm.num_rows for perm in permutations] - initial_offset = self._resume_offset local_consumed = [0] * n # Permutation position each split has consumed through (absolute, # i.e. counted from the start of the unskipped split). Runs ahead of @@ -365,14 +770,36 @@ class StreamingDataset(IterableDataset): pos_consumed = list(initial_positions) batch_size = self._read_batch_size - max_prefetch = self._prefetch_batches + io_queue_depth = self._io_queue_depth transform_workers = ( self._transform_parallelism if self._transform_parallelism is not None else (os.cpu_count() or 1) ) - final_transform = ( - self._transform if self._transform is not None else Transforms.arrow2python + final_transform: Callable[[pa.RecordBatch], Any] + if self._pack_sequences is not None: + # Packing consumes raw token lists, one per document. + def arrow_tokens(batch: pa.RecordBatch) -> list[list[int]]: + token_column = batch.column(0) + if token_column.null_count or token_column.flatten().null_count: + raise ValueError( + "pack_sequences does not support null token lists or values" + ) + return cast(list[list[int]], token_column.to_pylist()) + + final_transform = arrow_tokens + else: + final_transform = ( + self._transform + if self._transform is not None + else Transforms.arrow2python + ) + # None means no limit; otherwise cap rows per split to + # transform_queue_depth batches worth (including in-flight transforms). + max_cooked_rows = ( + self._transform_queue_depth * batch_size + if self._transform_queue_depth is not None + else None ) # Per-split pipeline state. Batches are paired with the absolute @@ -409,7 +836,9 @@ class StreamingDataset(IterableDataset): io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices))) def _fill_io(i: int) -> None: - while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]: + while ( + len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i] + ): _submit_io(i) def _drain_io(i: int) -> None: @@ -487,7 +916,19 @@ class StreamingDataset(IterableDataset): def _try_submit_tx(i: int) -> None: """Submit transforms for raw_batches[i] up to available capacity.""" - while raw_batches[i] and tx_semaphore.acquire(blocking=False): + while raw_batches[i]: + # Backpressure: only submit a new transform when there is room + # for a full batch in the post-transform queue. Checking for + # a full batch prevents submitting a transform that would + # overflow the limit mid-batch (e.g. 990 rows queued with a + # capacity of 1000 and a batch_size of 128 must wait until + # 128 rows have been consumed, not just 1). + if max_cooked_rows is not None: + in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size + if in_pipeline + batch_size > max_cooked_rows: + break + if not tx_semaphore.acquire(blocking=False): + break abs_start, batch = raw_batches[i].popleft() tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch)) @@ -529,19 +970,174 @@ class StreamingDataset(IterableDataset): else: break # split exhausted + def _update_stats(*, idle: bool = False) -> None: + """Refresh pipeline statistics visible to the parent process.""" + ws = self._worker_stats + ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) + ws[1] = ( + 0 + if idle + else sum(batch.num_rows for q in raw_batches for _, batch in q) + ) + ws[2] = 0 if idle else sum(len(q) for q in cooked) + ws[3] = sum(local_consumed) + ws[4] = self._bytes_loaded + ws[5] = int(self._fetch_time * 1_000_000) + ws[6] = int(self._transform_time * 1_000_000) + ws[7] = self._rows_skipped + + # Sequence-packing helpers + pack_len = cast(int, self._pack_sequences) + eos_id = cast(int, self._eos_id) + pad_id = cast(int, self._pad_id) + blocks_per_split = ( + cast(int, self._blocks_per_epoch) // self._num_splits + if self._pack_sequences is not None + else 0 + ) + pack_consumed = list(self._pack_consumed) + pack_buffers = deepcopy(self._pack_buffers) + pack_blocks_emitted = list(self._pack_blocks_emitted) + + def _pack_buffer(i: int) -> dict[str, list[int]]: + return pack_buffers.setdefault(my_splits[i], {"tokens": [], "starts": []}) + + def _fill_block(i: int) -> None: + """Fill split i's buffer to one block or exhaust the split.""" + buf = _pack_buffer(i) + while len(buf["tokens"]) < pack_len: + _ensure_cooked(i) + if not cooked[i]: + return + buf["starts"].append(len(buf["tokens"])) + pos, tokens = cooked[i].popleft() + buf["tokens"].extend(tokens) + buf["tokens"].append(eos_id) + pack_consumed[my_splits[i]] = pos + 1 + local_consumed[i] += 1 + _advance(i) + + def _emit_block(i: int) -> dict[str, Any]: + buf = _pack_buffer(i) + tokens, starts = buf["tokens"], buf["starts"] + # doc_ids label document segments within the block; 0 also covers + # the continuation of a document begun in a prior block. + doc_ids = torch.zeros(pack_len, dtype=torch.int64) + doc_starts = [s for s in starts if 0 < s < pack_len] + doc_ids[doc_starts] = 1 + doc_ids.cumsum_(dim=0) # cumulative sum marks document boundaries + block = { + "input_ids": torch.tensor(tokens[:pack_len], dtype=torch.int64), + "doc_ids": doc_ids, + } + del tokens[:pack_len] + # Shift start boundaries for the next call. + buf["starts"] = [s - pack_len for s in starts if s >= pack_len] + return block + + def _commit_pack_state() -> None: + self._pack_consumed = list(pack_consumed) + self._pack_buffers = { + split: { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + for split, buffer in pack_buffers.items() + } + self._pack_blocks_emitted = list(pack_blocks_emitted) + # ── Main loop ───────────────────────────────────────────────────────── - with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool: + with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool: with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool: self._raw_batches_ref = raw_batches self._cooked_ref = cooked self._fetch_head_ref = fetch_head self._split_sizes_ref = split_sizes self._local_consumed_ref = local_consumed + try: for i in range(n): _fill_io(i) + def _yield_row(i: int): + pos, row = cooked[i].popleft() + # Surface any completed prefetched failure before the + # current row becomes durable checkpoint progress. + _advance(i) + local_consumed[i] += 1 + pos_consumed[i] = pos + 1 + split_idx = my_splits[i] + self._resume_samples[split_idx] = ( + initial_samples[i] + local_consumed[i] + ) + self._resume_positions[split_idx] = pos_consumed[i] + return row + + def _update_progress_stats() -> None: + if not real_worker: + self._resume_offset = min( + initial_samples[j] + local_consumed[j] for j in range(n) + ) + _update_stats() + + if self._pack_sequences is not None: + first_count = pack_blocks_emitted[my_splits[0]] + if any( + pack_blocks_emitted[split] != first_count + for split in my_splits[1:] + ): + raise ValueError( + "Packed checkpoint is not aligned across the splits " + "owned by this iterator; merge every rank " + "state with merge_state_dicts before resuming on a " + "different topology" + ) + + while pack_blocks_emitted[my_splits[0]] < blocks_per_split: + # Each logical split gets one block per cycle. Exhausted + # splits are padded through the fixed global budget. + for i in range(n): + _fill_block(i) + + for i in range(n): + tokens = _pack_buffer(i)["tokens"] + if len(tokens) < pack_len: + tokens.extend([pad_id] * (pack_len - len(tokens))) + block = _emit_block(i) + pack_blocks_emitted[my_splits[i]] += 1 + # Checkpoint state must advance before yielding so + # StreamingDataLoader can attach the exact state to + # the batch it transports to the parent process. + _commit_pack_state() + if i == n - 1: + _update_stats() + yield block + return + + # A checkpoint taken between round-robin split turns has + # non-uniform counts. Resume lagging splits first so the + # exact canonical sequence continues without replaying + # already-consumed rows. + if len(set(initial_samples)) > 1: + catch_up_to = max(initial_samples) + pending = [ + (initial_samples[i], my_splits[i], i) + for i in range(n) + if initial_samples[i] < catch_up_to + ] + heapq.heapify(pending) + while pending: + consumed, _, i = heapq.heappop(pending) + _ensure_cooked(i) + if not cooked[i]: + return + row = _yield_row(i) + if consumed + 1 < catch_up_to: + heapq.heappush(pending, (consumed + 1, my_splits[i], i)) + _update_progress_stats() + yield row + while True: # A cycle only runs if every split can still produce a # row. Without skips all splits exhaust simultaneously @@ -562,34 +1158,14 @@ class StreamingDataset(IterableDataset): break for i in range(n): - pos, row = cooked[i].popleft() - local_consumed[i] += 1 - pos_consumed[i] = pos + 1 - _advance(i) + row = _yield_row(i) # After the last split in each cycle: update the # global offset and refresh the shared-memory stats # so the main process can observe pipeline depth # even when __iter__ runs in a worker process. if i == n - 1: - self._resume_offset = initial_offset + local_consumed[i] - for j, split_idx in enumerate(my_splits): - self._resume_positions[split_idx] = pos_consumed[j] - ws = self._worker_stats - ws[0] = sum( - split_sizes[j] - fetch_head[j] for j in range(n) - ) - ws[1] = sum( - batch.num_rows - for q in raw_batches - for _, batch in q - ) - ws[2] = sum(len(q) for q in cooked) - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_progress_stats() yield row finally: @@ -597,15 +1173,7 @@ class StreamingDataset(IterableDataset): # when iteration ends mid-cycle (e.g. a split whose rows # were all skipped before completing a single cycle), so # counters like rows_skipped would otherwise be stale. - ws = self._worker_stats - ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n)) - ws[1] = 0 # queue-depth properties document 0 when idle - ws[2] = 0 - ws[3] = sum(local_consumed) - ws[4] = self._bytes_loaded - ws[5] = int(self._fetch_time * 1_000_000) - ws[6] = int(self._transform_time * 1_000_000) - ws[7] = self._rows_skipped + _update_stats(idle=True) self._raw_batches_ref = None self._cooked_ref = None self._fetch_head_ref = None @@ -744,6 +1312,7 @@ class StreamingDataset(IterableDataset): "_local_consumed_ref", ): state[key] = None + state["_consumer_iterator_lock"] = None return state def __setstate__(self, state): @@ -754,48 +1323,232 @@ class StreamingDataset(IterableDataset): table_state = state.pop("_table") perm_name, perm_data = state.pop("_perm_table") self.__dict__.update(state) + self._consumer_iterator_lock = threading.Lock() if self._connection_factory is not None: self._table = self._connection_factory(table_name) else: self._table = _table_from_pickle_state(table_state) + if table_state["kind"] == "memory": + # Rebuilt from Arrow, so the recorded pin cannot resolve on it. + perm_data = _drop_base_version(perm_data) self._perm_table = _connect("memory://").create_table(perm_name, perm_data) def state_dict(self) -> dict: """Snapshot the dataset's consumption state. - The returned dict is topology-independent: at global step boundaries - every split has been consumed the same number of times (by the - round-robin design), so the per-split count is a single uniform value - that is identical across all ranks and DataLoader workers. + When using DataLoader workers, construct a + [StreamingDataLoader][lancedb.streaming.StreamingDataLoader]. It + commits worker state only when a prefetched batch is returned to the + trainer. A standard multi-process ``DataLoader`` cannot expose that + boundary, so calling this method after one has started raises + ``RuntimeError`` instead of returning stale producer state. - ``positions_consumed_per_split`` records how far into each split's - permutation iteration has advanced. It only differs from - ``samples_consumed_per_split`` when ``on_transform_error`` skipped - rows, in which case entries are exact for the splits this instance - iterated and a lower bound (the sample count) for splits owned by - other ranks or workers. Combine the state dicts from all ranks with + In row mode, the returned dict is topology-independent at global step + boundaries. ``positions_consumed_per_split`` records how far each + split's permutation has advanced, which can differ from the sample + count when ``on_transform_error`` skips rows. ``StreamingDataLoader`` + combines worker state in its parent process. Combine state dicts from + every rank with [merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts] - to recover the exact value for every split before resuming on a - different topology. + before resuming on a different topology. + + Packed state includes partial token buffers and emitted block counts + for every logical split. When packing is sharded, merge every rank + state with ``merge_state_dicts`` before loading it. """ + if self._untracked_worker_iteration[0] and get_worker_info() is None: + raise RuntimeError( + "StreamingDataset cannot checkpoint a standard DataLoader with " + "num_workers > 0 because prefetched worker progress is not " + "consumer-committed. Use StreamingDataLoader instead." + ) + if self._checkpoint_invalid_reason is not None: + raise RuntimeError( + "StreamingDataset checkpointing is invalid because " + f"{self._checkpoint_invalid_reason}. Load the last valid " + "checkpoint into a fresh dataset before continuing." + ) + state = self._checkpoint_snapshot() + if self._pack_sequences is not None: + rank_blocks = [ + state["blocks_emitted_per_split"][split] for split in self._rank_splits + ] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "Packed StreamingDataset checkpointing is only safe at a " + "complete logical step boundary, when every split assigned " + "to this rank has emitted the same block count. Consume more " + "batches before calling state_dict()." + ) + elif self._consumer_checkpoint_requires_uniform: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in self._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader checkpointing with multiple workers is " + "only safe at a complete logical step boundary, when every " + "split assigned to this rank has the same consumed-sample " + "count. Consume more batches before calling state_dict()." + ) + return state + + def _checkpoint_snapshot(self) -> dict: + if self._pack_sequences is not None: + return { + "shuffle_seed": self._shuffle_seed, + "num_splits": self._num_splits, + "epoch": self._epoch, + "pack_sequences": self._pack_sequences, + "eos_id": self._eos_id, + "pad_id": self._pad_id, + "blocks_per_epoch": self._blocks_per_epoch, + "samples_consumed_per_split": list(self._pack_consumed), + "blocks_emitted_per_split": list(self._pack_blocks_emitted), + "pack_buffers": deepcopy(self._pack_buffers), + } + samples = [ + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ] positions = [ - self._resume_positions.get(split, self._resume_offset) + self._resume_positions.get(split, samples[split]) for split in range(self._num_splits) ] return { "shuffle_seed": self._shuffle_seed, "num_splits": self._num_splits, "epoch": self._epoch, - "samples_consumed_per_split": [self._resume_offset] * self._num_splits, + "samples_consumed_per_split": samples, "positions_consumed_per_split": positions, } + def _invalidate_checkpoint(self, reason: str) -> None: + if self._checkpoint_invalid_reason is None: + self._checkpoint_invalid_reason = reason + + @property + def _consumer_iterator_active(self) -> bool: + return self._consumer_iterator_lease is not None + + @property + def _consumer_iterator_owner(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_token if lease is not None else None + + @property + def _consumer_iterator_owner_thread(self) -> Optional[int]: + lease = self._consumer_iterator_lease + return lease.owner_thread if lease is not None else None + + def _acquire_consumer_iterator(self) -> int: + """Reserve this parent dataset for one checkpoint-aware iterator.""" + with self._consumer_iterator_lock: + if self._consumer_iterator_active or self._raw_batches_ref is not None: + raise RuntimeError( + "StreamingDataset does not support concurrent iteration. " + "Only one active iterator per dataset instance is allowed." + ) + owner_thread = threading.get_ident() + owner_token = self._consumer_iterator_generation + 1 + lease = _ConsumerIteratorLease(owner_token, owner_thread) + self._consumer_iterator_generation = owner_token + self._consumer_iterator_lease = lease + return owner_token + + def _release_consumer_iterator(self, owner_token: int) -> None: + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if lease is not None and lease.owner_token == owner_token: + self._consumer_iterator_lease = None + + def _release_consumer_iterator_after_failed_acquire( + self, previous_lease: Optional[_ConsumerIteratorLease] + ) -> None: + """Clean up when an interrupted acquire set a lease but did not return it.""" + owner_thread = threading.current_thread().ident + with self._consumer_iterator_lock: + lease = self._consumer_iterator_lease + if ( + lease is not None + and lease is not previous_lease + and lease.owner_thread == owner_thread + ): + self._consumer_iterator_lease = None + + def _commit_worker_state(self, state: dict, *, require_uniform: bool) -> None: + """Merge one trainer-consumed worker batch into parent state.""" + for key, expected in ( + ("shuffle_seed", self._shuffle_seed), + ("num_splits", self._num_splits), + ("epoch", self._epoch), + ): + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + packed = "pack_buffers" in state + if packed != (self._pack_sequences is not None): + raise ValueError("worker checkpoint mode does not match the dataset") + if packed: + for key in ("pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"): + expected = getattr(self, f"_{key}") + if state.get(key) != expected: + raise ValueError( + f"{key} mismatch in worker checkpoint: " + f"{state.get(key)} != {expected}" + ) + samples = state["samples_consumed_per_split"] + emitted = state["blocks_emitted_per_split"] + if len(samples) != self._num_splits or len(emitted) != self._num_splits: + raise ValueError( + "packed worker checkpoint must contain one entry per split" + ) + buffers = state["pack_buffers"] + for split, (count, blocks) in enumerate(zip(samples, emitted)): + incoming = (int(blocks), int(count)) + current = ( + self._pack_blocks_emitted[split], + self._pack_consumed[split], + ) + if incoming > current: + self._pack_blocks_emitted[split] = incoming[0] + self._pack_consumed[split] = incoming[1] + buffer = buffers.get(split, buffers.get(str(split))) + if buffer is None: + self._pack_buffers.pop(split, None) + else: + self._pack_buffers[split] = { + "tokens": list(buffer["tokens"]), + "starts": list(buffer["starts"]), + } + self._consumer_checkpoint_requires_uniform |= require_uniform + return + + samples = state["samples_consumed_per_split"] + positions = state.get("positions_consumed_per_split", samples) + for split, count in enumerate(samples): + current = self._resume_samples.get(split, self._resume_offset) + self._resume_samples[split] = max(current, int(count)) + for split, position in enumerate(positions): + current = self._resume_positions.get( + split, self._resume_samples.get(split, self._resume_offset) + ) + self._resume_positions[split] = max(current, int(position)) + self._resume_offset = min( + self._resume_samples.get(split, self._resume_offset) + for split in range(self._num_splits) + ) + self._consumer_checkpoint_requires_uniform |= require_uniform + def load_state_dict(self, state: dict) -> None: """Resume from a previously snapshotted state. Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ from the checkpoint, since a different split structure or shuffle order - makes mid-epoch resumption meaningless. + makes mid-epoch resumption meaningless. Packed checkpoints + pin ``pack_sequences``, ``eos_id``, ``pad_id``, + ``blocks_per_epoch``, and ``epoch``. """ if state["num_splits"] != self._num_splits: raise ValueError( @@ -807,15 +1560,44 @@ class StreamingDataset(IterableDataset): f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, " f"current dataset has {self._shuffle_seed}" ) + self._consumer_checkpoint_requires_uniform = False + + if "pack_buffers" in state or self._pack_sequences is not None: + for key in ( + "pack_sequences", + "eos_id", + "pad_id", + "blocks_per_epoch", + "epoch", + ): + ours = getattr(self, f"_{key}") + if state.get(key) != ours: + raise ValueError( + f"{key} mismatch: checkpoint has {state.get(key)}, " + f"current dataset has {ours}" + ) + self._pack_consumed = [int(c) for c in state["samples_consumed_per_split"]] + self._pack_blocks_emitted = [ + int(c) for c in state["blocks_emitted_per_split"] + ] + self._pack_buffers = { + int(g): {"tokens": list(b["tokens"]), "starts": list(b["starts"])} + for g, b in state["pack_buffers"].items() + } + return + consumed = state["samples_consumed_per_split"] - # All entries are equal at step boundaries; use the first. if isinstance(consumed, list): - self._resume_offset = consumed[0] if consumed else 0 + self._resume_offset = min(consumed) if consumed else 0 + self._resume_samples = { + split: int(count) for split, count in enumerate(consumed) + } else: self._resume_offset = int(consumed) + self._resume_samples = {} # Older checkpoints predate positions_consumed_per_split; without # skipped rows positions equal sample counts, so falling back to - # _resume_offset (the .get default in __iter__) is exact. + # the per-split sample count (the .get default in __iter__) is exact. positions = state.get("positions_consumed_per_split") if positions is None: self._resume_positions = {} @@ -828,25 +1610,23 @@ class StreamingDataset(IterableDataset): def merge_state_dicts(states: list[dict]) -> dict: """Merge state dicts saved by different ranks into one exact state. - Only needed when ``on_transform_error`` skips rows in multi-rank - training: each rank then knows the exact permutation position only for - its own splits, and records a lower bound for the rest. Because - exactly one rank owns each split, the elementwise maximum across all - ranks' ``positions_consumed_per_split`` recovers the exact position of - every split. Without skipped rows every rank's state is already - identical and merging is a no-op. + In row mode, each rank records exact consumer-committed progress for + its own splits and lower bounds for the rest, so elementwise maxima + recover both sample counts and permutation positions. In packed mode, + the state that emitted the most blocks for each logical split supplies + that split's permutation position and partial token buffer. Packed + states must cover every rank at the same global step. - Raises ``ValueError`` if the states are empty or were not produced by - the same run (mismatched seed, split count, epoch, or sample counts). + Raises ``ValueError`` if the states are empty, were not produced by + the same run, or do not represent the same global step. The merge is always all-to-all and topology-agnostic: collect the - ``state_dict()`` from every rank of the *previous* run into one list, - merge that whole list, and hand the identical merged result to every - rank of the *next* run — regardless of whether the rank count grew, - shrank, or stayed the same. There is no pairwise or subset merging - step, because each split's exact position is only known to whichever - rank owned that split, and the elementwise maximum needs every rank's - contribution to be correct. + ``state_dict()`` from every rank of the *previous* run into + one list, merge that whole list, and hand the identical merged result + to every rank of the *next* run — regardless of whether the + topology grew, shrank, or stayed the same. There is no pairwise or + subset merging step, because each split's exact state is only known to + whichever iterator owned that split. For example, checkpointing 8 ranks and resuming on 4 (the same pattern applies when growing, e.g. 4 ranks resuming on 8):: @@ -879,23 +1659,79 @@ class StreamingDataset(IterableDataset): if not states: raise ValueError("merge_state_dicts requires at least one state dict") first = states[0] + packed = "pack_buffers" in first + config_keys = ["shuffle_seed", "num_splits", "epoch"] + if packed: + config_keys.extend( + ["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"] + ) + for state in states[1:]: - for key in ("shuffle_seed", "num_splits", "epoch"): + if ("pack_buffers" in state) != packed: + raise ValueError("cannot merge packed and unpacked state dicts") + for key in config_keys: if state[key] != first[key]: raise ValueError( f"{key} mismatch across state dicts: " f"{state[key]} != {first[key]}" ) - if ( - state["samples_consumed_per_split"] - != first["samples_consumed_per_split"] - ): - raise ValueError( - "samples_consumed_per_split mismatch across state dicts; " - "state_dict() must be called at the same global step " - "boundary on every rank" + + if packed: + num_splits = first["num_splits"] + for state in states: + for key in ( + "samples_consumed_per_split", + "blocks_emitted_per_split", + ): + if len(state[key]) != num_splits: + raise ValueError( + f"{key} must contain one entry per logical split" + ) + + merged_consumed = [] + merged_emitted = [] + merged_buffers = {} + for split in range(num_splits): + owner = states[0] + owner_progress = ( + owner["blocks_emitted_per_split"][split], + owner["samples_consumed_per_split"][split], ) + for state in states[1:]: + progress = ( + state["blocks_emitted_per_split"][split], + state["samples_consumed_per_split"][split], + ) + if progress > owner_progress: + owner = state + owner_progress = progress + merged_consumed.append(owner["samples_consumed_per_split"][split]) + merged_emitted.append(owner["blocks_emitted_per_split"][split]) + buffer = owner["pack_buffers"].get( + split, owner["pack_buffers"].get(str(split)) + ) + if buffer is not None: + merged_buffers[split] = deepcopy(buffer) + + if len(set(merged_emitted)) > 1: + raise ValueError( + "packed state dicts were not captured at the same global " + "step or do not cover every rank" + ) + + merged = dict(first) + merged["samples_consumed_per_split"] = merged_consumed + merged["blocks_emitted_per_split"] = merged_emitted + merged["pack_buffers"] = merged_buffers + return merged + merged = dict(first) + merged["samples_consumed_per_split"] = [ + max(per_split) + for per_split in zip( + *(state["samples_consumed_per_split"] for state in states) + ) + ] all_positions = [ state.get( "positions_consumed_per_split", state["samples_consumed_per_split"] @@ -906,3 +1742,113 @@ class StreamingDataset(IterableDataset): max(per_split) for per_split in zip(*all_positions) ] return merged + + +class StreamingDataLoader(DataLoader): + """A PyTorch DataLoader with consumer-committed dataset checkpoints. + + PyTorch workers prefetch batches ahead of the trainer, so worker-local + producer progress is not a safe checkpoint. This loader carries a state + snapshot alongside every internal batch and applies it to the parent + [StreamingDataset][lancedb.streaming.StreamingDataset] only when that batch + is returned by ``next()``. + The trainer receives the same collated batch it would receive from a + standard ``torch.utils.data.DataLoader``. + + With more than one worker, row-mode ``state_dict()`` is available only at + complete logical step boundaries, when every split assigned to the rank has + the same consumed-sample count. Packed checkpoints require equal emitted-block + counts across the rank's splits for any worker count. ``persistent_workers=True`` + is not supported because prefetched worker copies cannot be restored from + parent-committed state. If batch collation raises, checkpointing remains + invalid for that dataset instance; restore the last valid checkpoint into a + fresh dataset before continuing. + Only one active iterator may own a dataset at a time, including when worker + processes are used. Exhausting or explicitly shutting down the iterator + releases that ownership. ``drop_last=True`` is not supported because worker + replicas discard incomplete tails independently, which cannot produce a + topology-independent checkpoint. + + Parameters are the same as ``torch.utils.data.DataLoader`` except that + ``dataset`` must be a + [StreamingDataset][lancedb.streaming.StreamingDataset]. + Subclasses that override ``StreamingDataset.__iter__`` are not supported + because the custom iterator cannot provide the exact per-yield checkpoint + snapshots required by this loader. + + Examples + -------- + >>> # dataset = StreamingDataset(table, num_splits=2) + >>> # loader = StreamingDataLoader(dataset, batch_size=8, num_workers=2) + >>> # batch = next(iter(loader)) + >>> # checkpoint = dataset.state_dict() + """ + + def __init__(self, dataset: StreamingDataset, *args, **kwargs): + if not isinstance(dataset, StreamingDataset): + raise TypeError("StreamingDataLoader requires a StreamingDataset") + if type(dataset).__iter__ is not StreamingDataset.__iter__: + raise TypeError( + "StreamingDataLoader does not support StreamingDataset subclasses " + "that override __iter__ because they cannot provide exact " + "per-yield checkpoint state" + ) + if kwargs.get("in_order", True) is False: + raise ValueError( + "StreamingDataLoader requires in_order=True for deterministic " + "consumer checkpoints" + ) + if kwargs.get("persistent_workers", False): + raise ValueError( + "StreamingDataLoader does not support persistent_workers=True " + "because worker prefetch state cannot be reset from a checkpoint" + ) + self._streaming_dataset = dataset + super().__init__(_StreamingDatasetAdapter(dataset), *args, **kwargs) + if self.drop_last: + raise ValueError( + "StreamingDataLoader does not support drop_last=True because " + "discarded worker tails cannot be checkpointed " + "topology-independently" + ) + self.collate_fn = _CheckpointCollate(self.collate_fn) + + def __iter__(self): + dataset = self._streaming_dataset + previous_lease = dataset._consumer_iterator_lease + owner_token = None + try: + owner_token = dataset._acquire_consumer_iterator() + state = dataset._checkpoint_snapshot() + packed = dataset._pack_sequences is not None + if packed: + blocks = state["blocks_emitted_per_split"] + rank_blocks = [blocks[split] for split in dataset._rank_splits] + if len(set(rank_blocks)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start from a partial packed " + "logical step; resume from a checkpoint whose splits " + "assigned to this rank have equal emitted-block counts" + ) + elif self.num_workers > 1: + samples = state["samples_consumed_per_split"] + rank_samples = [samples[split] for split in dataset._rank_splits] + if len(set(rank_samples)) > 1: + raise RuntimeError( + "StreamingDataLoader cannot start multiple workers from a " + "partial logical step; resume from a checkpoint whose " + "splits assigned to this rank have equal consumed-sample " + "counts" + ) + return _ConsumerCommitIterator( + super().__iter__(), + dataset, + owner_token=owner_token, + require_uniform=self.num_workers > 1 or packed, + ) + except BaseException: + if owner_token is not None: + dataset._release_consumer_iterator(owner_token) + else: + dataset._release_consumer_iterator_after_failed_acquire(previous_lease) + raise diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 913ab5289..75b7b1db8 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -40,7 +40,7 @@ from ._blob import ( from .types import BlobMode from lancedb.arrow import peek_reader from lancedb.background_loop import LOOP, embedding_executor -from lancedb.job import AsyncJob, Job +from lancedb.job import AsyncJob, Job, _typed_job from .dependencies import ( _check_for_hugging_face, _check_for_lance, @@ -72,7 +72,10 @@ from .index import ( FTS, ) from .expr import Expr -from .functions import FunctionApplication +from .functions import ( + FunctionApplication, + RefreshColumnResult as RefreshColumnJobResult, +) from .merge import LanceMergeInsertBuilder from .pydantic import LanceModel, model_to_dict from .query import ( @@ -2039,7 +2042,7 @@ class Table(ABC): """ @abstractmethod - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -2050,6 +2053,12 @@ class Table(ABC): than failing the job. On local tables the job runs in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + Job[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import lancedb @@ -2058,7 +2067,9 @@ class Table(ABC): >>> table.add_columns(computed={"doubled": "x * 2"}) AddColumnsResult(version=2) >>> job = table.refresh_column_async("doubled") - >>> job.wait() + >>> result = job.wait() + >>> result.rows_assigned + 2 >>> job.status() 'finished' """ @@ -4082,7 +4093,7 @@ class LanceTable(Table): [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column].""" return LOOP.run(self._table.refresh_column(column)) - def refresh_column_async(self, column: str) -> Job: + def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: """Fill a computed column's unfilled rows, returning a handle to the refresh job. See [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async]. @@ -6122,7 +6133,9 @@ class AsyncTable: """ return await self._inner.refresh_column(column) - async def refresh_column_async(self, column: str) -> AsyncJob: + async def refresh_column_async( + self, column: str + ) -> AsyncJob[RefreshColumnJobResult]: """ Like :meth:`refresh_column`, but returns a handle to the refresh job instead of blocking until it completes. @@ -6134,6 +6147,12 @@ class AsyncTable: in-process; on LanceDB Cloud and Enterprise it is the server's backfill job. + Returns + ------- + AsyncJob[RefreshColumnResult] + A job whose successful ``wait`` returns row counts plus the source + and published table versions. + Examples -------- >>> import asyncio @@ -6143,12 +6162,16 @@ class AsyncTable: ... table = await db.create_table("computed_job_async_demo", [{"x": 1}]) ... await table.add_columns(computed={"doubled": "x * 2"}) ... job = await table.refresh_column_async("doubled") - ... await job.wait() + ... result = await job.wait() + ... assert result.rows_assigned == 1 ... return await job.status() >>> asyncio.run(refresh_in_background()) 'finished' """ - return AsyncJob(await self._inner.refresh_column_async(column)) + return _typed_job( + await self._inner.refresh_column_async(column), + RefreshColumnJobResult.from_json, + ) async def alter_columns( self, *alterations: Iterable[dict[str, Any]] @@ -6801,21 +6824,21 @@ class Branches: """Diff a branch against main.""" return LOOP.run(self._table.branches.diff(from_branch)) - def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. Parameters ---------- from_branch: str - Branch to merge from. + Branch to cherry-pick from. dry_run: bool, default False - When True, only preview. When False, attempt the merge. + When True, only preview. When False, attempt the cherry-pick. Notes ----- - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return LOOP.run(self._table.branches.merge(from_branch, dry_run)) + return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run)) def _wrap( self, async_table: "AsyncTable", version: Optional[int] = None @@ -6951,9 +6974,11 @@ class AsyncBranches: """Diff a branch against main.""" return await self._table.branches.diff(from_branch) - async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]: - """Merge a branch into main, or dry-run. + async def cherry_pick( + self, from_branch: str, dry_run: bool = False + ) -> Dict[str, Any]: + """Cherry-pick a branch onto main, or dry-run. - A rejected merge returns ``status="rejected"`` instead of raising. + A failed cherry-pick returns ``status="failed"`` instead of raising. """ - return await self._table.branches.merge(from_branch, dry_run) + return await self._table.branches.cherry_pick(from_branch, dry_run) diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 38bbb53fb..aeb9feeb8 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection): job = tmp_db.drop_table_async("test") assert job.id is None assert job.status() == "finished" - job.wait() + assert job.wait() is None assert tmp_db.table_names() == [] tmp_db.create_table("test", data=data) @@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection job = await tmp_db_async.drop_table_async("test") assert job.id is None assert await job.status() == "finished" - await job.wait() + assert await job.wait() is None assert await tmp_db_async.table_names() == [] diff --git a/python/python/tests/test_elastic_dataloader.py b/python/python/tests/test_elastic_dataloader.py index 0c1a70765..da27e5bfc 100644 --- a/python/python/tests/test_elastic_dataloader.py +++ b/python/python/tests/test_elastic_dataloader.py @@ -32,6 +32,7 @@ Parameters used throughout: import dataclasses import logging +import threading from unittest.mock import patch import lancedb @@ -46,6 +47,7 @@ from utils import ( torch = pytest.importorskip("torch") streaming = pytest.importorskip("lancedb.streaming") StreamingDataset = streaming.StreamingDataset +StreamingDataLoader = streaming.StreamingDataLoader # --------------------------------------------------------------------------- # Dataset parameters @@ -92,6 +94,27 @@ class FakeWorkerInfo: num_workers: int +def _collate_with_first_batch_error(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise ValueError("first batch fails") + return ids + + +def _collate_with_first_batch_stop(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise StopIteration("first batch stopped") + return ids + + +def _collate_with_first_batch_interrupt(samples): + ids = [sample["id"] for sample in samples] + if ids == [0, 1]: + raise KeyboardInterrupt("first batch interrupted") + return ids + + # --------------------------------------------------------------------------- # Fixtures # --------------------------------------------------------------------------- @@ -1008,6 +1031,565 @@ def test_multi_worker_elastic_det_across_worker_counts(lance_table): # ── Resumability with num_workers ───────────────────────────────────────────── +def test_streaming_dataloader_commits_only_consumed_worker_batches(tmp_path): + """Prefetched worker state is committed only as the trainer receives it.""" + db = lancedb.connect(tmp_path) + table = db.create_table( + "worker_commit", pa.table({"id": [1, 2, 3, 4, 10, 20, 30, 40]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + iterator = iter(loader) + try: + first = next(iterator)["id"].tolist() + + assert first == [1, 2] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2, 0] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + second = next(iterator)["id"].tolist() + assert second == [10, 20] + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [2, 2] + uninterrupted = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=4, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["id"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + assert remaining == uninterrupted == [[3, 4], [30, 40]] + + +def test_distributed_checkpoint_uses_rank_local_worker_boundary(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("rank_boundary", pa.table({"id": list(range(8))})) + dataset = StreamingDataset( + table, + num_splits=4, + shuffle=False, + rank=0, + world_size=2, + ) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [ + 1, + 0, + 0, + 0, + ] + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + assert next(iterator)["id"].tolist() == [2] + checkpoint = dataset.state_dict() + remaining = [batch["id"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + assert checkpoint["samples_consumed_per_split"] == [1, 1, 0, 0] + assert remaining == [[1], [3]] + + +def test_standard_dataloader_rejects_stale_parent_checkpoint(tmp_path): + """A standard DataLoader must not expose prefetched producer progress.""" + db = lancedb.connect(tmp_path) + table = db.create_table("untracked_workers", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + # Merely constructing the checkpoint-aware loader must not authorize a + # later plain DataLoader's worker progress. + StreamingDataLoader(dataset, batch_size=2, num_workers=0) + loader = torch.utils.data.DataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + ) + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [1, 2] + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_persistent_workers(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("persistent_workers", pa.table({"id": [1, 2]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + + with pytest.raises(ValueError, match="persistent_workers=True"): + StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + persistent_workers=True, + ) + + +def test_collate_failure_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "collate_failure", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=2, + multiprocessing_context="spawn", + collate_fn=_collate_with_first_batch_error, + prefetch_factor=2, + ) + iterator = iter(loader) + try: + with pytest.raises(ValueError, match="first batch fails"): + next(iterator) + assert next(iterator) == [100, 101] + assert next(iterator) == [2, 3] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + list(iterator) + finally: + iterator._shutdown_workers() + + +def test_collate_stop_iteration_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_stop", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_stop, + ) + iterator = iter(loader) + + with pytest.raises(RuntimeError, match="collate_fn raised StopIteration"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_batch_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("collate_interrupt", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=_collate_with_first_batch_interrupt, + ) + iterator = iter(loader) + + with pytest.raises(KeyboardInterrupt, match="first batch interrupted"): + next(iterator) + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + assert list(iterator) == [[2, 3], [4, 5]] + + +def test_parent_commit_base_exception_invalidates_consumer_checkpoint(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("commit_interrupt", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + iterator = iter(loader) + real_commit = dataset._commit_worker_state + + def interrupt_after_commit(state, *, require_uniform): + real_commit(state, require_uniform=require_uniform) + raise KeyboardInterrupt("after parent commit") + + with patch.object( + dataset, "_commit_worker_state", side_effect=interrupt_after_commit + ): + with pytest.raises(KeyboardInterrupt, match="after parent commit"): + next(iterator) + + assert dataset._checkpoint_snapshot()["samples_consumed_per_split"] == [2] + with pytest.raises(RuntimeError, match="failed before it was returned"): + dataset.state_dict() + + +def test_direct_iteration_surfaces_prefetch_failure_before_committing_row( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("prefetch_failure", pa.table({"id": list(range(4))})) + release = threading.Event() + failed = threading.Event() + real_getitems = streaming.Permutation.__getitems__ + + def controlled_getitems(permutation, indices): + if indices and indices[0] >= 2: + assert release.wait(timeout=5) + failed.set() + raise RuntimeError("later prefetched I/O failed") + return real_getitems(permutation, indices) + + class SignalDict(dict): + def __setitem__(self, key, value): + super().__setitem__(key, value) + release.set() + assert failed.wait(timeout=5) + + monkeypatch.setattr(streaming.Permutation, "__getitems__", controlled_getitems) + dataset = StreamingDataset( + table, + num_splits=1, + shuffle=False, + read_batch_size=2, + io_queue_depth=2, + ) + dataset._resume_positions = SignalDict() + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + with pytest.raises(RuntimeError, match="later prefetched I/O failed"): + next(iterator) + + checkpoint = dataset.state_dict() + assert checkpoint["samples_consumed_per_split"] == [1] + assert checkpoint["positions_consumed_per_split"] == [1] + + +@pytest.mark.parametrize("workers", [0, 1, 2]) +def test_streaming_dataloader_rejects_drop_last(tmp_path, workers): + db = lancedb.connect(tmp_path) + table = db.create_table("drop_last", pa.table({"id": [0, 1, 2]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + worker_options = {"multiprocessing_context": "spawn"} if workers else {} + + with pytest.raises(ValueError, match="drop_last=True"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=workers, + drop_last=True, + **worker_options, + ) + + +def test_streaming_dataloader_owns_one_iterator_until_teardown(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("iterator_owner", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader( + dataset, + batch_size=2, + num_workers=1, + multiprocessing_context="spawn", + ) + + first = iter(loader) + try: + assert next(first)["id"].tolist() == [0, 1] + with pytest.raises(RuntimeError, match="concurrent iteration"): + iter(loader) + finally: + first._shutdown_workers() + + second = iter(loader) + try: + assert [batch["id"].tolist() for batch in second] == [[2, 3]] + except BaseException: + second._shutdown_workers() + raise + + # Natural exhaustion releases ownership too. + third = iter(loader) + try: + assert list(third) == [] + finally: + third._shutdown_workers() + + +def test_zero_worker_shutdown_closes_inner_iterator_before_release(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("zero_worker_shutdown", pa.table({"id": list(range(6))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + first._shutdown_workers() + + assert dataset._consumer_iterator_active is False + assert dataset._raw_batches_ref is None + second = iter(loader) + try: + with pytest.raises(StopIteration): + next(first) + assert next(second)["id"].tolist() == [2, 3] + finally: + second._shutdown_workers() + + +def test_direct_and_loader_admission_share_one_atomic_lease(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("direct_loader_lease", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + entered = threading.Event() + release = threading.Event() + direct_result = [] + direct_error = [] + contender = [] + real_resolve = dataset._resolve_my_splits + + def controlled_resolve(): + if threading.current_thread().name == "direct-start": + entered.set() + assert release.wait(timeout=5) + return real_resolve() + + def advance_direct(iterator): + try: + direct_result.append(next(iterator)["id"]) + except BaseException as exc: + direct_error.append(exc) + + monkeypatch.setattr(dataset, "_resolve_my_splits", controlled_resolve) + direct = iter(dataset) + thread = threading.Thread( + target=advance_direct, args=(direct,), name="direct-start" + ) + thread.start() + assert entered.wait(timeout=5) + try: + with pytest.raises(RuntimeError, match="concurrent iteration"): + contender.append(iter(loader)) + finally: + release.set() + thread.join(timeout=5) + if contender: + contender[0]._shutdown_workers() + direct.close() + + assert not thread.is_alive() + assert direct_error == [] + assert direct_result == [0] + + +def test_loader_acquires_before_snapshot_and_cleans_interrupted_acquire( + tmp_path, monkeypatch +): + db = lancedb.connect(tmp_path) + table = db.create_table("lease_snapshot", pa.table({"id": list(range(4))})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=2, num_workers=0) + first = iter(loader) + assert next(first)["id"].tolist() == [0, 1] + + entered = threading.Event() + release = threading.Event() + pending = [] + pending_errors = [] + observed_snapshots = [] + real_acquire = dataset._acquire_consumer_iterator + real_snapshot = dataset._checkpoint_snapshot + + def controlled_acquire(): + if threading.current_thread().name == "stale-start": + entered.set() + assert release.wait(timeout=5) + return real_acquire() + + def recording_snapshot(): + state = real_snapshot() + if threading.current_thread().name == "stale-start": + observed_snapshots.append(state["samples_consumed_per_split"]) + return state + + def create_pending_iterator(): + try: + pending.append(iter(loader)) + except BaseException as exc: + pending_errors.append(exc) + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", controlled_acquire) + monkeypatch.setattr(dataset, "_checkpoint_snapshot", recording_snapshot) + thread = threading.Thread(target=create_pending_iterator, name="stale-start") + thread.start() + assert entered.wait(timeout=5) + assert next(first)["id"].tolist() == [2, 3] + with pytest.raises(StopIteration): + next(first) + release.set() + thread.join(timeout=5) + + assert not thread.is_alive() + assert pending_errors == [] + assert observed_snapshots == [[4]] + assert len(pending) == 1 + assert list(pending[0]) == [] + assert dataset.state_dict()["samples_consumed_per_split"] == [4] + + def interrupted_acquire(): + real_acquire() + raise KeyboardInterrupt("after acquire") + + monkeypatch.setattr(dataset, "_acquire_consumer_iterator", interrupted_acquire) + with pytest.raises(KeyboardInterrupt, match="after acquire"): + iter(loader) + assert dataset._consumer_iterator_active is False + + +def test_consumer_iterator_lease_publication_is_atomic(tmp_path, monkeypatch): + db = lancedb.connect(tmp_path) + table = db.create_table("atomic_lease", pa.table({"id": [0, 1]})) + dataset = StreamingDataset(table, num_splits=1, shuffle=False) + loader = StreamingDataLoader(dataset, batch_size=1, num_workers=0) + real_get_ident = streaming.threading.get_ident + calls = 0 + + def interrupt_during_publication(): + nonlocal calls + calls += 1 + if calls == 1: + raise KeyboardInterrupt("during lease mutation") + return real_get_ident() + + monkeypatch.setattr(streaming.threading, "get_ident", interrupt_during_publication) + with pytest.raises(KeyboardInterrupt, match="during lease mutation"): + iter(loader) + monkeypatch.setattr(streaming.threading, "get_ident", real_get_ident) + + assert dataset._consumer_iterator_active is False + iterator = iter(loader) + try: + assert next(iterator)["id"].tolist() == [0] + finally: + iterator._shutdown_workers() + + +def test_streaming_dataloader_rejects_dataset_iter_override(tmp_path): + class CustomizedDataset(StreamingDataset): + def __iter__(self): + return iter([1000, 1001]) + + db = lancedb.connect(tmp_path) + table = db.create_table("custom_iteration", pa.table({"id": [0, 1, 2]})) + dataset = CustomizedDataset(table, num_splits=1, shuffle=False) + + assert list(dataset) == [1000, 1001] + with pytest.raises(TypeError, match="override __iter__"): + StreamingDataLoader( + dataset, + batch_size=2, + num_workers=0, + collate_fn=list, + ) + + +def test_interleaved_adapters_do_not_authorize_plain_iteration(tmp_path): + db = lancedb.connect(tmp_path) + table_a = db.create_table("adapter_a", pa.table({"id": [0, 1]})) + table_b = db.create_table("adapter_b", pa.table({"id": [10, 11]})) + dataset_a = StreamingDataset(table_a, num_splits=1, shuffle=False) + dataset_b = StreamingDataset(table_b, num_splits=1, shuffle=False) + initial_state = dataset_a.state_dict() + + owner_a = dataset_a._acquire_consumer_iterator() + owner_b = dataset_b._acquire_consumer_iterator() + try: + iterator_a = iter(streaming._StreamingDatasetAdapter(dataset_a)) + iterator_b = iter(streaming._StreamingDatasetAdapter(dataset_b)) + assert next(iterator_a).data["id"] == 0 + assert next(iterator_b).data["id"] == 10 + assert [sample.data["id"] for sample in iterator_a] == [1] + assert [sample.data["id"] for sample in iterator_b] == [11] + finally: + dataset_a._release_consumer_iterator(owner_a) + dataset_b._release_consumer_iterator(owner_b) + + dataset_a.load_state_dict(initial_state) + with patch( + "lancedb.streaming.get_worker_info", + return_value=FakeWorkerInfo(id=0, num_workers=1), + ): + plain_iterator = iter(dataset_a) + assert next(plain_iterator)["id"] == 0 + plain_iterator.close() + + assert dataset_a._untracked_worker_iteration[0] == 1 + with pytest.raises(RuntimeError, match="Use StreamingDataLoader"): + dataset_a.state_dict() + + +def test_resume_from_partial_split_cycle_preserves_remaining_order(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("partial_cycle", pa.table({"id": [1, 2, 10, 20]})) + dataset = StreamingDataset(table, num_splits=2, shuffle=False) + iterator = iter(dataset) + + assert next(iterator)["id"] == 1 + checkpoint = dataset.state_dict() + iterator.close() + assert checkpoint["samples_consumed_per_split"] == [1, 0] + + resumed = StreamingDataset(table, num_splits=2, shuffle=False) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == [10, 2, 20] + + +def test_partial_cycle_resume_preserves_skip_truncation(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "partial_skip", pa.table({"id": [0, 1, 2, 3, 100, 101, 102, 103]}) + ) + kwargs = dict( + num_splits=2, + shuffle=False, + transform=_failing_transform({1, 2, 3}), + on_transform_error="skip", + ) + dataset = StreamingDataset(table, **kwargs) + iterator = iter(dataset) + + assert next(iterator)["id"] == 0 + checkpoint = dataset.state_dict() + uninterrupted = [row["id"] for row in iterator] + + resumed = StreamingDataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + assert [row["id"] for row in resumed] == uninterrupted == [100] + + def test_multi_worker_resumability_same_topology(lance_table): """Checkpoint with num_workers=2, resume with num_workers=2: exact continuation.""" world_size = 1 @@ -1374,6 +1956,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli ) +# --------------------------------------------------------------------------- +# Backpressure / transform_queue_depth tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("transform_queue_depth", [0, -1]) +def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth): + """transform_queue_depth=0 or negative must raise ValueError.""" + with pytest.raises( + ValueError, match="transform_queue_depth must be greater than 0" + ): + StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + transform_queue_depth=transform_queue_depth, + ) + + +@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4]) +def test_transform_queue_depth_correctness(lance_table, transform_queue_depth): + """With backpressure enabled, every row is still yielded exactly once.""" + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=transform_queue_depth, + read_batch_size=8, + ) + items = list(ds) + assert sorted(item["id"] for item in items) == list(range(NUM_ROWS)) + + +def test_transform_queue_depth_matches_no_backpressure(lance_table): + """With backpressure enabled the same samples are produced as without it.""" + ds_unlimited = StreamingDataset( + lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED + ) + ds_limited = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=1, + ) + assert [item["id"] for item in ds_unlimited] == [ + item["id"] for item in ds_limited + ], "transform_queue_depth must not affect the sample ordering or set" + + +def test_transform_queue_depth_bounds_cooked_rows(lance_table): + """prefetch_queue_depth stays within transform_queue_depth * read_batch_size + per split when observed from the main thread during iteration.""" + n_splits = 4 + batch_size = 8 + cooked_depth = 2 + # max cooked rows across all 4 splits: 4 * 2 * 8 = 64 + max_allowed = n_splits * cooked_depth * batch_size + + ds = StreamingDataset( + lance_table, + num_splits=n_splits, + shuffle_seed=SHUFFLE_SEED, + transform_queue_depth=cooked_depth, + read_batch_size=batch_size, + transform_parallelism=1, + world_size=1, + ) + + peak = 0 + for _ in ds: + depth = ds.prefetch_queue_depth + if depth > peak: + peak = depth + + # The main thread observes depth *after* popping a row, so the peak is at + # most max_allowed (one row already popped from the split just served). + assert peak <= max_allowed, ( + f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}" + ) + + +def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path): + """Admission requires a full read_batch_size of free space, not just one slot. + + The test intercepts ThreadPoolExecutor.submit to make I/O calls execute + synchronously on the main thread. This ensures all raw batches land in + raw_batches (via _drain_io) before _try_submit_tx evaluates the admission + predicate for the first time. Without this, the I/O future for batch N+1 + might still be in io_pending at the capacity-minus-one transition, leaving + raw_batches empty and causing _try_submit_tx to skip the admission check + entirely — so both the correct and the broken predicate produce depth=0 + observations and the test cannot distinguish them. + + With all raw batches pre-loaded in raw_batches the 4→3 cooked transition + (consuming one row from a full cooked queue) always triggers _try_submit_tx + against a non-empty raw_batches. + + With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4. + A transform may only be submitted when in_pipeline + batch_size <= 4, i.e. + when in_pipeline == 0 (cooked is completely empty). Under the old broken + predicate (in_pipeline >= max_cooked_rows) the second transform would be + admitted with cooked containing batch_size-1 rows still unconsumed. + """ + import concurrent.futures as cf + from concurrent.futures import ThreadPoolExecutor + from unittest.mock import patch + + db = lancedb.connect(tmp_path) + batch_size = 4 + # Four full batches → four transform submissions to observe. + table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))})) + + cooked_at_submit: list[int] = [] + + original_submit = ThreadPoolExecutor.submit + + def tracking_submit(self, fn, *args, **kwargs): + name = getattr(fn, "__name__", "") + if name == "_io_call": + # Run I/O synchronously on the calling (main) thread and return an + # already-completed Future. _drain_io checks fut.done(), so a + # completed Future is moved to raw_batches immediately on the next + # _advance call — making raw-batch readiness deterministic at the + # capacity-minus-one transition instead of depending on I/O thread + # scheduling. + fut = cf.Future() + try: + fut.set_result(fn(*args, **kwargs)) + except Exception as exc: + fut.set_exception(exc) + return fut + if name == "_tx_call_guarded": + # Capture cooked depth synchronously on the main thread before the + # transform worker can drain the queue. + ref = ds._cooked_ref + cooked_at_submit.append(len(ref[0]) if ref is not None else -1) + return original_submit(self, fn, *args, **kwargs) + + with patch.object(ThreadPoolExecutor, "submit", tracking_submit): + ds = StreamingDataset( + table, + num_splits=1, + shuffle_seed=42, + read_batch_size=batch_size, + transform_queue_depth=1, + transform_parallelism=1, + ) + list(ds) + + assert len(cooked_at_submit) == 4, ( + f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}" + ) + # With full-batch backpressure each transform is only admitted when the + # cooked queue is completely empty (depth == 0). The old broken predicate + # would admit at depth == batch_size - 1 == 3. + assert all(depth == 0 for depth in cooked_at_submit), ( + "Transform admitted with non-empty cooked queue; full-batch backpressure " + "requires in_pipeline + batch_size <= max_cooked_rows before admission. " + f"Cooked depths at each submission: {cooked_at_submit}" + ) + + +# --------------------------------------------------------------------------- +# Deprecated parameter name tests +# --------------------------------------------------------------------------- + + +def test_prefetch_batches_deprecated_warns(lance_table, caplog): + """prefetch_batches logs a deprecation warning and behaves like io_queue_depth.""" + with caplog.at_level(logging.WARNING, logger="lancedb.streaming"): + ds = StreamingDataset( + lance_table, + num_splits=NUM_SPLITS, + shuffle_seed=SHUFFLE_SEED, + prefetch_batches=2, + ) + messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING] + assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), ( + f"Expected deprecation warning mentioning io_queue_depth; got: {messages}" + ) + assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS)) + + def test_filter_limits_rows(tmp_path): """A filter expression is applied to the permutation so only matching rows are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded.""" @@ -1836,6 +2600,23 @@ def test_merge_state_dicts_validates_consistency(lance_table): StreamingDataset.merge_state_dicts([]) +def test_merge_state_dicts_combines_nonuniform_consumer_progress(lance_table): + dataset = StreamingDataset( + lance_table, num_splits=2, shuffle=False, shuffle_seed=SHUFFLE_SEED + ) + rank0 = dataset.state_dict() + rank0["samples_consumed_per_split"] = [2, 0] + rank0["positions_consumed_per_split"] = [2, 0] + rank1 = dataset.state_dict() + rank1["samples_consumed_per_split"] = [0, 2] + rank1["positions_consumed_per_split"] = [0, 2] + + merged = StreamingDataset.merge_state_dicts([rank0, rank1]) + + assert merged["samples_consumed_per_split"] == [2, 2] + assert merged["positions_consumed_per_split"] == [2, 2] + + def test_load_state_dict_without_positions_key(lance_table): """Checkpoints from before positions_consumed_per_split existed still resume exactly (positions equal sample counts when nothing is skipped).""" @@ -1931,6 +2712,273 @@ def test_shuffle_seed_none_generates_stable_seed(lance_table): assert first == second, "Same resolved seed must produce the same ordering" +# Sequence packing tests + + +def _create_token_table(tmp_path, documents): + db = lancedb.connect(tmp_path) + tokens = pa.array(documents, type=pa.list_(pa.int64())) + return db.create_table("tokens", pa.table({"tokens": tokens})) + + +def _packed_dataset(table, pack_sequences, *, blocks_per_epoch, pad_id=0, **kwargs): + return StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=pack_sequences, + eos_id=9, + pad_id=pad_id, + blocks_per_epoch=blocks_per_epoch, + **kwargs, + ) + + +def test_pack_sequences_emits_blocks_and_pads_final_tail(tmp_path): + table = _create_token_table(tmp_path, [[1, 2], [3, 4], [5]]) + dataset = _packed_dataset(table, 6, blocks_per_epoch=2) + + blocks = list(dataset) + + assert len(blocks) == 2 + assert blocks[0]["input_ids"].tolist() == [1, 2, 9, 3, 4, 9] + assert blocks[0]["doc_ids"].tolist() == [0, 0, 0, 1, 1, 1] + assert blocks[1]["input_ids"].tolist() == [5, 9, 0, 0, 0, 0] + assert blocks[1]["doc_ids"].tolist() == [0, 0, 0, 0, 0, 0] + assert blocks[0]["input_ids"].dtype == torch.int64 + assert blocks[0]["doc_ids"].dtype == torch.int64 + + +def test_pack_sequences_pads_lagging_splits(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + dataset = _packed_dataset(table, 5, blocks_per_epoch=6, num_splits=2) + input_ids = [block["input_ids"].tolist() for block in dataset] + # Split 0 has four real tokens including EOS markers, while split 1 has + # eleven. Packing must emit three complete two-split cycles. + assert input_ids == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + [0, 0, 0, 0, 0], + [15, 16, 17, 9, 20], + [0, 0, 0, 0, 0], + [9, 0, 0, 0, 0], + ] + + per_rank = [] + for rank in range(2): + rank_dataset = _packed_dataset( + table, + 5, + blocks_per_epoch=6, + num_splits=2, + world_size=2, + rank=rank, + ) + per_rank.append([block["input_ids"].tolist() for block in rank_dataset]) + + assert [len(blocks) for blocks in per_rank] == [3, 3] + sharded = [block for cycle in zip(*per_rank) for block in cycle] + assert sharded == input_ids + + +def test_pack_sequences_auto_estimates_filtered_token_column(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table( + "tokens", + pa.table( + { + "tokens": pa.array([[1] * 4, [2] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ), + ) + table.add( + pa.table( + { + "tokens": pa.array([[3] * 4, [4] * 9], type=pa.list_(pa.int64())), + "keep": [True, False], + } + ) + ) + + with pytest.warns(UserWarning, match="approximate token-count sample"): + dataset = _packed_dataset( + table, + 5, + blocks_per_epoch="auto", + num_splits=2, + filter="keep", + ) + + # Two kept documents contain 8 tokens plus 2 EOS tokens: two blocks. + assert dataset.state_dict()["blocks_per_epoch"] == 2 + + +def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]], + ) + kwargs = dict(pack_sequences=5, blocks_per_epoch=6, num_splits=2) + reference = list(_packed_dataset(table, **kwargs)) + + datasets = [ + _packed_dataset(table, world_size=2, rank=rank, **kwargs) for rank in range(2) + ] + iterators = [iter(dataset) for dataset in datasets] + first_cycle = [next(iterator) for iterator in iterators] + checkpoint = StreamingDataset.merge_state_dicts( + [dataset.state_dict() for dataset in datasets] + ) + for iterator in iterators: + iterator.close() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + actual_remaining = list(resumed) + + assert [block["input_ids"].tolist() for block in first_cycle] == [ + [1, 9, 2, 9, 0], + [10, 11, 12, 13, 14], + ] + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert [block["input_ids"].tolist() for block in actual_remaining] == [ + block["input_ids"].tolist() for block in reference[2:] + ] + assert [block["doc_ids"].tolist() for block in actual_remaining] == [ + block["doc_ids"].tolist() for block in reference[2:] + ] + + +def test_packed_checkpoint_requires_complete_split_cycle(tmp_path): + table = _create_token_table(tmp_path, [[1], [2], [10], [20]]) + dataset = _packed_dataset(table, pack_sequences=3, blocks_per_epoch=4, num_splits=2) + iterator = iter(dataset) + + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + assert dataset.state_dict()["blocks_emitted_per_split"] == [1, 1] + iterator.close() + + +def test_streaming_dataloader_commits_consumed_packed_batches(tmp_path): + table = _create_token_table( + tmp_path, + [[1], [2], [3], [4], [10], [20], [30], [40]], + ) + kwargs = dict(pack_sequences=4, blocks_per_epoch=4, num_splits=2) + dataset = _packed_dataset(table, **kwargs) + loader = StreamingDataLoader( + dataset, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + iterator = iter(loader) + try: + next(iterator) + with pytest.raises(RuntimeError, match="complete logical step boundary"): + dataset.state_dict() + + next(iterator) + checkpoint = dataset.state_dict() + uninterrupted = [batch["input_ids"].tolist() for batch in iterator] + finally: + iterator._shutdown_workers() + + resumed = _packed_dataset(table, **kwargs) + resumed.load_state_dict(checkpoint) + resumed_loader = StreamingDataLoader( + resumed, + batch_size=1, + num_workers=2, + multiprocessing_context="spawn", + prefetch_factor=2, + ) + resumed_iterator = iter(resumed_loader) + try: + remaining = [batch["input_ids"].tolist() for batch in resumed_iterator] + finally: + resumed_iterator._shutdown_workers() + + assert checkpoint["blocks_emitted_per_split"] == [1, 1] + assert remaining == uninterrupted + + +def test_pack_sequences_validates_configuration_and_tokens(tmp_path): + table = _create_token_table(tmp_path, [[1, 2]]) + + with pytest.raises(ValueError, match="pad_id is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + ) + + with pytest.raises(ValueError, match="blocks_per_epoch is required"): + StreamingDataset( + table, + shuffle=False, + columns=["tokens"], + pack_sequences=4, + eos_id=9, + pad_id=0, + ) + + with pytest.raises(ValueError, match="must be divisible"): + _packed_dataset(table, 4, blocks_per_epoch=3, num_splits=2) + + with pytest.raises(ValueError, match="positive integer or 'auto'"): + _packed_dataset(table, 4, blocks_per_epoch="estimate") + + checkpoint = _packed_dataset(table, 4, blocks_per_epoch=1).state_dict() + resumed = _packed_dataset(table, 4, blocks_per_epoch=1, pad_id=8) + with pytest.raises(ValueError, match="pad_id mismatch"): + resumed.load_state_dict(checkpoint) + + float_db = lancedb.connect(tmp_path / "float") + float_table = float_db.create_table( + "tokens", + pa.table({"tokens": pa.array([[1.5, 2.5]], type=pa.list_(pa.float64()))}), + ) + with pytest.raises(ValueError, match="token column with integer values"): + _packed_dataset(float_table, 4, blocks_per_epoch=1) + + null_db = lancedb.connect(tmp_path / "null") + null_table = null_db.create_table( + "tokens", + pa.table({"tokens": pa.array([None], type=pa.list_(pa.int64()))}), + ) + with pytest.raises(ValueError, match="does not support null token lists"): + list(_packed_dataset(null_table, 4, blocks_per_epoch=1)) + + null_value_db = lancedb.connect(tmp_path / "null_value") + null_value_table = null_value_db.create_table( + "tokens", + pa.table( + {"tokens": pa.array([[1], [2, None], [3]], type=pa.list_(pa.int64()))} + ), + ) + blocks = list( + _packed_dataset( + null_value_table, + 2, + blocks_per_epoch=2, + on_transform_error="skip", + ) + ) + assert [block["input_ids"].tolist() for block in blocks] == [[1, 9], [3, 9]] + + # --------------------------------------------------------------------------- # Doc examples — each test mirrors the code snippet in index.mdx so that # broken doc examples are caught before they ship. @@ -1954,7 +3002,7 @@ def test_doc_example_basic(tmp_path): def test_doc_example_prefetch_params(tmp_path): - """doc: Prefetching — read_batch_size and prefetch_batches still cover all rows.""" + """doc: Prefetching — read_batch_size and io_queue_depth still cover all rows.""" db = lancedb.connect(tmp_path) table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))})) @@ -1963,7 +3011,7 @@ def test_doc_example_prefetch_params(tmp_path): num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED, read_batch_size=8, - prefetch_batches=2, + io_queue_depth=2, ) assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS)) diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 99c68876c..c67e17520 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -3,7 +3,12 @@ from __future__ import annotations +import base64 import contextlib +import functools +import importlib.util +import types +from datetime import date import http.server import json from pathlib import Path @@ -16,6 +21,9 @@ import pytest import lancedb from lancedb.functions import UdfDefinition, udf +THRESHOLD = 20 +_CACHE = None + FIXTURES = ( Path(__file__).parents[3] @@ -71,9 +79,396 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): _assert_no_secret_values(request) +def _run_packaged(definition, *args): + """Execute the shipped artifact in a fresh namespace, as a worker would.""" + source = base64.b64decode(definition.registration_request.artifact.content.data) + namespace: dict = {} + exec(compile(source, "", "exec"), namespace) + return namespace[definition.registration_request.artifact.entrypoint](*args) + + +def test_udf_packages_attribute_access_and_body_imports(): + @udf + def word_norm(body: str) -> float: + import numpy as np + + try: + words = body.split() + except AttributeError as error: + raise ValueError(str(error)) from error + return float(np.linalg.norm([len(w) for w in words])) + + assert _run_packaged(word_norm, "aa bb") == pytest.approx(8**0.5) + + +def test_udf_packages_module_globals_and_global_caches(): + @udf + def label(value: int) -> str: + return "big" if value >= THRESHOLD else "small" + + assert _run_packaged(label, 21) == "big" + + @udf + def cached(value: int) -> int: + global _CACHE + if _CACHE is None: + _CACHE = 40 + return _CACHE + value + + assert _run_packaged(cached, 2) == 42 + + +def test_udf_annotations_are_not_runtime_names(): + @udf + def identity(value: date) -> date: + return value + + assert _run_packaged(identity, date(2026, 8, 25)) == date(2026, 8, 25) + + +def test_udf_nested_scopes_resolve_lexically(): + @udf + def score(value: int) -> int: + offset = 2 + + def add_offset() -> int: + return value + offset + + return add_offset() + sum(v for v in [0]) + + assert _run_packaged(score, 3) == 5 + + +def test_udf_resolves_module_globals_before_builtins(tmp_path): + module_path = tmp_path / "shadowing_udfs.py" + module_path.write_text( + "max = 7\n" + "len = lambda _: 99\n" + "\n" + "def uses_literal_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return max\n" + " return nested() + value\n" + "\n" + "def uses_callable_shadow(value: int) -> int:\n" + " def nested() -> int:\n" + " return len([1])\n" + " return nested() + value\n" + ) + spec = importlib.util.spec_from_file_location("shadowing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + + # The module's `max = 7` is what the interpreter would use, so it ships. + assert _run_packaged(udf(module.uses_literal_shadow), 1) == 8 + # A callable global cannot ship; it must not be silently swapped for the builtin. + with pytest.raises(TypeError, match="unsupported global value of type function"): + udf(module.uses_callable_shadow) + + +def test_canonical_arrow_type_is_exactly_the_grammar(): + from lancedb.functions import _GRAMMAR_PRIMITIVES, _canonical_arrow_type + + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + primitives = [ + case["arrow_type"] for case in golden["valid"] if "<" not in case["arrow_type"] + ] + assert [name for _, name in _GRAMMAR_PRIMITIVES] == primitives + for outside in [ + pa.timestamp("us"), + pa.decimal128(10, 2), + pa.large_string(), + pa.large_binary(), + pa.binary(4), + pa.duration("s"), + pa.struct([pa.field("a", pa.int32())]), + pa.list_(pa.float32(), 0), + pa.list_(pa.timestamp("us")), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + + +def test_udf_nested_annotations_are_postponed_in_the_artifact(): + @udf + def score(value: int) -> int: + def identity(item: date) -> date: + return item + + identity(date(2026, 8, 25)) + return value + + assert _run_packaged(score, 3) == 3 + + +def test_udf_ships_globals_the_body_deletes(): + @udf + def clear(value: int) -> int: + global _CACHE + del _CACHE + return value + + assert _run_packaged(clear, 3) == 3 + + +def test_udf_rejects_a_module_global_that_does_not_import_as_itself(tmp_path): + module_path = tmp_path / "fake_module_udfs.py" + module_path.write_text( + "import types\n" + "np = types.ModuleType('numpy')\n" + "np.sqrt = lambda x: 0\n" + "\n" + "def score(value: int) -> int:\n" + " return int(np.sqrt(value))\n" + ) + spec = importlib.util.spec_from_file_location("fake_module_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(TypeError, match="does not import as 'numpy'"): + udf(module.score) + + +def test_udf_rejects_a_module_level_namespace_alias(tmp_path): + module_path = tmp_path / "aliasing_udfs.py" + module_path.write_text( + "import builtins as b\n" + "THRESHOLD = 5\n" + "\n" + "def score(value: int) -> int:\n" + " return value + b.vars(b.__import__('aliasing_udfs'))['THRESHOLD']\n" + ) + spec = importlib.util.spec_from_file_location("aliasing_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with pytest.raises(ValueError, match="dynamic namespace access"): + udf(module.score) + + +@pytest.mark.parametrize( + "access", + [ + "globals()['THRESHOLD']", + "eval('THRESHOLD')", + "(lambda g: g()['THRESHOLD'])(globals)", + "__import__('sys').modules[__name__].THRESHOLD", + "sys.modules[__name__].THRESHOLD", + ], +) +def test_udf_rejects_dynamic_namespace_access(access): + namespace: dict = {} + exec( + f"def score(value: int) -> int:\n return value + {access}\n", + {"THRESHOLD": 5}, + namespace, + ) + with pytest.raises(ValueError, match="dynamic namespace access"): + _package_from_text( + "def score(value: int) -> int:\n" + " import sys\n" + f" return value + {access}\n" + ) + + +def _package_from_text(source: str, module_globals: dict | None = None): + """Load `source` as a real module file so the packager can inspect it.""" + import tempfile + + directory = tempfile.mkdtemp() + path = Path(directory) / "generated_udf_module.py" + path.write_text(source) + spec = importlib.util.spec_from_file_location(f"generated_udf_{id(source)}", path) + module = importlib.util.module_from_spec(spec) + if module_globals: + module.__dict__.update(module_globals) + spec.loader.exec_module(module) + functions = [ + value + for value in vars(module).values() + if callable(value) and getattr(value, "__module__", None) == module.__name__ + ] + return udf(functions[0]) + + +def test_udf_rejects_a_non_standard_builtins_environment(): + def score(value: int) -> int: + return len([1]) + value + + score.__globals__ # noqa: B018 -- real function, real globals + import builtins + + patched = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "len": lambda _: 99}}, + "score", + ) + patched.__annotations__ = score.__annotations__ + assert patched(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(patched) + + class ReportingDict(dict): # reports standard entries, resolves differently + def __missing__(self, key): + return vars(builtins)[key] + + disguised = types.FunctionType( + score.__code__, {"__builtins__": ReportingDict(len=lambda _: 99)}, "score" + ) + disguised.__annotations__ = score.__annotations__ + assert disguised(3) == 102 + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(disguised) + + hooked = types.FunctionType( + score.__code__, + {"__builtins__": {**vars(builtins), "__import__": lambda *a, **k: None}}, + "score", + ) + hooked.__annotations__ = score.__annotations__ + with pytest.raises(ValueError, match="non-standard builtins environment"): + udf(hooked) + + +def test_udf_recursion_versus_a_rebound_module_name(tmp_path): + module_path = tmp_path / "rebound_udfs.py" + module_path.write_text( + "def fact(value: int) -> int:\n" + " return 1 if value <= 1 else value * fact(value - 1)\n" + "\n" + "def score(value: int) -> int:\n" + " return score + value\n" + ) + spec = importlib.util.spec_from_file_location("rebound_udfs", module_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert _run_packaged(udf(module.fact), 5) == 120 + raw = module.score + module.score = 10 + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # A wrapper that merely exposes __wrapped__ is not the function. + module.score = functools.wraps(raw)(lambda value: 41) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw) + # The decorator's own result is; a subclass of it is not. + module.fact = udf(module.fact) + assert _run_packaged(module.fact, 4) == 24 + + class Twisted(UdfDefinition): + def __call__(self, *args, **kwargs): + return 41 + + raw_fact = module.fact._function + module.fact = Twisted( + raw_fact, + name=None, + input_schema=None, + output_schema=None, + pip=(), + env={}, + secrets=(), + python_version=None, + ) + with pytest.raises(ValueError, match="binds that name to another value"): + udf(raw_fact) + + +def test_canonical_arrow_type_rejects_unrepresentable_list_children(): + from lancedb.functions import _canonical_arrow_type + + for outside in [ + pa.list_(pa.float32()), # pyarrow default: nullable child + pa.list_(pa.field("custom", pa.float32(), nullable=False)), + pa.list_(pa.field("item", pa.float32(), nullable=False, metadata={"k": "v"})), + pa.list_(pa.field("item", pa.float32(), nullable=False), 0), + ]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(outside) + assert ( + _canonical_arrow_type( + pa.list_(pa.field("item", pa.float32(), nullable=False), 3) + ) + == "fixed_size_list" + ) + + +def _calls_missing(value: int) -> int: + return missing(value) # noqa: F821 + + +def _shadows_missing_in_a_comprehension(value: int) -> int: + return missing(value) + sum(missing for missing in ()) # noqa: F821 + + +def _shadows_missing_in_a_lambda(value: int) -> int: + return (lambda missing: missing)(value) + missing # noqa: F821 + + +@pytest.mark.parametrize( + "function", + [_calls_missing, _shadows_missing_in_a_comprehension, _shadows_missing_in_a_lambda], +) +def test_udf_rejects_a_truly_unresolved_global(function): + with pytest.raises(ValueError, match=r"unresolved global names: \['missing'\]"): + udf(function) + + +def _arrow_type_from_golden(spec: dict) -> pa.DataType: + kind = spec["type"] + if kind in ("list", "large_list", "fixed_size_list"): + item = _arrow_type_from_golden(spec["fields"][0]["type"]) + field = pa.field("item", item, nullable=False) + if kind == "list": + return pa.list_(field) + if kind == "large_list": + return pa.large_list(field) + return pa.list_(field, spec["length"]) + return { + "null": pa.null(), + "bool": pa.bool_(), + "utf8": pa.string(), + "binary": pa.binary(), + "float16": pa.float16(), + "float32": pa.float32(), + "float64": pa.float64(), + "date32": pa.date32(), + "date64": pa.date64(), + }.get(kind) or getattr(pa, kind)() + + +def test_arrow_type_grammar_matches_the_shared_golden(): + golden = json.loads( + ( + Path(__file__).parents[3] + / "rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json" + ).read_text() + ) + from lancedb.functions import _canonical_arrow_type + + emitted = { + case["arrow_type"]: _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + for case in golden["valid"] + } + assert emitted == { + case["arrow_type"]: case["arrow_type"] for case in golden["valid"] + } + assert not set(emitted) & set(golden["invalid"]) + for case in golden["server_only"]: + with pytest.raises(TypeError, match="unsupported Arrow type"): + _canonical_arrow_type(_arrow_type_from_golden(case["json"])) + + def test_explicit_arrow_schema_is_deterministic(): input_schema = pa.schema([pa.field("value", pa.float32(), nullable=True)]) - output_schema = pa.field("embedding", pa.list_(pa.float32(), 3), nullable=False) + output_schema = pa.field( + "embedding", + pa.list_(pa.field("item", pa.float32(), nullable=False), 3), + nullable=False, + ) @udf(input_schema=input_schema, output_schema=output_schema) def explicit(value): @@ -82,7 +477,7 @@ def test_explicit_arrow_schema_is_deterministic(): signature = explicit.registration_request.signature assert signature.inputs[0].arrow_type == "float32" assert signature.inputs[0].nullable is True - assert signature.output.arrow_type == "fixed_size_list[3]" + assert signature.output.arrow_type == "fixed_size_list" assert signature.output.nullable is False @@ -187,7 +582,7 @@ def _mock_remote_function_catalog(): "job_state": "DONE", "result": state["version"], } - elif self.path == "/v1/functions/get": + elif self.path == "/v1/functions/describe": assert body == { "name": "normalize_score", "version": "fv_exact", diff --git a/python/python/tests/test_fts.py b/python/python/tests/test_fts.py index f791f9886..625198d92 100644 --- a/python/python/tests/test_fts.py +++ b/python/python/tests/test_fts.py @@ -245,6 +245,14 @@ def test_create_inverted_index_rejects_invalid_block_size(table): table.create_index("text", config=FTS(block_size=129)) +def test_create_inverted_index_respects_build_memory_limit(table): + with pytest.raises(ValueError, match="exceeds worker memory limit"): + table.create_index( + "text", + config=FTS(memory_limit=0, num_workers=1), + ) + + def test_custom_stop_words_list(table): table.create_index( "text", diff --git a/python/python/tests/test_index.py b/python/python/tests/test_index.py index 94268a53e..fe6ebe87a 100644 --- a/python/python/tests/test_index.py +++ b/python/python/tests/test_index.py @@ -88,7 +88,7 @@ async def binary_table(db_async): async def test_create_index_async_returns_done_job(some_table: AsyncTable): job = await some_table.create_index_async("id", config=BTree()) assert job.id is None - await job.wait() + assert await job.wait() is None assert len(await some_table.list_indices()) == 1 await job.cancel() diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py new file mode 100644 index 000000000..5fa3aa4fb --- /dev/null +++ b/python/python/tests/test_materialized_views.py @@ -0,0 +1,268 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import lancedb +import pytest +from lancedb.materialized_view import MaterializedViewDefinition + + +STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"} + + +def make_db(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table( + "people", + [ + {"name": "ada", "age": 36}, + {"name": "kid", "age": 7}, + {"name": "grace", "age": 85}, + ], + ) + return db + + +def test_create_refresh_and_query(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view( + "adults", + "people", + select=["name", ("shout", "upper(name)")], + where="age >= 18", + ) + assert view.name == "adults" + assert view.table.count_rows() == 0 + + result = view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 2 + + rows = view.table.search().to_list() + assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] + + +def test_definition_round_trips(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + view = db.open_materialized_view("adults") + assert view.definition == MaterializedViewDefinition( + source_table="people", + projections=[("name", "`name`"), ("age", "`age`")], + filter="age >= 18", + inputs=["age", "name"], + ) + + +def test_incremental_refresh_after_append(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").add([{"name": "alan", "age": 41}]) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + assert view.table.count_rows() == 4 + + assert view.refresh().mode == "no_op" + + +def test_incremental_refresh_after_update(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36, 85] + + +def test_legacy_storage_source_update_rebuilds(tmp_path): + db = lancedb.connect( + tmp_path, + storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"}, + ) + db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) + view = db.create_materialized_view("copy", "people") + view.refresh() + + db.open_table("people").update(where="name = 'kid'", values={"age": 8}) + result = view.refresh() + assert result.mode == "rebuild" + rows = view.table.search().to_list() + assert sorted(row["age"] for row in rows) == [8, 36] + + +def test_list_and_not_a_view(tmp_path): + db = make_db(tmp_path) + db.create_materialized_view("adults", "people", where="age >= 18") + + assert db.list_materialized_views() == ["adults"] + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +def test_invalid_expression_fails_at_create(tmp_path): + db = make_db(tmp_path) + with pytest.raises(Exception, match="missing"): + db.create_materialized_view("bad", "people", select=[("x", "missing + 1")]) + assert "bad" not in db.list_tables().tables + + +@pytest.mark.asyncio +async def test_async_create_refresh_and_open(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + + view = await db.create_materialized_view( + "shouts", "people", select=[("shout", "upper(name)")] + ) + result = await view.refresh() + assert result.mode == "rebuild" + assert result.rows_written == 1 + + reopened = await db.open_materialized_view("shouts") + definition = await reopened.definition() + assert definition.projections == [("shout", "upper(name)")] + assert await db.list_materialized_views() == ["shouts"] + + +@pytest.mark.asyncio +async def test_async_incremental(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("copy", "people") + await view.refresh() + + table = await db.open_table("people") + await table.add([{"name": "alan", "age": 41}]) + result = await view.refresh() + assert result.mode == "incremental" + assert result.rows_written == 1 + + +def test_source_requires_stable_row_ids(tmp_path): + db = lancedb.connect(tmp_path) + db.create_table("plain", [{"x": 1}]) + with pytest.raises(Exception, match="stable row ids"): + db.create_materialized_view("v", "plain") + + +def test_bare_select_names_are_quoted(tmp_path): + db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS) + db.create_table("odd_names", [{"order item": "widget", "select": 2}]) + + view = db.create_materialized_view( + "quoted", "odd_names", select=["order item", "select"] + ) + result = view.refresh() + assert result.rows_written == 1 + rows = view.table.search().to_list() + assert rows[0]["order item"] == "widget" + assert rows[0]["select"] == 2 + + +@pytest.mark.asyncio +async def test_async_remote_is_refused_without_network(): + db = await lancedb.connect_async( + "db://nowhere", api_key="sk_test", region="us-east-1" + ) + with pytest.raises(NotImplementedError, match="local"): + await db.create_materialized_view("v", "src") + with pytest.raises(NotImplementedError, match="local"): + await db.open_materialized_view("v") + with pytest.raises(NotImplementedError, match="local"): + await db.list_materialized_views() + + +def test_scalar_select_is_one_column(tmp_path): + db = make_db(tmp_path) + view = db.create_materialized_view("just_name", "people", select="name") + view.refresh() + rows = view.table.search().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"] + + +@pytest.mark.asyncio +async def test_async_scalar_select_is_one_column(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + view = await db.create_materialized_view("just_name", "people", select="name") + await view.refresh() + rows = await view.table.query().to_list() + assert set(rows[0]) - {"__source_row_id"} == {"name"} + + +def test_limit_above_i64_max_is_refused(tmp_path): + db = make_db(tmp_path) + with pytest.raises(ValueError, match="exceeds the maximum"): + db.create_materialized_view("too_big", "people", limit=2**63) + # The boundary is fine, and zero still means an empty view. + db.create_materialized_view("at_max", "people", limit=2**63 - 1) + empty = db.create_materialized_view("none", "people", limit=0) + empty.refresh() + assert empty.table.count_rows() == 0 + + +def _namespace_db(tmp_path): + return lancedb.connect_namespace( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + + +def test_namespace_connection_materialized_views(tmp_path): + db = _namespace_db(tmp_path) + db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = db.create_materialized_view("adults", "people", where="age >= 18") + view.refresh() + assert view.table.count_rows() == 1 + assert db.list_materialized_views() == ["adults"] + + reopened = db.open_materialized_view("adults") + assert reopened.definition.source_table == "people" + with pytest.raises(ValueError, match="not a materialized view"): + db.open_materialized_view("people") + + +@pytest.mark.asyncio +async def test_async_namespace_connection_materialized_views(tmp_path): + db = lancedb.connect_namespace_async( + "dir", + {"root": str(tmp_path)}, + storage_options=STABLE_ROW_IDS, + ) + await db.create_table( + "people", + [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}], + storage_options=STABLE_ROW_IDS, + ) + + view = await db.create_materialized_view("adults", "people", where="age >= 18") + await view.refresh() + assert await view.table.count_rows() == 1 + assert await db.list_materialized_views() == ["adults"] + + reopened = await db.open_materialized_view("adults") + assert (await reopened.definition()).source_table == "people" + + # The view's table came through the namespace, not straight from the + # inner connection: a bare inner table carries no namespace context, so + # its pushdown routing differs from a table the namespace opened. + through_namespace = await db.open_table("adults") + for handle in (view.table, reopened.table): + assert ( + handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust + ) + assert handle._namespace_path == through_namespace._namespace_path diff --git a/python/python/tests/test_permutation.py b/python/python/tests/test_permutation.py index 135742c84..142fc84f1 100644 --- a/python/python/tests/test_permutation.py +++ b/python/python/tests/test_permutation.py @@ -56,6 +56,31 @@ def test_execute_does_not_reenter_background_loop(tmp_path, monkeypatch): assert permutation_tbl._conn.read_consistency_interval is None +def test_pickled_permutation_reads_pinned_version(tmp_path): + """An unpickled copy must still read the pinned version, which also covers the + version surviving the ``to_arrow()`` round trip in ``__getstate__``.""" + import pickle + + db = connect(tmp_path) + tbl = db.create_table("base", pa.table({"idx": range(20)})) + permutation_tbl = permutation_builder(tbl).execute() + perm = Permutation.from_tables(tbl, permutation_tbl) + + payload = pickle.dumps(perm) + + # Compact so the stored row addresses no longer describe these rows at latest. + tbl.delete("true") + tbl.optimize() + assert tbl.count_rows() == 0 + + # Unpickle after the mutation: __setstate__ reopens at latest, so this only + # passes if the recorded version is applied on reopen. + restored = pickle.loads(payload) + assert len(restored) == 20 + rows = restored.__getitems__(list(range(20))) + assert sorted(row["idx"] for row in rows) == list(range(20)) + + def test_split_random_counts(mem_db): """Test random splitting with absolute counts.""" tbl = mem_db.create_table( diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 13ffc4415..2952f00a4 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -242,8 +242,8 @@ def test_remote_table_branches_sync(): table.branches.delete("exp") -def test_remote_table_branch_merge_defaults_to_execute(): - merge_bodies = [] +def test_remote_table_cherry_pick_defaults_to_execute(): + cherry_pick_bodies = [] diff = { "fromBranch": "exp", "parentVersion": 1, @@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute(): "changedColumns": [], "addedIndexes": [], "removedIndexes": [], - "mergeable": True, - "mergeBlockers": [], + "errors": [], } def handler(request): @@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute(): else: content_len = int(request.headers.get("Content-Length")) request_body = json.loads(request.rfile.read(content_len)) - merge_bodies.append(request_body) + cherry_pick_bodies.append(request_body) dry_run = request_body["dry_run"] status = 200 if dry_run else 409 body = { - "status": "ready" if dry_run else "rejected", + "status": "ready" if dry_run else "failed", "diff": diff, "preview": {"promotedColumns": []}, } @@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute(): with mock_lancedb_connection(handler) as db: branches = db.open_table("test").branches - assert branches.merge("exp")["status"] == "rejected" - assert branches.merge("exp", dry_run=True)["status"] == "ready" + assert branches.cherry_pick("exp")["status"] == "failed" + assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready" - assert merge_bodies == [ + assert cherry_pick_bodies == [ {"from_branch": "exp", "dry_run": False}, {"from_branch": "exp", "dry_run": True}, ] @@ -876,11 +875,85 @@ def test_remote_create_index_async_returns_job(): table = db.create_table("test", [{"id": 1}]) job = table.create_index_async("id", config=BTree()) assert job.id == "job-1" - job.wait(timeout=timedelta(seconds=30)) + assert job.wait(timeout=timedelta(seconds=30)) is None assert len(describe_calls) == 2 job.cancel() +def test_remote_refresh_async_returns_typed_terminal_result(): + terminal_result = { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + + def handler(request): + content_len = int(request.headers.get("Content-Length", 0)) + body = request.rfile.read(content_len) if content_len > 0 else b"" + if request.path == "/v1/table/test/backfill_column": + assert json.loads(body)["column"] == "derived" + request.send_response(202) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"job_id": "refresh-1"}') + elif request.path == "/v1/jobs/describe": + assert json.loads(body)["job_id"] == "refresh-1" + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "job_id": "refresh-1", + "job_type": "function_refresh", + "job_state": "DONE", + "result": terminal_result, + } + ).encode() + ) + elif request.path == "/v1/table/test/create/?mode=create": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + elif request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps( + { + "version": 1, + "schema": { + "fields": [ + { + "name": "id", + "type": {"type": "int64"}, + "nullable": False, + } + ] + }, + } + ).encode() + ) + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.create_table("test", [{"id": 1}]) + job = table.refresh_column_async("derived") + assert job.id == "refresh-1" + result = job.wait(timeout=timedelta(seconds=30)) + + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.model_dump() == terminal_result + assert result.rows_filled == 12 + assert result.version == 8 + + def test_remote_job_wait_raises_on_failure(): from lancedb.exceptions import JobFailedError from lancedb.index import BTree diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index 0f98219bf..56e0eacfd 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection): table = mem_db.create_table("job_test", [{"id": i} for i in range(10)]) job = table.create_index_async("id", config=BTree()) assert job.id is None - job.wait() + assert job.wait() is None assert len(table.list_indices()) == 1 job.cancel() @@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path): job = table.refresh_column_async("doubled") assert job.id is None # in-process jobs have no server id - assert job.wait() is None + result = job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 2 + assert result.rows_failed == 0 + assert result.rows_remaining == 0 + assert result.source_version == 2 + assert result.published_version == 3 assert job.status() == "finished" assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4] + no_op = table.refresh_column_async("doubled").wait() + assert no_op.rows_assigned == 0 + assert no_op.source_version == 3 + assert no_op.published_version is None + # Bad input raises at the call, not through the job. with pytest.raises(Exception, match="not a computed column"): table.refresh_column_async("x") @@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path): await table.add_columns(computed={"tripled": "x * 3"}) job = await table.refresh_column_async("tripled") - assert await job.wait() is None + result = await job.wait() + assert isinstance(result, lancedb.RefreshColumnResult) + assert result.rows_assigned == 1 + assert result.source_version == 2 + assert result.published_version == 3 assert await job.status() == "finished" assert (await table.to_arrow())["tripled"].to_pylist() == [9] diff --git a/python/src/connection.rs b/python/src/connection.rs index 87870b800..902489f4f 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -333,6 +333,40 @@ impl Connection { }) } + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None))] + pub fn create_materialized_view( + self_: PyRef<'_, Self>, + name: String, + source: String, + projections: Option>, + filter: Option, + limit: Option, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let mut builder = inner.create_materialized_view(name, source); + if let Some(projections) = projections { + builder = builder.select(projections); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + builder = builder.limit(limit); + } + let view = builder.execute().await.infer_error()?; + Ok(Table::new(view.table().clone())) + }) + } + + pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let views = inner.list_materialized_views().await.infer_error()?; + Ok(views.into_iter().map(|view| view.name).collect::>()) + }) + } + #[pyo3(signature = (name, namespace_path=None))] pub fn drop_table( self_: PyRef<'_, Self>, @@ -575,7 +609,7 @@ impl Connection { .create_function_async(request) .await .infer_error() - .map(crate::job::FunctionJob::new) + .map(crate::job::Job::new_typed) }) } diff --git a/python/src/index.rs b/python/src/index.rs index dd362373e..a5ca63c68 100644 --- a/python/src/index.rs +++ b/python/src/index.rs @@ -42,7 +42,7 @@ pub fn extract_index_params(source: &Option>) -> PyResult Ok(LanceDbIndex::Fm(FmIndexBuilder::default())), "FTS" => { let params = source.extract::()?; - let inner_opts = FtsIndexBuilder::default() + let mut inner_opts = FtsIndexBuilder::default() .base_tokenizer(params.base_tokenizer) .language(¶ms.language) .map_err(|_| { @@ -61,6 +61,12 @@ pub fn extract_index_params(source: &Option>) -> PyResult, + num_workers: Option, } #[derive(FromPyObject)] @@ -444,3 +452,51 @@ impl IndexConfig { } } } + +#[cfg(test)] +mod tests { + use super::*; + use pyo3::types::{PyDict, PyDictMethods}; + use serde_json::json; + + #[test] + fn fts_build_controls_are_forwarded() { + Python::initialize(); + Python::attach(|py| { + let locals = PyDict::new(py); + py.run( + c"class FTS: + with_position = True + base_tokenizer = 'simple' + language = 'English' + max_token_length = None + lower_case = True + stem = False + remove_stop_words = False + custom_stop_words = None + ascii_folding = False + ngram_min_length = 3 + ngram_max_length = 3 + prefix_only = False + block_size = 128 + memory_limit = 2048 + num_workers = 7 + +config = FTS()", + None, + Some(&locals), + ) + .unwrap(); + + let config = locals.get_item("config").unwrap().unwrap(); + let index = extract_index_params(&Some(config)).unwrap(); + let LanceDbIndex::FTS(params) = index else { + panic!("expected FTS index parameters"); + }; + let training_json = params.to_training_json().unwrap(); + + assert_eq!(training_json.get("memory_limit"), Some(&json!(2048))); + assert_eq!(training_json.get("num_workers"), Some(&json!(7))); + }); + } +} diff --git a/python/src/job.rs b/python/src/job.rs index a08b958a6..688cba7f9 100644 --- a/python/src/job.rs +++ b/python/src/job.rs @@ -5,72 +5,33 @@ use std::sync::Arc; use crate::runtime::future_into_py; use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods}; +use serde::Serialize; use crate::error::PythonErrorExt; #[pyclass] pub struct Job { - inner: Arc, -} - -/// Python bridge for a typed remote Function registration job. -/// -/// The public Python layer decodes the canonical JSON returned by `wait` -/// into its immutable `FunctionVersion` model. -#[pyclass] -pub struct FunctionJob { - inner: Arc>, -} - -impl FunctionJob { - pub(crate) fn new(inner: lancedb::Job) -> Self { - Self { - inner: Arc::new(inner), - } - } + inner: Arc, String>>>, } impl Job { pub(crate) fn new(inner: lancedb::Job) -> Self { Self { - inner: Arc::new(inner), + inner: Arc::new(inner.map(|()| Ok(None))), } } -} -#[pymethods] -impl FunctionJob { - #[getter] - pub fn id(&self) -> Option { - self.inner.id().map(str::to_string) - } - - pub fn status(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py( - self_.py(), - async move { inner.status().await.infer_error() }, - ) - } - - pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner - .wait() - .await - .infer_error()? - .to_canonical_json() - .infer_error() - }) - } - - pub fn cancel(self_: PyRef<'_, Self>) -> PyResult> { - let inner = self_.inner.clone(); - future_into_py(self_.py(), async move { - inner.cancel().await.infer_error()?; - Ok(()) - }) + pub(crate) fn new_typed(inner: lancedb::Job) -> Self + where + T: Clone + Serialize + Send + Sync + 'static, + { + Self { + inner: Arc::new(inner.map(|result| { + serde_json::to_string(&result) + .map(Some) + .map_err(|error| format!("failed to serialize typed job result: {error}")) + })), + } } } @@ -92,8 +53,10 @@ impl Job { pub fn wait(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { - inner.wait().await.infer_error()?; - Ok(None::<()>) + let result = inner.wait().await.infer_error()?; + result + .map_err(|message| lancedb::Error::Runtime { message }) + .infer_error() }) } diff --git a/python/src/lib.rs b/python/src/lib.rs index 756b3557f..8d3eab787 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult, - UpdateResult, + LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, + Table, UpdateFieldMetadataResult, UpdateResult, }; pub mod arrow; @@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::
()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -60,6 +59,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index cb4752cce..b225b191f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -441,6 +441,41 @@ impl From for RefreshColumnResult { } } +#[pyclass(get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct RefreshMaterializedViewResult { + pub mode: String, + pub rows_written: u64, + pub source_version: u64, + pub version: u64, +} + +#[pymethods] +impl RefreshMaterializedViewResult { + pub fn __repr__(&self) -> String { + format!( + "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})", + self.mode, self.rows_written, self.source_version, self.version + ) + } +} + +impl From for RefreshMaterializedViewResult { + fn from(result: lancedb::RefreshMaterializedViewResult) -> Self { + let mode = match result.mode { + lancedb::RefreshMode::Rebuild => "rebuild", + lancedb::RefreshMode::Incremental => "incremental", + lancedb::RefreshMode::NoOp => "no_op", + }; + Self { + mode: mode.to_string(), + rows_written: result.rows_written, + source_version: result.source_version, + version: result.version, + } + } +} + #[pymethods] impl AddColumnsResult { pub fn __repr__(&self) -> String { @@ -1584,7 +1619,27 @@ impl Table { let inner = self_.inner_ref()?.clone(); future_into_py(self_.py(), async move { let job = inner.refresh_column_async(column).await.infer_error()?; - Ok(crate::job::Job::new(job)) + Ok(crate::job::Job::new_typed(job)) + }) + } + + #[pyo3(signature = (full=false, source_version=None))] + pub fn refresh_materialized_view( + self_: PyRef<'_, Self>, + full: bool, + source_version: Option, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let view = lancedb::MaterializedView::from_table(inner) + .await + .infer_error()?; + let mut builder = view.refresh().full(full); + if let Some(version) = source_version { + builder = builder.source_version(version); + } + let result = builder.execute().await.infer_error()?; + Ok(RefreshMaterializedViewResult::from(result)) }) } @@ -1885,7 +1940,7 @@ impl Branches { } #[pyo3(signature = (from_branch, dry_run=false))] - pub fn merge( + pub fn cherry_pick( self_: PyRef<'_, Self>, from_branch: String, dry_run: bool, @@ -1893,7 +1948,7 @@ impl Branches { let inner = self_.inner.clone(); future_into_py(self_.py(), async move { let result = inner - .merge_branch(&from_branch, dry_run) + .cherry_pick(&from_branch, dry_run) .await .infer_error()?; Python::attach(|py| struct_to_wire_py(py, &result)) diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index f785e6743..7290694f1 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.3" +version = "0.38.0-beta.8" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 8935855a8..5f66d9dee 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -41,7 +41,7 @@ use lance_io::object_store::{StorageOptionsAccessor, StorageOptionsProvider}; mod create_table; -fn merge_storage_options( +pub(crate) fn merge_storage_options( store_params: &mut ObjectStoreParams, pairs: impl IntoIterator, ) { @@ -1679,6 +1679,50 @@ mod tests { assert_eq!(tables, names[..7]); } + #[tokio::test] + async fn test_list_tables_walks_page_boundaries() { + let tc = new_test_connection().await.unwrap(); + if tc.is_remote { + // What resumes a page is the server's to decide, and asserting it here would be + // asserting the server's contract rather than this one. + return; + } + let db = tc.connection; + let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Int32, false)])); + let mut names = Vec::with_capacity(5); + for _ in 0..5 { + let name = uuid::Uuid::new_v4().to_string(); + names.push(name.clone()); + db.create_empty_table(name, schema.clone()) + .execute() + .await + .unwrap(); + } + names.sort(); + + // Walking in pages has to reach every table exactly once, with nothing lost at a + // page boundary. + let mut seen = Vec::with_capacity(names.len()); + let mut page_token = None; + loop { + let page = db + .list_tables(ListTablesRequest { + id: Some(Vec::new()), + limit: Some(2), + page_token, + ..Default::default() + }) + .await + .unwrap(); + seen.extend(page.tables); + page_token = page.page_token.filter(|token| !token.is_empty()); + if page_token.is_none() { + break; + } + } + assert_eq!(seen, names); + } + #[tokio::test] async fn test_open_table() { let tc = new_test_connection().await.unwrap(); diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 574cc1b1b..17dc82756 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -974,17 +974,15 @@ impl Database for ListingDatabase { f.drain(0..index); } - // Determine if there's a next page - let next_page_token = if let Some(limit) = request.limit { - if f.len() > limit as usize { - let token = f[limit as usize].clone(); + // Determine if there's a next page. The token is the last name of this page, + // not the first of the next one: the next page resumes strictly after the + // token, so naming the next page's first entry would skip it. + let next_page_token = match request.limit { + Some(limit) if f.len() > limit as usize => { f.truncate(limit as usize); - Some(token) - } else { - None + f.last().cloned() } - } else { - None + _ => None, }; Ok(ListTablesResponse { @@ -2747,6 +2745,21 @@ mod tests { } } + /// Regression test for https://github.com/lancedb/lancedb/issues/2283. + /// + /// Object-store URIs must use `/` on every platform. In particular, joining + /// with `std::path::Path` used to insert a `\\` into Azure blob keys on + /// Windows. + #[tokio::test] + async fn test_table_uri_uses_forward_slashes_for_azure() { + let (_tempdir, mut db) = setup_database().await; + db.uri = "az://test/db/test".to_string(); + + let uri = db.table_uri("test").unwrap(); + + assert_eq!(uri, "az://test/db/test/test.lance"); + } + /// Regression: connecting via a URL-style URI (which goes through /// `url::Url::parse` and the `query_pairs_mut()` path) must not /// append a trailing `?` to per-table URIs when the input URI has diff --git a/rust/lancedb/src/dataloader/permutation/builder.rs b/rust/lancedb/src/dataloader/permutation/builder.rs index a3700d0ef..ba0d0e180 100644 --- a/rust/lancedb/src/dataloader/permutation/builder.rs +++ b/rust/lancedb/src/dataloader/permutation/builder.rs @@ -27,6 +27,12 @@ pub const SRC_ROW_ID_COL: &str = "row_id"; pub const SPLIT_NAMES_CONFIG_KEY: &str = "split_names"; +/// Base table version the permutation was built against. +pub const BASE_VERSION_CONFIG_KEY: &str = "base_version"; + +/// Base table branch the permutation was built against. Absent means main. +pub const BASE_BRANCH_CONFIG_KEY: &str = "base_branch"; + pub const DEFAULT_MEMORY_LIMIT: usize = 100 * 1024 * 1024; /// Where to store the permutation table @@ -214,21 +220,11 @@ impl PermutationBuilder { Ok(Box::pin(SimpleRecordBatchStream { schema, stream })) } - fn add_split_names( + fn add_config_metadata( data: SendableRecordBatchStream, - split_names: &[String], + metadata: HashMap, ) -> Result { - let schema = data - .schema() - .as_ref() - .clone() - .with_metadata(HashMap::from([( - SPLIT_NAMES_CONFIG_KEY.to_string(), - serde_json::to_string(split_names).map_err(|e| Error::Other { - message: format!("Failed to serialize split names: {}", e), - source: Some(e.into()), - })?, - )])); + let schema = data.schema().as_ref().clone().with_metadata(metadata); let schema = Arc::new(schema); let schema_clone = schema.clone(); let stream = data.map_ok(move |batch| batch.with_schema(schema.clone()).unwrap()); @@ -239,7 +235,20 @@ impl PermutationBuilder { } /// Builds the permutation table and stores it in the given database. - pub async fn build(self) -> Result
{ + pub async fn build(mut self) -> Result
{ + // Remote tables resolve latest independently for each request. Use a + // separate pinned handle so count, projection, and scan all refer to one + // snapshot without changing the caller's table checkout state. Native + // tables return `None` here and retain their existing behavior. + if let Some(snapshot) = self + .base_table + .base_table() + .snapshot_at_current_version() + .await? + { + self.base_table = Table::from(snapshot); + } + // Unflushed rows have no row id, so a permutation cannot address them. match self.base_table.base_table().get_lsm_write_spec().await { Ok(Some(_)) => { @@ -256,9 +265,14 @@ impl PermutationBuilder { Err(err) => return Err(err), } + // The handle above is already pinned to one version. Record which one, so a + // reader -- in a DataLoader worker, against a table that has since moved -- + // resolves these row addresses against the same snapshot. + let base_version = self.base_table.version().await?; + let base_branch = self.base_table.current_branch(); + // First pass, apply filter and load row ids. `Shuffler` permutes positions, so // every rank must scan the rows in the same order to build the same permutation. - // TODO: pin the version resolved here; remote does not implement Lazy. let mut rows = self.base_table.query().select(Select::columns(&[ROW_ID])); if let Some(filter) = &self.config.filter { @@ -318,11 +332,24 @@ impl PermutationBuilder { // Rename _rowid to row_id let renamed = rename_column(sorted, ROW_ID, SRC_ROW_ID_COL)?; - let streaming_data = if let Some(split_names) = &self.config.split_names { - Self::add_split_names(renamed, split_names)? - } else { - renamed - }; + let mut metadata = HashMap::from([( + BASE_VERSION_CONFIG_KEY.to_string(), + base_version.to_string(), + )]); + // Version numbers are per-branch, so the branch is part of the coordinate. + if let Some(branch) = &base_branch { + metadata.insert(BASE_BRANCH_CONFIG_KEY.to_string(), branch.clone()); + } + if let Some(split_names) = &self.config.split_names { + metadata.insert( + SPLIT_NAMES_CONFIG_KEY.to_string(), + serde_json::to_string(split_names).map_err(|e| Error::Other { + message: format!("Failed to serialize split names: {}", e), + source: Some(e.into()), + })?, + ); + } + let streaming_data = Self::add_config_metadata(renamed, metadata)?; let (name, database) = match &self.config.destination { PermutationDestination::Permanent(database, table_name) => { @@ -409,6 +436,253 @@ mod tests { assert!(table.base_table().scan_order_is_deterministic()); } + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_permutation_builder_pins_snapshot() { + use std::sync::{ + Mutex, + atomic::{AtomicU64, Ordering}, + }; + + use arrow_array::{RecordBatch, UInt64Array}; + use arrow_schema::{DataType, Field, Schema}; + + let row_ids = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + ROW_ID, + DataType::UInt64, + false, + )])), + vec![Arc::new(UInt64Array::from(vec![100]))], + ) + .unwrap(); + let mut query_body = Vec::new(); + { + let mut writer = + arrow_ipc::writer::FileWriter::try_new(&mut query_body, &row_ids.schema()).unwrap(); + writer.write(&row_ids).unwrap(); + writer.finish().unwrap(); + } + + let latest = Arc::new(AtomicU64::new(7)); + let expected_snapshot = Arc::new(AtomicU64::new(7)); + let planning_versions = Arc::new(Mutex::new(Vec::new())); + let latest_ref = latest.clone(); + let expected_snapshot_ref = expected_snapshot.clone(); + let planning_versions_ref = planning_versions.clone(); + let table = Table::new_with_handler("remote_base", move |request| { + let path = request.url().path(); + let body = request + .body() + .and_then(|body| body.as_bytes()) + .map(|body| serde_json::from_slice::(body).unwrap()); + + match path { + "/v1/table/remote_base/describe/" => { + let requested = body.as_ref().and_then(|body| body["version"].as_u64()); + let version = requested.unwrap_or_else(|| latest_ref.load(Ordering::SeqCst)); + http::Response::builder() + .status(200) + .body( + format!(r#"{{"version":{version},"schema":{{"fields":[]}}}}"#) + .into_bytes(), + ) + .unwrap() + } + "/v1/table/remote_base/get_lsm_write_spec/" => http::Response::builder() + .status(200) + .body(br#"{"lsm_write_spec":null}"#.to_vec()) + .unwrap(), + "/v1/table/remote_base/count_rows/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["predicate"], "value > 0"); + planning_versions_ref.lock().unwrap().push(version); + + // Simulate a concurrent append after count_rows. An unpinned + // scan would now resolve version 8 and include different rows. + latest_ref.store(8, Ordering::SeqCst); + http::Response::builder() + .status(200) + .body(b"1".to_vec()) + .unwrap() + } + "/v1/table/remote_base/query/" => { + let body = body.unwrap(); + let version = body["version"].as_u64().unwrap(); + assert_eq!(version, expected_snapshot_ref.load(Ordering::SeqCst)); + assert_eq!(body["filter"], "value > 0"); + assert_eq!(body["columns"], serde_json::json!([ROW_ID])); + planning_versions_ref.lock().unwrap().push(version); + http::Response::builder() + .status(200) + .header("content-type", "application/vnd.apache.arrow.file") + .body(query_body.clone()) + .unwrap() + } + _ => panic!("unexpected request: {path}"), + } + }); + + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + + // Building uses a separate handle and must not pin the caller's table. + assert_eq!(table.version().await.unwrap(), 8); + + // An explicit checkout is copied as-is and remains checked out afterward. + expected_snapshot.store(6, Ordering::SeqCst); + table.checkout(6).await.unwrap(); + let permutation = PermutationBuilder::new(table.clone()) + .with_filter("value > 0".to_string()) + .build() + .await + .unwrap(); + assert_eq!(permutation.count_rows(None).await.unwrap(), 1); + assert_eq!(table.version().await.unwrap(), 6); + assert_eq!(*planning_versions.lock().unwrap(), vec![7, 7, 6, 6]); + } + + #[tokio::test] + async fn test_permutation_records_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(2)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let build_version = data_table.version().await.unwrap(); + let permutation_table = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + let recorded = permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .expect("permutation should record the base version") + .parse::() + .unwrap(); + assert_eq!(recorded, build_version); + + // Advancing the base table must not move the recorded version. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert!(data_table.version().await.unwrap() > recorded); + assert_eq!( + permutation_table + .schema() + .await + .unwrap() + .metadata + .get(BASE_VERSION_CONFIG_KEY) + .unwrap() + .parse::() + .unwrap(), + recorded, + ); + } + + /// Version numbers are per-branch, so a permutation built on a branch must record + /// it -- a worker reopens by name and lands on main at the same number. + #[tokio::test] + async fn test_permutation_records_base_branch() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(10), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + let branch = data_table + .create_branch("exp", lance::dataset::refs::Ref::from(("main", 1))) + .await + .unwrap(); + let permutation_table = PermutationBuilder::new(branch.clone()) + .build() + .await + .unwrap(); + + let metadata = permutation_table.schema().await.unwrap().metadata.clone(); + assert_eq!( + metadata.get(BASE_BRANCH_CONFIG_KEY).map(String::as_str), + Some("exp") + ); + + // Main records nothing, so an absent key keeps meaning main. + let main_permutation = PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + assert!( + !main_permutation + .schema() + .await + .unwrap() + .metadata + .contains_key(BASE_BRANCH_CONFIG_KEY) + ); + } + + #[tokio::test] + async fn test_build_does_not_pin_the_callers_table() { + let temp_dir = tempfile::tempdir().unwrap(); + + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let initial_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(100), BatchCount::from(1)); + let data_table = db + .create_table("base_tbl", initial_data) + .execute() + .await + .unwrap(); + + PermutationBuilder::new(data_table.clone()) + .build() + .await + .unwrap(); + + // The builder pins its own handle; the caller's must still track latest. + let more_data = lance_datagen::gen_batch() + .col("col_a", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(50), BatchCount::from(1)); + data_table.add(more_data).execute().await.unwrap(); + assert_eq!(data_table.count_rows(None).await.unwrap(), 150); + } + #[tokio::test] async fn test_permutation_builder() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/rust/lancedb/src/dataloader/permutation/reader.rs b/rust/lancedb/src/dataloader/permutation/reader.rs index 6da92e986..9757dc552 100644 --- a/rust/lancedb/src/dataloader/permutation/reader.rs +++ b/rust/lancedb/src/dataloader/permutation/reader.rs @@ -8,7 +8,9 @@ //! the rows from a source table that correspond to row IDs stored in a separate table. use crate::arrow::{SendableRecordBatchStream, SimpleRecordBatchStream}; -use crate::dataloader::permutation::builder::SRC_ROW_ID_COL; +use crate::dataloader::permutation::builder::{ + BASE_BRANCH_CONFIG_KEY, BASE_VERSION_CONFIG_KEY, SRC_ROW_ID_COL, +}; use crate::dataloader::permutation::split::SPLIT_ID_COLUMN; use crate::error::Error; use crate::query::{ @@ -23,6 +25,7 @@ use arrow_array::{RecordBatch, UInt64Array}; use arrow_schema::SchemaRef; use datafusion_expr::{Expr, col, lit}; use futures::{StreamExt, TryStreamExt}; +use lance::dataset::refs::MAIN_BRANCH; use lance::dataset::scanner::DatasetRecordBatchStream; use lance::io::RecordBatchStream; use lance_arrow::RecordBatchExt; @@ -69,6 +72,10 @@ impl PermutationReader { permutation_table: Option>, split: u64, ) -> Result { + let base_table = match &permutation_table { + Some(permutation_table) => Self::pin_base_table(base_table, permutation_table).await?, + None => base_table, + }; let mut slf = Self { base_table, permutation_table, @@ -89,6 +96,34 @@ impl PermutationReader { Ok(slf) } + /// Pins the base table to the version the permutation was built against. + /// Permutations written before that was recorded carry no key and stay unpinned. + async fn pin_base_table( + base_table: Arc, + permutation_table: &Arc, + ) -> Result> { + let schema = permutation_table.schema().await?; + let Some(raw) = schema.metadata.get(BASE_VERSION_CONFIG_KEY) else { + return Ok(base_table); + }; + let version = raw.parse::().map_err(|e| Error::InvalidInput { + message: format!( + "Permutation table has an unreadable {} of {:?}: {}", + BASE_VERSION_CONFIG_KEY, raw, e + ), + })?; + // The recorded branch, not the handle's: a worker reopens by name and lands + // on main, and version numbers are per-branch. + let branch = schema + .metadata + .get(BASE_BRANCH_CONFIG_KEY) + .map(String::as_str) + .unwrap_or(MAIN_BRANCH); + base_table + .checkout_branch_version(branch, Some(version)) + .await + } + pub async fn try_from_tables( base_table: Arc, permutation_table: Arc, @@ -511,9 +546,13 @@ mod tests { use lance_datagen::{BatchCount, RowCount}; use rand::seq::SliceRandom; + // Aliased: `test_utils::datagen` exports a trait of the same name. + use crate::arrow::LanceDbDatagenExt as _; use crate::{ Table, arrow::SendableRecordBatchStream, + connect, + dataloader::permutation::builder::PermutationBuilder, query::{ExecutableQuery, QueryBase}, test_utils::datagen::{LanceDbDatagenExt, virtual_table}, }; @@ -545,6 +584,58 @@ mod tests { .await } + /// Compaction moves row addresses, so the reader must read the pinned version. + #[tokio::test] + async fn test_reader_pins_base_version() { + let temp_dir = tempfile::tempdir().unwrap(); + let db = connect(temp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + + let data = lance_datagen::gen_batch() + .col("idx", lance_datagen::array::step::()) + .into_ldb_stream(RowCount::from(20), BatchCount::from(1)); + let base_table = db.create_table("base_tbl", data).execute().await.unwrap(); + + let permutation_table = PermutationBuilder::new(base_table.clone()) + .build() + .await + .unwrap(); + + base_table.delete("true").await.unwrap(); + base_table + .optimize(crate::table::OptimizeAction::All) + .await + .unwrap(); + assert_eq!(base_table.count_rows(None).await.unwrap(), 0); + + let reader = PermutationReader::try_from_tables( + base_table.base_table().clone(), + permutation_table.base_table().clone(), + 0, + ) + .await + .unwrap(); + + let values = collect_from_stream::( + reader + .read( + Select::Columns(vec!["idx".to_string()]), + QueryExecutionOptions::default(), + ) + .await + .unwrap(), + "idx", + ) + .await; + assert_eq!( + values.len(), + 20, + "reader should still see the pinned version" + ); + } + #[tokio::test] async fn test_permutation_reader() { let base_table = lance_datagen::gen_batch() diff --git a/rust/lancedb/src/error.rs b/rust/lancedb/src/error.rs index 6bd1ffa2b..be4641388 100644 --- a/rust/lancedb/src/error.rs +++ b/rust/lancedb/src/error.rs @@ -77,6 +77,8 @@ pub enum Error { ColumnAlreadyExists { name: String }, #[snafu(display("Column '{name}' is not a computed column"))] NotAComputedColumn { name: String }, + #[snafu(display("Table '{name}' is not a materialized view"))] + NotAMaterializedView { name: String }, #[snafu(display("Invalid expression for column '{column}': {message}"))] InvalidExpression { column: String, message: String }, diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 835fca9a2..f02158871 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Canonical values exchanged with the Enterprise Function service. +//! Canonical Function values exchanged with the Enterprise service, plus the +//! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, //! environment bake, secret resolution, and execution are owned by Sophon. @@ -580,13 +581,22 @@ impl FunctionBinding { impl_json!(FunctionBinding); -/// Stable terminal result of a remote Function-column refresh Job. +/// Stable terminal result of an expression-backed or Function-backed column +/// refresh [`crate::Job`]. +/// +/// Local refresh jobs produce this value in process. LanceDB Cloud and +/// Enterprise decode the same value from the durable job's terminal payload. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RefreshColumnResult { + /// Rows assigned a value by this refresh. pub rows_assigned: u64, + /// Rows whose computation failed. pub rows_failed: u64, + /// Rows that still need a value when the job completes. pub rows_remaining: u64, + /// Exact table version the refresh read. pub source_version: u64, + /// Table version made visible by the refresh, when one was published. #[serde(default, skip_serializing_if = "Option::is_none")] pub published_version: Option, } diff --git a/rust/lancedb/src/job.rs b/rust/lancedb/src/job.rs index 1a76c7683..94d1ba2b6 100644 --- a/rust/lancedb/src/job.rs +++ b/rust/lancedb/src/job.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use async_trait::async_trait; -use serde::de::DeserializeOwned; +use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; use tokio::sync::watch; use tokio::task::{AbortHandle, JoinHandle}; @@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync { } /// A backend-neutral successful terminal result. -/// -/// Local operations do not carry a value. Remote operations may carry JSON -/// that the public [`Job`] decodes according to its result type. +#[derive(Clone)] pub(crate) struct TerminalResult { - #[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1. value: Option, - #[allow(dead_code)] // Preserved so typed decode errors retain request correlation. request_id: Option, } impl TerminalResult { - pub(crate) fn local() -> Self { + fn local(value: Value) -> Self { Self { - value: None, + value: Some(value), request_id: None, } } @@ -51,23 +47,31 @@ impl TerminalResult { } } - #[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1. fn decode(self) -> Result { - let request_id = self.request_id.unwrap_or_default(); - let value = self.value.ok_or_else(|| Error::Http { - source: "successful typed job response did not contain a result".into(), - request_id: request_id.clone(), - status_code: None, + let value = self.value.ok_or_else(|| match &self.request_id { + Some(request_id) => Error::Http { + source: "successful typed job response did not contain a result".into(), + request_id: request_id.clone(), + status_code: None, + }, + None => Error::Runtime { + message: "successful typed job did not contain a result".to_string(), + }, })?; - serde_json::from_value(value).map_err(|error| Error::Http { - source: format!("failed to parse typed job result: {error}").into(), - request_id, - status_code: None, + serde_json::from_value(value).map_err(|error| match self.request_id { + Some(request_id) => Error::Http { + source: format!("failed to parse typed job result: {error}").into(), + request_id, + status_code: None, + }, + None => Error::Runtime { + message: format!("failed to parse typed job result: {error}"), + }, }) } } -type ResultDecoder = fn(TerminalResult) -> Result; +type ResultDecoder = Arc Result + Send + Sync>; enum JobInner { Handle { @@ -79,7 +83,9 @@ enum JobInner { /// A handle to an operation that may still be running. /// -/// The operation may already be complete when the handle is created. +/// The operation may already be complete when the handle is created. `T` is +/// the endpoint's successful terminal result; unit-result operations use the +/// default `Job<()>`. pub struct Job where T: Clone + Send + Sync + 'static, @@ -111,15 +117,10 @@ impl Job<()> { Self { inner: JobInner::Handle { handle, - decode: |_| Ok(()), + decode: Arc::new(|_| Ok(())), }, } } - - /// A unit-result job running as a task in this process. - pub(crate) fn spawned(task: JoinHandle>) -> Self { - Self::new(Box::new(SpawnedJob::new(task))) - } } impl Job @@ -131,12 +132,22 @@ where Self { inner: JobInner::Handle { handle, - decode: TerminalResult::decode::, + decode: Arc::new(TerminalResult::decode::), }, } } } +impl Job +where + T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static, +{ + /// A typed job running as a task in this process. + pub(crate) fn spawned(task: JoinHandle>) -> Self { + Self::new_typed(Box::new(SpawnedJob::new(task))) + } +} + impl Job where T: Clone + Send + Sync + 'static, @@ -169,11 +180,13 @@ where /// Waits until the operation reaches a terminal state. /// + /// Returns the endpoint's typed result. Unit-result jobs return `()`. + /// /// Returns [`crate::Error::JobFailed`] if the operation failed and /// [`crate::Error::JobCancelled`] if it was cancelled. pub async fn wait(&self) -> Result { match &self.inner { - JobInner::Handle { handle, decode } => decode(handle.wait().await?), + JobInner::Handle { handle, decode } => (decode)(handle.wait().await?), JobInner::Completed(result) => Ok(result.clone()), } } @@ -187,21 +200,53 @@ where JobInner::Completed(_) => Ok(()), } } + + /// Maps a successful terminal result without changing the job lifecycle. + /// The mapping may run once for each call to [`Job::wait`], so it should + /// be deterministic and free of externally visible side effects. + /// + /// ``` + /// use lancedb::{Job, function::RefreshColumnResult}; + /// + /// # async fn rows_assigned( + /// # job: Job, + /// # ) -> lancedb::Result { + /// let job = job.map(|result| result.rows_assigned); + /// job.wait().await + /// # } + /// ``` + pub fn map(self, map: F) -> Job + where + U: Clone + Send + Sync + 'static, + F: Fn(T) -> U + Send + Sync + 'static, + { + match self.inner { + JobInner::Handle { handle, decode } => Job { + inner: JobInner::Handle { + handle, + decode: Arc::new(move |result| Ok(map((decode)(result)?))), + }, + }, + JobInner::Completed(result) => Job { + inner: JobInner::Completed(map(result)), + }, + } + } } /// How an in-process operation ended. Cloneable so every waiter can be given /// the outcome; [`Error`] is not, so failures share one behind an [`Arc`]. #[derive(Clone)] enum Outcome { - Succeeded, + Succeeded(TerminalResult), Failed(Arc), Cancelled, } impl Outcome { - fn into_result(self) -> Result<()> { + fn into_result(self) -> Result { match self { - Self::Succeeded => Ok(()), + Self::Succeeded(result) => Ok(result), Self::Failed(source) => Err(Error::JobFailed { job_id: None, failure: JobFailure::from_source(source), @@ -220,12 +265,20 @@ struct SpawnedJob { } impl SpawnedJob { - fn new(task: JoinHandle>) -> Self { + fn new(task: JoinHandle>) -> Self + where + T: Serialize + Send + 'static, + { let abort = task.abort_handle(); let (tx, outcome) = watch::channel(None); tokio::spawn(async move { let outcome = match task.await { - Ok(Ok(())) => Outcome::Succeeded, + Ok(Ok(result)) => match serde_json::to_value(result) { + Ok(value) => Outcome::Succeeded(TerminalResult::local(value)), + Err(err) => Outcome::Failed(Arc::new(Error::Runtime { + message: format!("failed to serialize job result: {err}"), + })), + }, Ok(Err(err)) => Outcome::Failed(Arc::new(err)), Err(err) if err.is_cancelled() => Outcome::Cancelled, Err(err) => Outcome::Failed(Arc::new(Error::Runtime { @@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob { async fn status(&self) -> Result { let label = match &*self.outcome.borrow() { None => "running", - Some(Outcome::Succeeded) => "finished", + Some(Outcome::Succeeded(_)) => "finished", Some(Outcome::Failed(_)) => "failed", Some(Outcome::Cancelled) => "cancelled", }; @@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob { .wait_for(|outcome| outcome.is_some()) .await .map_err(|_| Error::Runtime { - message: "index job outcome was dropped before it completed".to_string(), + message: "job outcome was dropped before it completed".to_string(), })? .clone() .expect("wait_for returns once an outcome is set"); - settled.into_result()?; - Ok(TerminalResult::local()) + settled.into_result() } async fn cancel(&self) -> Result<()> { @@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob { Ok(()) } } + +#[cfg(test)] +mod tests { + use std::future::pending; + + use super::*; + + #[tokio::test] + async fn mapped_spawned_job_reuses_outcome() { + let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1); + + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.wait().await.unwrap(), 42); + assert_eq!(job.status().await.unwrap(), "finished"); + } + + #[tokio::test] + async fn mapped_spawned_job_preserves_cancellation() { + let job = Job::spawned(tokio::spawn(async { pending::>().await })) + .map(|value| value.to_string()); + + job.cancel().await.unwrap(); + assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. }))); + assert_eq!(job.status().await.unwrap(), "cancelled"); + } +} diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 291dcaf65..9c3c199ff 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -186,6 +186,7 @@ pub mod index; pub mod io; pub mod ipc; pub mod job; +pub mod materialized_view; #[cfg(feature = "metrics-otel")] pub mod metrics_otel; #[cfg(feature = "polars")] @@ -210,6 +211,9 @@ pub use function::FunctionVersion; pub use job::Job; use lance_index::vector::ApproxMode as LanceApproxMode; use lance_linalg::distance::DistanceType as LanceDistanceType; +pub use materialized_view::{ + MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode, +}; /// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable /// the `metrics` feature to publish LanceDB's internal metrics; install any /// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs new file mode 100644 index 000000000..b28d52931 --- /dev/null +++ b/rust/lancedb/src/materialized_view.rs @@ -0,0 +1,2106 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Materialized views. +//! +//! A materialized view is a table whose contents are defined by a query over +//! one source table and maintained by refresh rather than by writes. Creation +//! commits an empty table carrying the kind-tagged definition in schema +//! metadata; a kind added later reads back as unrefreshable, not as a plain +//! table. Queries, indexes and search work on the view unchanged. + +pub mod refresh; + +#[cfg(test)] +mod differential; + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow_schema::{DataType, Field as ArrowField, FieldRef, Schema as ArrowSchema, SchemaRef}; +use datafusion_common::ScalarValue; +use lance_core::ROW_ID; +use lance_datafusion::planner::Planner; +use serde::{Deserialize, Serialize}; + +use crate::connection::Connection; +use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; +use crate::database::{CreateTableRequest, Database, OpenTableRequest}; +use crate::embeddings::EmbeddingDefinition; +use crate::table::Table; +use crate::table::refresh::quote_identifier; +use crate::table::{ColumnDefinition, ColumnKind}; +use crate::{Error, Result}; + +pub use refresh::{RefreshMaterializedViewResult, RefreshMode}; + +/// Schema metadata key holding the view definition, as kind-tagged JSON. +pub const DEFINITION_META_KEY: &str = "mv.definition"; + +/// Schema metadata key holding the view's incarnation: a token minted at each +/// physical creation of a view table, so a view dropped and recreated under +/// the same name and definition is still told apart from the one a caller +/// captured. A view whose metadata was replaced wholesale, or one declared +/// before tokens existed, carries none until its next refresh mints one. +pub const INCARNATION_META_KEY: &str = "mv.incarnation"; + +/// Schema metadata key holding the source table version the view was last +/// refreshed to. Absent until the first refresh. +pub const SOURCE_VERSION_META_KEY: &str = "mv.source_version"; + +/// Schema metadata key holding the wall-clock time of the last refresh, +/// in milliseconds since the epoch. +pub const REFRESHED_AT_MS_META_KEY: &str = "mv.refreshed_at_ms"; + +/// Column recording which source row produced each view row: the source's +/// stable `_rowid` at refresh time, which is why sources must keep stable +/// row ids. +pub const SOURCE_ROW_ID_COLUMN: &str = "__source_row_id"; + +/// Field metadata namespace for declarations about schema structure, such as +/// an unenforced primary key. +const SCHEMA_DECLARATION_META_PREFIX: &str = "lance-schema:"; + +/// A field's identity in its own schema, which is not the view's. +const LANCE_FIELD_ID_KEY: &str = "lance:field_id"; + +/// Schema metadata key holding embedding-function configuration. It describes +/// columns rather than storage, so a view carries it through. +const EMBEDDING_FUNCTIONS_META_KEY: &str = "embedding_functions"; + +/// Schema metadata key holding lancedb's own column definitions, one per +/// field in schema order. It marks which columns an embedding function +/// produces, which is what lets a query embed its own text. +const COLUMN_DEFINITIONS_META_KEY: &str = "lancedb::column_definitions"; + +/// Value of the definition's `kind` tag for the projected `select` form. +pub const SELECT_KIND: &str = "select"; + +/// Which view outputs each source column is projected to directly. A column +/// may be projected more than once, so each carries every name the view gives +/// it, in projection order. +type Lineage = HashMap>; + +/// One projected output column of a view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ViewProjection { + /// Name of the column in the view. + pub output: String, + /// SQL expression over the source table that computes it. + pub expression: String, +} + +/// The query that defines a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaterializedViewDefinition { + /// Name of the source table, in the same database as the view. + pub source_table: String, + /// The projected output columns, in view schema order. + pub projections: Vec, + /// SQL predicate selecting the source rows the view holds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub filter: Option, + /// Cap on the number of rows the view holds, in materialization order. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// Source columns the projections and filter read, derived at creation. + #[serde(default)] + pub inputs: Vec, +} + +/// A view definition as read back from schema metadata. Non-exhaustive so a +/// kind added later is additive. +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum MaterializedViewKind { + /// The projected `select` form. + Select(MaterializedViewDefinition), + /// A kind written by a newer version, reported so a caller can tell an + /// unrefreshable view apart from a plain table. Nothing produces this. + Unrecognized { + /// The kind as it was found in the metadata. + kind: String, + }, +} + +/// Serialize `definition` into the kind-tagged form stored under +/// [`DEFINITION_META_KEY`]. +pub(crate) fn definition_to_metadata(definition: &MaterializedViewDefinition) -> Result { + let mut value = serde_json::to_value(definition).map_err(|e| Error::Runtime { + message: format!("failed to serialize view definition: {e}"), + })?; + value["kind"] = serde_json::Value::String(SELECT_KIND.to_string()); + Ok(value.to_string()) +} + +/// Read a view declaration off a schema metadata map, if it carries one. +/// `Ok(None)` for a plain table; a declaration that does not parse is an +/// error, because treating a view as plain would let it be rewritten. +pub fn materialized_view_kind( + metadata: &HashMap, +) -> Result> { + let Some(raw) = metadata.get(DEFINITION_META_KEY) else { + return Ok(None); + }; + let unreadable = |e: &dyn std::fmt::Display| Error::Runtime { + message: format!("unreadable materialized view definition: {e}"), + }; + let value: serde_json::Value = serde_json::from_str(raw).map_err(|e| unreadable(&e))?; + let kind = value + .get("kind") + .and_then(|k| k.as_str()) + .ok_or_else(|| unreadable(&"missing kind tag"))?; + if kind != SELECT_KIND { + return Ok(Some(MaterializedViewKind::Unrecognized { + kind: kind.to_string(), + })); + } + let definition = serde_json::from_value(value).map_err(|e| unreadable(&e))?; + Ok(Some(MaterializedViewKind::Select(definition))) +} + +/// Resolve a definition against the source schema into the view's projected +/// fields, with `inputs` filled in. Everything statically checkable is +/// checked here rather than at refresh time. Empty `projections` selects +/// every source column as the schema stands now. +pub(crate) fn plan( + source_schema: SchemaRef, + source_table: &str, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result<(MaterializedViewDefinition, Vec, Lineage)> { + let projections: Vec<(String, String)> = if projections.is_empty() { + source_schema + .fields() + .iter() + // A source that is itself a view carries its own provenance + // column; the new view records its own, not a copy. + .filter(|f| f.name() != SOURCE_ROW_ID_COLUMN) + .map(|f| (f.name().clone(), quote_identifier(f.name()))) + .collect() + } else { + projections.to_vec() + }; + + // A scan takes the cap as i64. Rejecting it here keeps creation and + // refresh from disagreeing about whether a view is valid. + if let Some(limit) = limit + && i64::try_from(limit).is_err() + { + return Err(Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + }); + } + + let planner = Planner::new(source_schema.clone()); + let mut fields = Vec::with_capacity(projections.len()); + let mut inputs = Vec::new(); + let mut declared: Vec<&str> = Vec::with_capacity(projections.len()); + let mut lineage: Lineage = HashMap::new(); + + for (output, expression) in &projections { + if declared.contains(&output.as_str()) { + return Err(Error::ColumnAlreadyExists { + name: output.clone(), + }); + } + if output == SOURCE_ROW_ID_COLUMN || output == ROW_ID { + return Err(Error::InvalidInput { + message: format!("view column name '{output}' is reserved"), + }); + } + + let parsed = planner + .parse_expr(expression) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + // Before optimization: the simplifier folds a stable-but-not-immutable + // call like now() into a literal, hiding it from the check while the + // stored definition keeps the call. + ensure_immutable(&parsed, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + let expr = planner + .optimize_expr(parsed) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let expr_inputs = + resolve_inputs(&source_schema, &expr, |message| Error::InvalidExpression { + column: output.clone(), + message, + })?; + + // Physical expressions address columns by position, so the planner + // that types the expression is built on the projected schema. + let read_schema = project_schema(&source_schema, &expr_inputs); + let physical = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + let data_type = + physical + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidExpression { + column: output.clone(), + message: e.to_string(), + })?; + + // Always nullable: what a refresh appends must fit the declared field + // whatever nullability the evaluator reports for a given batch. + let mut field = ArrowField::new(output, data_type, true); + // Identity projections keep descriptive field metadata (blob markers); + // computed values carry none. Structural declarations never come along. + if let Some(source_field) = projected_field(&expr, &source_schema) { + field = field.with_metadata(source_field.metadata().clone()); + } + if let Some(path) = projected_path(&expr) + && let [column] = path.as_slice() + { + lineage + .entry(column.clone()) + .or_default() + .push(output.clone()); + } + fields.push(without_declarations(&field)); + inputs.extend(expr_inputs); + declared.push(output); + } + + if let Some(filter) = filter { + let expr = planner + .parse_filter(filter) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + ensure_immutable(&expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + let filter_inputs = resolve_inputs(&source_schema, &expr, |message| Error::InvalidInput { + message: format!("invalid view filter: {message}"), + })?; + // A committed filter has to be usable as a predicate. + let read_schema = project_schema(&source_schema, &filter_inputs); + let data_type = Planner::new(read_schema.clone()) + .create_physical_expr(&expr) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })? + .data_type(read_schema.as_ref()) + .map_err(|e| Error::InvalidInput { + message: format!("invalid view filter: {e}"), + })?; + if data_type != DataType::Boolean { + return Err(Error::InvalidInput { + message: format!("view filter must be a boolean predicate, not {data_type}"), + }); + } + inputs.extend(filter_inputs); + } + + inputs.sort(); + inputs.dedup(); + + let definition = MaterializedViewDefinition { + source_table: source_table.to_string(), + projections: projections + .into_iter() + .map(|(output, expression)| ViewProjection { output, expression }) + .collect(), + filter: filter.map(String::from), + limit, + inputs, + }; + Ok((definition, fields, lineage)) +} + +/// Reject any function that is not immutable: a view definition has to +/// evaluate identically across refreshes, or incremental maintenance would +/// mix rows from different evaluations of the same definition. +fn ensure_immutable(expr: &datafusion_expr::Expr, error: impl Fn(String) -> Error) -> Result<()> { + use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion}; + use datafusion_expr::Volatility; + + // Labeled immutable but not determined by row values alone: version() + // depends on the build, the arrow_* introspectors on schema state. + const NOT_VALUE_DETERMINED: &[&str] = + &["version", "arrow_typeof", "arrow_field", "arrow_metadata"]; + + let mut offending: Option = None; + expr.apply(|node| { + if let datafusion_expr::Expr::ScalarFunction(function) = node { + let name = function.func.name(); + if function.func.signature().volatility != Volatility::Immutable + || NOT_VALUE_DETERMINED.contains(&name) + { + offending = Some(name.to_string()); + return Ok(TreeNodeRecursion::Stop); + } + } + Ok(TreeNodeRecursion::Continue) + }) + .map_err(|e| error(e.to_string()))?; + match offending { + Some(name) => Err(error(format!( + "function '{name}' is not immutable and would evaluate differently \ + across refreshes" + ))), + None => Ok(()), + } +} + +/// The root of a possibly-dotted column path: `metadata.age` -> `metadata`. +fn root(path: &str) -> &str { + path.split('.').next().unwrap_or(path) +} + +/// The columns `expr` reads, kept as the planner reports them (a nested +/// reference stays a dotted path) but resolved by root field. +/// Embedding configuration rewritten for the view: entries whose columns the +/// view projects directly are kept under the view's names; the rest describe +/// a table that does not exist and are dropped. +fn embedding_config_for_view(raw: &str, lineage: &Lineage) -> Option { + // Every representation the writers use: the Python bindings name the + // destination `vector_column`, the Rust definition `dest_column`, and the + // Node bindings spell both halves in camelCase. + const SOURCE_KEYS: [&str; 2] = ["source_column", "sourceColumn"]; + const DEST_KEYS: [&str; 4] = ["vector_column", "dest_column", "vectorColumn", "destColumn"]; + + let entries: Vec = serde_json::from_str(raw).ok()?; + let mut kept = Vec::new(); + for entry in &entries { + let Some(object) = entry.as_object() else { + continue; + }; + let named = |keys: &[&str]| { + let key = keys.iter().find(|key| object.contains_key(**key))?; + let outputs = lineage.get(object.get(*key)?.as_str()?)?; + Some(((*key).to_string(), outputs)) + }; + let (Some((source_key, sources)), Some((dest_key, dests))) = + (named(&SOURCE_KEYS), named(&DEST_KEYS)) + else { + continue; + }; + // A projection may give one source column several names, and every + // pairing of the two is a real relationship in the view. + for source in sources { + for dest in dests { + let mut object = object.clone(); + object.insert(source_key.clone(), source.clone().into()); + object.insert(dest_key.clone(), dest.clone().into()); + kept.push(serde_json::Value::Object(object)); + } + } + } + (!kept.is_empty()).then(|| serde_json::Value::Array(kept).to_string()) +} + +/// Lancedb's column definitions rewritten for the view: positional, one per +/// view field. Directly projected embedding columns keep their definition +/// under the view's names; everything else is physical. `None` = no key. +fn column_definitions_for_view( + raw: &str, + source_schema: &ArrowSchema, + view_fields: &[ArrowField], + lineage: &Lineage, +) -> Option { + let source_definitions: Vec = serde_json::from_str(raw).ok()?; + // The definition sits on the column the function writes, so the source + // schema's field name at that position is the embedding's destination. + let embeddings: HashMap<&str, &EmbeddingDefinition> = source_schema + .fields() + .iter() + .zip(&source_definitions) + .filter_map(|(field, definition)| match &definition.kind { + ColumnKind::Embedding(embedding) => Some((field.name().as_str(), embedding)), + ColumnKind::Physical => None, + }) + .collect(); + let sources: HashMap<&str, &str> = lineage + .iter() + .flat_map(|(source, outputs)| outputs.iter().map(move |o| (o.as_str(), source.as_str()))) + .collect(); + + let mut kept = false; + let definitions: Vec = view_fields + .iter() + .map(|field| { + let kind = embedding_for_output(field.name(), &embeddings, &sources, lineage) + .map(|embedding| { + kept = true; + ColumnKind::Embedding(embedding) + }) + .unwrap_or(ColumnKind::Physical); + ColumnDefinition { kind } + }) + .collect(); + kept.then(|| serde_json::to_string(&definitions).ok())? +} + +/// The embedding `output` inherits, renamed to the view's columns. `None` +/// unless the view projects both the function's input and its output +/// directly: anything else advertises a column the view cannot recompute. +fn embedding_for_output( + output: &str, + embeddings: &HashMap<&str, &EmbeddingDefinition>, + sources: &HashMap<&str, &str>, + lineage: &Lineage, +) -> Option { + let embedding = embeddings.get(sources.get(output)?)?; + // The input may be projected several times; the first name the view gives + // it is the one this column is defined against. + let input = lineage.get(&embedding.source_column)?.first()?; + Some(EmbeddingDefinition { + source_column: input.clone(), + dest_column: Some(output.to_string()), + embedding_name: embedding.embedding_name.clone(), + }) +} + +/// The source field a projection reads directly, if it reads one: a bare +/// column, or a path of struct field accesses over one. Anything computed +/// produces a new value and has no source field. +fn projected_field<'a>( + expr: &datafusion_expr::Expr, + schema: &'a ArrowSchema, +) -> Option<&'a ArrowField> { + let path = projected_path(expr)?; + let mut segments = path.iter(); + let mut field = schema.field_with_name(segments.next()?).ok()?; + for segment in segments { + let DataType::Struct(children) = field.data_type() else { + return None; + }; + field = children.iter().find(|c| c.name() == segment)?; + } + Some(field) +} + +/// The dotted path a projection reads directly, root first. +fn projected_path(expr: &datafusion_expr::Expr) -> Option> { + let mut path = Vec::new(); + let mut node = expr; + loop { + match node { + datafusion_expr::Expr::Column(column) => { + path.push(column.name.clone()); + break; + } + // `a.b` parses to get_field(a, "b"), nested for deeper paths. + datafusion_expr::Expr::ScalarFunction(call) if call.func.name() == "get_field" => { + let [ + inner, + datafusion_expr::Expr::Literal(ScalarValue::Utf8(Some(name)), _), + ] = call.args.as_slice() + else { + return None; + }; + path.push(name.clone()); + node = inner; + } + _ => return None, + } + } + + path.reverse(); + Some(path) +} + +/// `field` without the metadata that declares how a column is written, at +/// every depth; descriptive metadata (blob markers) stays. A view is written +/// by refresh alone, and its always-nullable fields contradict declarations. +fn is_declaration(key: &str) -> bool { + key.starts_with(SCHEMA_DECLARATION_META_PREFIX) + || key == LANCE_FIELD_ID_KEY + || crate::table::computed_columns::is_declaration_key(key) +} + +fn without_declarations(field: &ArrowField) -> ArrowField { + let metadata: HashMap = field + .metadata() + .iter() + .filter(|(key, _)| !is_declaration(key)) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let strip = |child: &FieldRef| Arc::new(without_declarations(child)); + // Every Arrow variant that carries a field carries that field's metadata + // with it, so all of them are descended. + let data_type = match field.data_type() { + DataType::Struct(children) => DataType::Struct(children.iter().map(strip).collect()), + DataType::List(child) => DataType::List(strip(child)), + DataType::ListView(child) => DataType::ListView(strip(child)), + DataType::LargeList(child) => DataType::LargeList(strip(child)), + DataType::LargeListView(child) => DataType::LargeListView(strip(child)), + DataType::Map(entries, sorted) => DataType::Map(strip(entries), *sorted), + DataType::FixedSizeList(child, len) => DataType::FixedSizeList(strip(child), *len), + DataType::Union(variants, mode) => DataType::Union( + variants + .iter() + .map(|(id, child)| (id, strip(child))) + .collect(), + *mode, + ), + DataType::RunEndEncoded(run_ends, values) => { + DataType::RunEndEncoded(strip(run_ends), strip(values)) + } + other => other.clone(), + }; + ArrowField::new(field.name(), data_type, field.is_nullable()).with_metadata(metadata) +} + +fn resolve_inputs( + schema: &ArrowSchema, + expr: &datafusion_expr::Expr, + error: impl Fn(String) -> Error, +) -> Result> { + let mut inputs = Planner::column_names_in_expr(expr); + inputs.sort(); + inputs.dedup(); + for input in &inputs { + if schema.field_with_name(root(input)).is_err() { + return Err(error(format!("unknown column '{input}'"))); + } + } + Ok(inputs) +} + +/// Project the root fields of `columns`, deduplicated, in schema order. +fn project_schema(schema: &ArrowSchema, columns: &[String]) -> SchemaRef { + let roots: std::collections::HashSet<&str> = columns.iter().map(|c| root(c)).collect(); + let fields: Vec = schema + .fields() + .iter() + .filter(|f| roots.contains(f.name().as_str())) + .map(|f| f.as_ref().clone()) + .collect(); + Arc::new(ArrowSchema::new(fields)) +} + +/// A validated view declaration, ready to become a table: the projected +/// fields plus [`SOURCE_ROW_ID_COLUMN`], definition stamped in metadata. +/// Produced only by [`prepare_declaration`]. +#[derive(Clone)] +pub struct PreparedDeclaration { + schema: SchemaRef, + definition: MaterializedViewDefinition, + /// The source's own database: the only place + /// [`PreparedDeclaration::create`] will put the view, because refresh + /// resolves the recorded source name through the view's database. + database: Arc, +} + +impl std::fmt::Debug for PreparedDeclaration { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PreparedDeclaration") + .field("definition", &self.definition) + .finish_non_exhaustive() + } +} + +impl PreparedDeclaration { + /// The query the declaration records. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } + + /// Create the view table and verify it, consuming the declaration. + /// + /// The view goes in the source's own database, where refresh resolves the + /// recorded source name. Stable row ids are requested at both levels and + /// verified rather than trusted; nothing is rolled back on failure. + pub async fn create(self, name: &str) -> Result { + let empty: Vec> = + vec![]; + // Minted here, not at preparation: a declaration can be cloned and + // create more than one physical table, and each needs its own token. + let incarnation = uuid::Uuid::new_v4().to_string(); + let mut metadata = self.schema.metadata().clone(); + metadata.insert(INCARNATION_META_KEY.to_string(), incarnation.clone()); + let schema = Arc::new(ArrowSchema::new_with_metadata( + self.schema.fields().clone(), + metadata, + )); + let reader: Box = + Box::new(arrow_array::RecordBatchIterator::new(empty, schema)); + let mut request = CreateTableRequest::new(name.to_string(), Box::new(reader)); + let write_params = request + .write_options + .lance_write_params + .get_or_insert_with(Default::default); + write_params.enable_stable_row_ids = true; + let store_params = write_params + .store_params + .get_or_insert_with(Default::default); + crate::connection::merge_storage_options( + store_params, + [( + OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS.to_string(), + "true".to_string(), + )], + ); + let table = self.database.clone().create_table(request).await?; + let table = Table::new(table, self.database); + let stable = match table.as_native() { + Some(native) => native.dataset.get().await?.manifest.uses_stable_row_ids(), + None => false, + }; + if !stable { + return Err(Error::Runtime { + message: format!( + "view '{name}' was created without stable row ids: the database \ + ignored the creation option; the table remains and is not \ + usable as a materialized view" + ), + }); + } + Ok(MaterializedView { + table, + definition: self.definition, + incarnation: Some(incarnation), + }) + } +} + +/// Validate a view declaration against its live source and hold what its +/// creation needs. The declaration is canonicalized through the coordinate a +/// refresh will resolve, so a handle that does not resolve back to itself is +/// rejected, as is a namespaced source. Same creation-time checks as +/// [`Connection::create_materialized_view`]. +/// +/// ```no_run +/// # #![recursion_limit = "256"] +/// # use lancedb::materialized_view::prepare_declaration; +/// # async fn declare(source: &lancedb::Table) -> Result<(), Box> { +/// let prepared = prepare_declaration( +/// source, +/// &[("id".into(), "id".into()), ("double".into(), "value * 2".into())], +/// Some("value > 0"), +/// None, +/// ) +/// .await?; +/// let view = prepared.create("doubles").await?; +/// # Ok(()) +/// # } +/// ``` +pub async fn prepare_declaration( + source: &Table, + projections: &[(String, String)], + filter: Option<&str>, + limit: Option, +) -> Result { + let Some(caller_native) = source.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + // The definition records the source by bare name; any other source + // form would be recorded as a name its refresh cannot resolve. + if !source.namespace().is_empty() { + return Err(Error::NotSupported { + message: format!( + "a namespaced source cannot be recorded in a view definition; \ + '{}' must be a root-namespace table", + source.name() + ), + }); + } + let database = source + .database_opt() + .ok_or_else(|| Error::InvalidInput { + message: "the source was not opened through a database connection".into(), + })? + .clone(); + + // Canonicalize: resolve the recorded coordinate exactly the way a + // refresh will, and plan from what it reaches. A handle that does not + // resolve back to itself must not be declared under this name. + let resolved = database + .open_table(OpenTableRequest { + name: source.name().to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let resolved = Table::new(resolved, database.clone()); + let Some(native) = resolved.as_native() else { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + }; + let caller_uri = caller_native.dataset.get().await?.uri().to_string(); + let resolved_uri = native.dataset.get().await?.uri().to_string(); + if caller_uri != resolved_uri { + return Err(Error::InvalidInput { + message: format!( + "the source handle does not resolve to itself through its \ + database: '{}' resolves to '{resolved_uri}', but the handle \ + reads '{caller_uri}'", + source.name() + ), + }); + } + if !native.dataset.get().await?.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "materialized views require stable row ids on the source table; \ + create '{}' with storage option new_table_enable_stable_row_ids=true", + source.name() + ), + }); + } + refresh::ensure_no_mem_wal( + native.dataset.get().await?.as_ref(), + "source table", + resolved.name(), + ) + .await?; + let source_schema = resolved.schema().await?; + let source_metadata = source_schema.metadata().clone(); + let (definition, mut fields, lineage) = plan( + source_schema.clone(), + resolved.name(), + projections, + filter, + limit, + )?; + fields.push(ArrowField::new( + SOURCE_ROW_ID_COLUMN, + DataType::UInt64, + false, + )); + // Only column-describing metadata comes along: structural declarations + // describe how a table is written, and a view is written by refresh alone. + let mut metadata: HashMap = HashMap::new(); + if let Some(raw) = source_metadata.get(EMBEDDING_FUNCTIONS_META_KEY) + && let Some(rewritten) = embedding_config_for_view(raw, &lineage) + { + metadata.insert(EMBEDDING_FUNCTIONS_META_KEY.to_string(), rewritten); + } + if let Some(raw) = source_metadata.get(COLUMN_DEFINITIONS_META_KEY) + && let Some(rewritten) = column_definitions_for_view(raw, &source_schema, &fields, &lineage) + { + metadata.insert(COLUMN_DEFINITIONS_META_KEY.to_string(), rewritten); + } + metadata.insert( + DEFINITION_META_KEY.to_string(), + definition_to_metadata(&definition)?, + ); + Ok(PreparedDeclaration { + schema: Arc::new(ArrowSchema::new_with_metadata(fields, metadata)), + definition, + database, + }) +} + +/// One row of [`Connection::list_materialized_views`]: a view's name and its +/// definition kind, which may be one this version cannot refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MaterializedViewEntry { + /// Name of the view's table. + pub name: String, + /// The view's definition as stored. + pub kind: MaterializedViewKind, +} + +/// Materialized views are local-only; refuse a remote connection before any +/// request is made. +fn ensure_local(connection: &Connection) -> Result<()> { + if connection.uri().starts_with("db://") { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + Ok(()) +} + +/// Builds a materialized view. Created by +/// [`Connection::create_materialized_view`]. +pub struct CreateMaterializedViewBuilder { + connection: Connection, + name: String, + source: String, + projections: Vec<(String, String)>, + filter: Option, + limit: Option, +} + +impl CreateMaterializedViewBuilder { + pub(crate) fn new(connection: Connection, name: String, source: String) -> Self { + Self { + connection, + name, + source, + projections: Vec::new(), + filter: None, + limit: None, + } + } + + /// The view's columns, as `(name, SQL expression)` pairs. Not calling + /// this selects every source column, expanded at creation time. + pub fn select( + mut self, + columns: impl IntoIterator, impl Into)>, + ) -> Self { + self.projections = columns + .into_iter() + .map(|(output, expression)| (output.into(), expression.into())) + .collect(); + self + } + + /// Only source rows matching the SQL predicate appear in the view. + pub fn only_if(mut self, filter: impl Into) -> Self { + self.filter = Some(filter.into()); + self + } + + /// Cap the view at `limit` rows, in materialization order. + pub fn limit(mut self, limit: u64) -> Self { + self.limit = Some(limit); + self + } + + /// Create the view: an empty table carrying the definition; refresh + /// computes the rows. The source must keep stable row ids -- they hold + /// provenance across compaction, and cannot be enabled later. + pub async fn execute(self) -> Result { + ensure_local(&self.connection)?; + let source = self.connection.open_table(&self.source).execute().await?; + let prepared = prepare_declaration( + &source, + &self.projections, + self.filter.as_deref(), + self.limit, + ) + .await?; + prepared.create(&self.name).await + } +} + +/// A handle on a materialized view: the view table plus its parsed definition. +#[derive(Debug, Clone)] +pub struct MaterializedView { + table: Table, + definition: MaterializedViewDefinition, + incarnation: Option, +} + +impl MaterializedView { + /// Interpret `table` as a materialized view: [`Error::NotAMaterializedView`] + /// for a plain table, [`Error::NotSupported`] for a kind this version + /// cannot refresh. + pub async fn from_table(table: Table) -> Result { + // Same local-only boundary the connection-level entry points hold, + // applied before the schema read so a remote table costs no request. + if table.as_native().is_none() { + return Err(Error::NotSupported { + message: "materialized views are supported only on local databases".into(), + }); + } + let schema = table.schema().await?; + let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned(); + match materialized_view_kind(schema.metadata())? { + Some(MaterializedViewKind::Select(definition)) => Ok(Self { + table, + definition, + incarnation, + }), + Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this version of \ + lancedb cannot refresh", + table.name() + ), + }), + None => Err(Error::NotAMaterializedView { + name: table.name().to_string(), + }), + } + } + + /// The view, as the table it is. Queries, indexes and search all apply. + pub fn table(&self) -> &Table { + &self.table + } + + /// The view's table name. + pub fn name(&self) -> &str { + self.table.name() + } + + /// The query that defines the view. + pub fn definition(&self) -> &MaterializedViewDefinition { + &self.definition + } + + /// The view's incarnation token as of when this handle was opened; see + /// [`RefreshMaterializedViewBuilder::expect_incarnation`]. `None` for a + /// view that has none yet (see [`INCARNATION_META_KEY`]). + pub fn incarnation(&self) -> Option<&str> { + self.incarnation.as_deref() + } + + /// Recompute the view from its source. + /// + /// By default the refresh is incremental when the source's changes can be + /// reconciled into the view, and otherwise rebuilds; see + /// [`RefreshMaterializedViewBuilder`]. + /// + /// ```no_run + /// # #![recursion_limit = "256"] + /// # use lancedb::materialized_view::MaterializedView; + /// # async fn refresh(view: &MaterializedView) -> Result<(), Box> { + /// let result = view.refresh().execute().await?; + /// println!("{:?}: {} rows", result.mode, result.rows_written); + /// # Ok(()) + /// # } + /// ``` + pub fn refresh(&self) -> RefreshMaterializedViewBuilder { + RefreshMaterializedViewBuilder { + view: self.clone(), + full: false, + source_version: None, + expected_incarnation: None, + } + } +} + +/// Builds a refresh. Created by [`MaterializedView::refresh`]. +pub struct RefreshMaterializedViewBuilder { + view: MaterializedView, + full: bool, + source_version: Option, + expected_incarnation: Option, +} + +impl RefreshMaterializedViewBuilder { + /// Rebuild the view even where an incremental refresh would do. + pub fn full(mut self, full: bool) -> Self { + self.full = full; + self + } + + /// Refresh to this source table version instead of the latest. + pub fn source_version(mut self, version: u64) -> Self { + self.source_version = Some(version); + self + } + + /// Refresh only if the view is still the incarnation that minted `token` + /// (see [`MaterializedView::incarnation`]): a refresh requested against + /// one declaration must not land in a view dropped and recreated since, + /// even under the same name and definition. + /// + /// Best effort. The token is read from the latest stored manifest before + /// planning and again immediately before every commit, but it is not part + /// of the commit's own condition, so a recreation that lands between that + /// final read and the commit is not caught. + pub fn expect_incarnation(mut self, token: impl Into) -> Self { + self.expected_incarnation = Some(token.into()); + self + } + + pub async fn execute(self) -> Result { + refresh::execute_refresh( + &self.view.table, + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await + } +} + +impl Connection { + /// Define a materialized view named `name` over `source`. + /// + /// The view is created empty, with the definition recorded in its schema + /// metadata; refresh computes the rows. Local databases only. + /// + /// ```no_run + /// # #![recursion_limit = "256"] + /// # use lancedb::Connection; + /// # async fn create(conn: &Connection) -> Result<(), Box> { + /// let view = conn + /// .create_materialized_view("loud_adults", "people") + /// .select([("name", "upper(name)"), ("age", "age")]) + /// .only_if("age >= 18") + /// .execute() + /// .await?; + /// view.refresh().execute().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn create_materialized_view( + &self, + name: impl Into, + source: impl Into, + ) -> CreateMaterializedViewBuilder { + CreateMaterializedViewBuilder::new(self.clone(), name.into(), source.into()) + } + + /// Open the materialized view named `name`. + pub async fn open_materialized_view( + &self, + name: impl Into, + ) -> Result { + ensure_local(self)?; + let table = self.open_table(name).execute().await?; + MaterializedView::from_table(table).await + } + + /// The materialized views in this database, unrefreshable kinds included. + /// Costs a table open per table; one that cannot be opened is skipped + /// rather than failing the listing. + pub async fn list_materialized_views(&self) -> Result> { + ensure_local(self)?; + let names = self.table_names().execute().await?; + let mut views = Vec::new(); + for name in names { + let Ok(table) = self.open_table(&name).execute().await else { + continue; + }; + let schema = table.schema().await?; + if let Some(kind) = materialized_view_kind(schema.metadata())? { + views.push(MaterializedViewEntry { name, kind }); + } + } + Ok(views) + } +} + +#[cfg(test)] +mod tests { + use arrow_array::record_batch; + + use super::*; + use crate::connect; + use crate::table::WriteOptions; + + async fn people_db() -> Connection { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("name", Utf8, ["ada", "grace", "alan"]), + ("age", Int32, [36, 85, 41]) + ) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + conn + } + + /// Sources must keep stable row ids; see the create-time gate. + pub(super) fn stable_row_ids() -> WriteOptions { + WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + ..Default::default() + }), + } + } + + /// The error a doomed declaration against `people` produces. + async fn declare_err( + cfg: impl FnOnce(CreateMaterializedViewBuilder) -> CreateMaterializedViewBuilder, + ) -> Error { + let conn = people_db().await; + cfg(conn.create_materialized_view("bad", "people")) + .execute() + .await + .unwrap_err() + } + + #[tokio::test] + async fn test_create_records_the_definition() { + let conn = people_db().await; + let view = conn + .create_materialized_view("adults", "people") + .select([("name", "name"), ("shout", "upper(name)")]) + .only_if("age >= 18") + .limit(10) + .execute() + .await + .unwrap(); + + assert_eq!(view.name(), "adults"); + assert_eq!( + view.definition(), + &MaterializedViewDefinition { + source_table: "people".into(), + projections: vec![ + ViewProjection { + output: "name".into(), + expression: "name".into() + }, + ViewProjection { + output: "shout".into(), + expression: "upper(name)".into() + }, + ], + filter: Some("age >= 18".into()), + limit: Some(10), + inputs: vec!["age".into(), "name".into()], + } + ); + + // The definition round-trips off the stored schema, not the handle. + let reopened = conn.open_materialized_view("adults").await.unwrap(); + assert_eq!(reopened.definition(), view.definition()); + } + + #[tokio::test] + async fn test_view_schema_is_derived_from_the_query() { + let conn = people_db().await; + let view = conn + .create_materialized_view("shapes", "people") + .select([("shout", "upper(name)"), ("next_age", "age + 1")]) + .execute() + .await + .unwrap(); + + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("shout").unwrap().data_type(), + &DataType::Utf8 + ); + assert_eq!( + schema.field_with_name("next_age").unwrap().data_type(), + &DataType::Int32 + ); + assert_eq!( + schema + .field_with_name(SOURCE_ROW_ID_COLUMN) + .unwrap() + .data_type(), + &DataType::UInt64 + ); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + } + + /// No projection selects every source column, expanded now: the schema + /// captured at creation is the definition. + #[tokio::test] + async fn test_default_projection_captures_the_source_schema() { + let conn = people_db().await; + let view = conn + .create_materialized_view("copy", "people") + .execute() + .await + .unwrap(); + assert_eq!( + view.definition() + .projections + .iter() + .map(|p| p.output.as_str()) + .collect::>(), + vec!["name", "age"] + ); + assert_eq!(view.definition().inputs, vec!["age", "name"]); + } + + #[tokio::test] + async fn test_unknown_column_fails_at_create_time() { + let conn = people_db().await; + let err = conn + .create_materialized_view("bad", "people") + .select([("x", "missing + 1")]) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidExpression { column, .. } if column == "x")); + let names = conn.table_names().execute().await.unwrap(); + assert!(!names.contains(&"bad".to_string())); + } + + #[tokio::test] + async fn test_unknown_filter_column_fails_at_create_time() { + let err = declare_err(|b| b.only_if("missing > 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("missing"))); + } + + #[tokio::test] + async fn test_duplicate_output_is_rejected() { + let err = declare_err(|b| b.select([("dup", "age"), ("dup", "age + 1")])).await; + assert!(matches!(err, Error::ColumnAlreadyExists { name } if name == "dup")); + } + + #[tokio::test] + async fn test_reserved_output_name_is_rejected() { + let err = declare_err(|b| b.select([(SOURCE_ROW_ID_COLUMN, "age")])).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("reserved"))); + } + + #[tokio::test] + async fn test_missing_source_fails() { + let conn = connect("memory://").execute().await.unwrap(); + let err = conn + .create_materialized_view("v", "nope") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableNotFound { .. })); + } + + /// Provenance has to survive source compactions and updates, and stable + /// row ids cannot be enabled after a table exists -- so the requirement + /// is checked at the last moment the caller can still act on it. + #[tokio::test] + async fn test_source_without_stable_row_ids_is_refused() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + conn.create_table("plain", batch).execute().await.unwrap(); + + let err = conn + .create_materialized_view("v", "plain") + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + assert!( + !conn + .table_names() + .execute() + .await + .unwrap() + .contains(&"v".to_string()) + ); + } + + #[tokio::test] + async fn test_name_collision_fails() { + let conn = people_db().await; + let err = conn + .create_materialized_view("people", "people") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::TableAlreadyExists { .. })); + } + + #[tokio::test] + async fn test_a_plain_table_is_not_a_view() { + let conn = people_db().await; + let table = conn.open_table("people").execute().await.unwrap(); + let err = MaterializedView::from_table(table).await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { name } if name == "people")); + + let err = conn.open_materialized_view("people").await.unwrap_err(); + assert!(matches!(err, Error::NotAMaterializedView { .. })); + } + + /// The reason the kind is tagged: a definition written by a newer version + /// reads back as a view this one cannot refresh, not as a plain table. + #[tokio::test] + async fn test_unrecognized_kind_is_refused_by_name() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { message } if message.contains("join"))); + } + + #[tokio::test] + async fn test_list_reports_views_and_only_views() { + let conn = people_db().await; + conn.create_materialized_view("adults", "people") + .only_if("age >= 18") + .execute() + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!( + views.iter().map(|v| v.name.as_str()).collect::>(), + vec!["adults"] + ); + let MaterializedViewKind::Select(definition) = &views[0].kind else { + panic!("expected a select view"); + }; + assert_eq!(definition.filter.as_deref(), Some("age >= 18")); + } + + /// The creation option outranks a connection configured to create + /// unstable tables: the view still gets stable row ids, on the same + /// store (no fork -- the table must be reachable through the + /// connection afterwards). + #[tokio::test] + async fn test_view_is_stable_despite_connection_override() { + let conn = connect("memory://") + .storage_options([("new_table_enable_stable_row_ids", "false")]) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1])).unwrap(); + conn.create_table("src", batch) + .storage_option("new_table_enable_stable_row_ids", "true") + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let stable = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .manifest + .uses_stable_row_ids(); + assert!(stable); + conn.open_materialized_view("v").await.unwrap(); + } + + /// A committed filter has to be usable as a predicate. + #[tokio::test] + async fn test_non_boolean_filter_is_rejected() { + let err = declare_err(|b| b.only_if("age + 1")).await; + assert!(matches!(err, Error::InvalidInput { message } if message.contains("boolean"))); + } + + /// Nested references stay dotted paths; resolution is by root field. + #[tokio::test] + async fn test_struct_columns_can_be_declared() { + use arrow_array::{ArrayRef, Int32Array, StructArray}; + + let conn = connect("memory://").execute().await.unwrap(); + let ages = StructArray::from(vec![( + Arc::new(ArrowField::new("age", DataType::Int32, false)), + Arc::new(Int32Array::from(vec![36, 17])) as ArrayRef, + )]); + let batch = + arrow_array::RecordBatch::try_from_iter(vec![("metadata", Arc::new(ages) as ArrayRef)]) + .unwrap(); + conn.create_table("people", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("ages", "people") + .select([("age", "metadata.age")]) + .only_if("metadata.age >= 18") + .execute() + .await + .unwrap(); + assert_eq!(view.definition().inputs, vec!["metadata.age"]); + let schema = view.table().schema().await.unwrap(); + assert_eq!( + schema.field_with_name("age").unwrap().data_type(), + &DataType::Int32 + ); + } + + /// A newer-kind view must not disappear from the listing. + #[tokio::test] + async fn test_unrecognized_kind_is_listed_with_its_kind() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + let table = conn.open_table("v").execute().await.unwrap(); + table + .as_native() + .unwrap() + .replace_schema_metadata(HashMap::from([( + DEFINITION_META_KEY.to_string(), + r#"{"kind": "join"}"#.to_string(), + )])) + .await + .unwrap(); + + let views = conn.list_materialized_views().await.unwrap(); + assert_eq!(views.len(), 1); + assert_eq!(views[0].name, "v"); + assert_eq!( + views[0].kind, + MaterializedViewKind::Unrecognized { + kind: "join".into() + } + ); + } + + /// Remote connections are refused before any request is made. + #[cfg(feature = "remote")] + #[tokio::test] + async fn test_remote_connection_is_refused_up_front() { + let conn = connect("db://nowhere") + .api_key("sk_test") + .region("us-east-1") + .execute() + .await + .unwrap(); + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.open_materialized_view("v").await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + let err = conn.list_materialized_views().await.unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. })); + } + + /// A definition must evaluate identically across refreshes; anything + /// less makes incremental maintenance a mix of evaluations. + #[tokio::test] + async fn test_volatile_and_unstable_expressions_are_rejected() { + let conn = people_db().await; + for expression in [ + "random()", + "now()", + "version()", + "arrow_typeof(age)", + "arrow_metadata(age, 'k')", + ] { + let err = conn + .create_materialized_view("bad", "people") + .select([("x", expression)]) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidExpression { message, .. } + if message.contains("not immutable")), + "{expression} was not rejected" + ); + } + for filter in ["age > random() * 100", "age >= 0 and now() is not null"] { + let err = conn + .create_materialized_view("bad", "people") + .only_if(filter) + .execute() + .await + .unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("not immutable")), + "{filter} was not rejected" + ); + } + } + + /// A column projected as itself stays the column it was: blob discovery + /// and the blob APIs key off field metadata, which a bare rebuild of the + /// field would drop. + #[tokio::test] + async fn test_identity_projection_keeps_field_metadata() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("id", DataType::Int32, true), + crate::blob("payload", true), + ], + HashMap::new(), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let view_schema = view.table().schema().await.unwrap(); + + let payload = view_schema.field_with_name("payload").unwrap(); + assert!( + crate::blob::is_blob(payload), + "default projection dropped the blob marker: {:?}", + payload.metadata() + ); + assert_eq!( + view.table().blob_columns().await.unwrap(), + vec!["payload".to_string()], + "blob discovery no longer finds the projected column" + ); + assert!(view_schema.metadata().contains_key(DEFINITION_META_KEY)); + // Structural declarations describe how a table is written; a view is + // written by refresh, and its fields are always nullable. + assert!(!view_schema.metadata().contains_key("lance:primary_key")); + + // A computed column is a new value and carries no source metadata. + let computed = conn + .create_materialized_view("c", "src") + .select([("payload", "payload"), ("n", "id + 1")]) + .execute() + .await + .unwrap(); + let computed_schema = computed.table().schema().await.unwrap(); + assert!(crate::blob::is_blob( + computed_schema.field_with_name("payload").unwrap() + )); + assert!( + computed_schema + .field_with_name("n") + .unwrap() + .metadata() + .is_empty() + ); + } + + /// A nested column projected straight through is still that column, and a + /// declaration buried in a struct child binds as hard as one on top. + #[tokio::test] + async fn test_nested_projection_metadata_and_declarations() { + let conn = connect("memory://").execute().await.unwrap(); + let payload = crate::blob("payload", true).with_metadata(HashMap::from([ + ("lance-encoding:blob".to_string(), "true".to_string()), + ( + "lance-schema:unenforced-primary-key".to_string(), + "0".to_string(), + ), + ])); + let schema = Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("meta", DataType::Struct(vec![payload].into()), true), + ])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + // A nested path is a direct projection: the leaf's metadata comes with + // it, so the blob stays a blob rather than a plain struct. + let lifted = conn + .create_materialized_view("lifted", "src") + .select([("payload", "meta.payload")]) + .execute() + .await + .unwrap(); + let field = lifted.table().schema().await.unwrap(); + let field = field.field_with_name("payload").unwrap().clone(); + assert_eq!( + field.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()), + "nested projection lost the leaf's metadata" + ); + assert!( + !field + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a structural declaration rode along" + ); + + // Projecting the struct whole must not carry the child's declaration + // out to a view whose fields are nullable. + let whole = conn + .create_materialized_view("whole", "src") + .select([("meta", "meta")]) + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let DataType::Struct(children) = schema.field_with_name("meta").unwrap().data_type() else { + panic!("meta is not a struct"); + }; + let child = children.iter().find(|c| c.name() == "payload").unwrap(); + assert!( + !child + .metadata() + .contains_key("lance-schema:unenforced-primary-key"), + "a nested declaration survived: {:?}", + child.metadata() + ); + assert_eq!( + child.metadata().get("lance-encoding:blob"), + Some(&"true".to_string()) + ); + } + + /// A map's entries are fields like any other, and a declaration on one + /// binds the view's writes just as hard as one on top. + #[tokio::test] + async fn test_map_declarations_are_stripped() { + let conn = connect("memory://").execute().await.unwrap(); + let value = + ArrowField::new("value", DataType::Utf8, false).with_metadata(HashMap::from([( + "lance-schema:unenforced-clustering-key:position".to_string(), + "1".to_string(), + )])); + let entries = ArrowField::new( + "entries", + DataType::Struct(vec![ArrowField::new("key", DataType::Utf8, false), value].into()), + false, + ); + let schema = Arc::new(ArrowSchema::new(vec![ArrowField::new( + "props", + DataType::Map(Arc::new(entries), false), + true, + )])); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let DataType::Map(entries, _) = schema.field_with_name("props").unwrap().data_type() else { + panic!("props is not a map"); + }; + let DataType::Struct(children) = entries.data_type() else { + panic!("map entries are not a struct"); + }; + let value = children.iter().find(|c| c.name() == "value").unwrap(); + assert!( + !value + .metadata() + .contains_key("lance-schema:unenforced-clustering-key:position"), + "a declaration survived inside a map: {:?}", + value.metadata() + ); + } + + /// A computed column is declared by field metadata. Projecting one -- + /// as itself or under an alias -- must carry its description without its + /// declaration, which the target table would reject as foreign. + #[tokio::test] + async fn test_view_over_a_computed_column() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2])).unwrap(); + let source = conn + .create_table("src", batch) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + source + .add_columns() + .computed("doubled", "id * 2") + .execute() + .await + .unwrap(); + + // Default projection reaches the computed column too. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let schema = whole.table().schema().await.unwrap(); + let field = schema.field_with_name("doubled").unwrap(); + assert!( + !field + .metadata() + .keys() + .any(|k| k.starts_with("computed_column")), + "a computed-column declaration rode along: {:?}", + field.metadata() + ); + + // And under an alias. + conn.create_materialized_view("aliased", "src") + .select([("twice", "doubled")]) + .execute() + .await + .unwrap(); + } + + /// Embedding configuration names columns. It comes along only for the + /// columns a view actually projects, under the names the view gives them. + #[tokio::test] + async fn test_embedding_config_follows_the_projection() { + let config = r#"[{"name":"f","model":{},"source_column":"text","vector_column":"vec"}]"#; + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + conn.create_empty_table("src", schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let carried = |view: &MaterializedView| { + let view = view.table().clone(); + async move { + view.schema() + .await + .unwrap() + .metadata() + .get("embedding_functions") + .cloned() + } + }; + + // Both columns projected as themselves: kept as it stands. + let whole = conn + .create_materialized_view("whole", "src") + .execute() + .await + .unwrap(); + let kept = carried(&whole).await.expect("config dropped"); + assert!(kept.contains(r#""source_column":"text""#), "{kept}"); + assert!(kept.contains(r#""vector_column":"vec""#), "{kept}"); + + // Only the source column: the configuration names a vector column the + // view does not have, so it describes nothing and goes. + let partial = conn + .create_materialized_view("partial", "src") + .select([("text", "text")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&partial).await, None); + + // Renamed: the configuration follows the names the view uses. + let renamed = conn + .create_materialized_view("renamed", "src") + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let remapped = carried(&renamed).await.expect("config dropped"); + assert!(remapped.contains(r#""source_column":"body""#), "{remapped}"); + assert!( + remapped.contains(r#""vector_column":"embedding""#), + "{remapped}" + ); + + // The Node bindings spell the same configuration in camelCase, and + // the Rust definition names the destination `dest_column`. + for (config, source_key, dest_key) in [ + ( + r#"[{"name":"f","model":{},"sourceColumn":"text","vectorColumn":"vec"}]"#, + "sourceColumn", + "vectorColumn", + ), + ( + r#"[{"name":"f","model":{},"source_column":"text","dest_column":"vec"}]"#, + "source_column", + "dest_column", + ), + ] { + let schema = Arc::new(ArrowSchema::new_with_metadata( + vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vec", DataType::Float32, true), + ], + HashMap::from([("embedding_functions".to_string(), config.to_string())]), + )); + let name = format!("src_{source_key}"); + conn.create_empty_table(&name, schema) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view(format!("v_{source_key}"), &name) + .select([("body", "text"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&view).await.expect("config dropped"); + assert!( + carried.contains(&format!(r#""{source_key}":"body""#)), + "{carried}" + ); + assert!( + carried.contains(&format!(r#""{dest_key}":"embedding""#)), + "{carried}" + ); + } + + // A computed column is not the source column under another name. + let computed = conn + .create_materialized_view("computed", "src") + .select([("body", "upper(text)"), ("embedding", "vec")]) + .execute() + .await + .unwrap(); + assert_eq!(carried(&computed).await, None); + + // One column projected twice is two columns in the view, and the + // configuration has to describe both rather than whichever came last. + let twice = conn + .create_materialized_view("twice", "src") + .select([("body", "text"), ("a", "vec"), ("b", "vec")]) + .execute() + .await + .unwrap(); + let carried = carried(&twice).await.expect("config dropped"); + let entries: Vec = serde_json::from_str(&carried).unwrap(); + let mut vectors: Vec<&str> = entries + .iter() + .filter_map(|e| e["vector_column"].as_str()) + .collect(); + vectors.sort_unstable(); + assert_eq!(vectors, ["a", "b"], "{carried}"); + } + + /// The native Rust producer records embeddings as column definitions + /// rather than as `embedding_functions`, and a query embeds its own text + /// through them. They are positional, so the view's list covers every one + /// of its fields. + #[tokio::test] + async fn test_native_column_definitions_follow_the_projection() { + let conn = connect("memory://").execute().await.unwrap(); + let rich = crate::table::TableDefinition::new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("text", DataType::Utf8, true), + ArrowField::new("vector", DataType::Float32, true), + ])), + vec![ + ColumnDefinition { + kind: ColumnKind::Physical, + }, + ColumnDefinition { + kind: ColumnKind::Embedding(EmbeddingDefinition::new( + "text", + "model", + Some("vector"), + )), + }, + ], + ) + .into_rich_schema(); + conn.create_empty_table("src", rich) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let view = conn + .create_materialized_view("view", "src") + .select([("body", "text"), ("embedding", "vector")]) + .execute() + .await + .unwrap(); + let schema = view.table().schema().await.unwrap(); + let raw = schema + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY) + .expect("the view dropped the native column definitions"); + let definitions: Vec = serde_json::from_str(raw).unwrap(); + assert_eq!( + definitions.len(), + schema.fields().len(), + "column definitions are positional" + ); + let ColumnKind::Embedding(embedding) = &definitions[1].kind else { + panic!("the embedding column came back physical: {raw}"); + }; + assert_eq!(embedding.source_column, "body"); + assert_eq!(embedding.dest_column.as_deref(), Some("embedding")); + assert_eq!(embedding.embedding_name, "model"); + assert!(matches!(definitions[0].kind, ColumnKind::Physical)); + assert!(matches!(definitions[2].kind, ColumnKind::Physical)); + + // Without the column the function reads, the view cannot recompute + // the embedding, so it carries no definition for it. + let partial = conn + .create_materialized_view("partial", "src") + .select([("embedding", "vector")]) + .execute() + .await + .unwrap(); + assert_eq!( + partial + .table() + .schema() + .await + .unwrap() + .metadata() + .get(COLUMN_DEFINITIONS_META_KEY), + None + ); + } + + /// A scan takes the cap as i64, so a larger one is refused where it is + /// declared rather than at the refresh that cannot run it. What a cap of + /// zero means is a refresh question, tested there. + #[tokio::test] + async fn test_limit_above_i64_max_is_refused_at_creation() { + let conn = people_db().await; + let err = conn + .create_materialized_view("too_big", "people") + .limit(i64::MAX as u64 + 1) + .execute() + .await + .unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("exceeds the maximum")), + "got {err:?}" + ); + + // The boundary itself is accepted. + conn.create_materialized_view("at_max", "people") + .limit(i64::MAX as u64) + .execute() + .await + .unwrap(); + } + + #[tokio::test] + async fn test_drop_is_drop_table() { + let conn = people_db().await; + conn.create_materialized_view("v", "people") + .execute() + .await + .unwrap(); + conn.drop_table("v", &[]).await.unwrap(); + assert!(conn.list_materialized_views().await.unwrap().is_empty()); + } + + /// The public declaration contract: prepare validates the source and + /// create consumes the declaration into a verified view table; a + /// source that cannot anchor refresh and a target outside the + /// source's database are both refused. + #[tokio::test] + async fn prepare_and_create_bind_the_declaration_lifecycle() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(); + let source = conn + .create_table("src", batch.clone()) + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + + let projections = [ + ("id".to_string(), "id".to_string()), + ("double".to_string(), "value * 2".to_string()), + ]; + let prepared = prepare_declaration(&source, &projections, Some("value > 0"), None) + .await + .unwrap(); + assert_eq!(prepared.definition().source_table, "src"); + + let view = prepared.create("v").await.unwrap(); + let schema = view.table().schema().await.unwrap(); + let names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect(); + assert_eq!(names, ["id", "double", SOURCE_ROW_ID_COLUMN]); + assert!(schema.metadata().contains_key(DEFINITION_META_KEY)); + + // The same call rejects a source without stable row ids, so an + // external creation path cannot skip the check. + conn.create_table("plain", batch).execute().await.unwrap(); + let plain = conn.open_table("plain").execute().await.unwrap(); + let err = prepare_declaration(&plain, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("stable row ids"), "{err}"); + + // A handle whose location does not resolve back through its name is + // refused: the definition would record a name reaching other data. + let plain_uri = plain + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + let masquerade = conn + .open_table("src") + .location(plain_uri) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&masquerade, &[], None, None) + .await + .unwrap_err(); + assert!( + err.to_string().contains("does not resolve to itself"), + "{err}" + ); + + // A table created at a custom location is refused outright: its + // recorded name reaches nothing at the database root, so the + // canonical reopen fails before any URI comparison. + let custom = conn + .create_table( + "custom_loc", + record_batch!(("id", Int32, [1, 2]), ("value", Int32, [3, 4])).unwrap(), + ) + .location("memory://elsewhere/custom_loc") + .write_options(stable_row_ids()) + .execute() + .await + .unwrap(); + let err = prepare_declaration(&custom, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("custom_loc"), "{err}"); + + // A namespaced source cannot be recorded in the definition: the + // bare name refresh resolves would reach a different table or none. + let namespaced = crate::table::NativeTable::create( + "memory://ns_src", + "ns_src", + vec!["ns".to_string()], + Box::new(arrow_array::RecordBatchIterator::new( + vec![], + std::sync::Arc::new(arrow_schema::Schema::new(vec![arrow_schema::Field::new( + "id", + arrow_schema::DataType::Int32, + true, + )])), + )) as Box, + None, + None, + None, + None, + std::collections::HashSet::new(), + ) + .await + .unwrap(); + let namespaced = Table::new(std::sync::Arc::new(namespaced), conn.database().clone()); + let err = prepare_declaration(&namespaced, &[], None, None) + .await + .unwrap_err(); + assert!(err.to_string().contains("namespaced source"), "{err}"); + } +} diff --git a/rust/lancedb/src/materialized_view/differential.rs b/rust/lancedb/src/materialized_view/differential.rs new file mode 100644 index 000000000..396b4e65b --- /dev/null +++ b/rust/lancedb/src/materialized_view/differential.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Differential refresh testing. +//! +//! The refresh contract is a property: after any sequence of source +//! mutations, a view maintained by default (incremental-where-possible) +//! refreshes equals the definition evaluated against the source directly, +//! and so does a forced rebuild. The oracle is an independent read of the +//! source -- plain column scan, filter applied in Rust -- so it shares +//! nothing with the refresh path it checks. +//! +//! The oracle runs after every step, not just at the end: a later mutation +//! that forces a rebuild would silently heal an incremental error, and those +//! transient errors are exactly the bugs this exists to catch. + +use arrow_array::{Float32Array, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; +use futures::{StreamExt, TryStreamExt}; +use lance::dataset::NewColumnTransform; +use std::sync::Arc; + +use super::MaterializedView; +use super::refresh::RefreshMode; +use crate::connect; +use crate::connection::Connection; +use crate::query::{ExecutableQuery, QueryBase, Select}; +use crate::table::{CompactionOptions, OptimizeAction, Table}; + +/// One source mutation, one per correctness-relevant class. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SrcOp { + /// Fresh non-colliding ids of both parities, so every other op has + /// view-resident rows to act on: the only op that should refresh + /// incrementally. + AppendNew, + /// Deletion in surviving fragments must break the pure-append check. + DeleteEven, + /// An in-place update; on the filtered shape it crosses the predicate, + /// so rows must leave the view. + UpdateOddScore, + /// Fragment rewrite/renumber must break the pure-append check. + Compact, + /// A column the view does not read must NOT force a rebuild. + AddColumn, + /// merge_insert commits an Update whose by-source arm deletes rows, so a + /// classifier that reads Update as "changed only" loses those deletions. + MergeDropLargest, + /// merge_insert that both changes existing rows and inserts new ones in + /// one transaction. + MergeUpsert, +} + +const ALL_OPS: [SrcOp; 7] = [ + SrcOp::AppendNew, + SrcOp::DeleteEven, + SrcOp::UpdateOddScore, + SrcOp::Compact, + SrcOp::AddColumn, + SrcOp::MergeDropLargest, + SrcOp::MergeUpsert, +]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Shape { + /// SELECT id, score. + Identity, + /// SELECT id, score WHERE score > 50: additionally sensitive to rows + /// crossing the predicate. + Filtered, + /// SELECT id, score LIMIT 4. Which rows are held depends on the order + /// they were first materialized, so the oracle checks containment and + /// the cap rather than equality. + Limited, +} + +impl Shape { + fn filter(&self) -> Option<&'static str> { + match self { + Self::Identity | Self::Limited => None, + Self::Filtered => Some("score > 50"), + } + } + + fn matches(&self, score: f32) -> bool { + match self { + Self::Identity | Self::Limited => true, + Self::Filtered => score > 50.0, + } + } + + fn limit(&self) -> Option { + match self { + Self::Limited => Some(4), + _ => None, + } + } +} + +struct Case { + conn: Connection, + source: Table, + view: MaterializedView, + shape: Shape, + next_id: i32, + added_columns: u32, +} + +fn rows_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +fn merge_batch(ids: &[i32]) -> RecordBatch { + let scores: Vec = ids.iter().map(|id| (*id * 10 + 5) as f32).collect(); + RecordBatch::try_new( + Arc::new(ArrowSchema::new(vec![ + ArrowField::new("id", DataType::Int32, true), + ArrowField::new("score", DataType::Float32, true), + ])), + vec![ + Arc::new(Int32Array::from(ids.to_vec())), + Arc::new(Float32Array::from(scores)), + ], + ) + .unwrap() +} + +impl Case { + async fn new(shape: Shape) -> Self { + let conn = connect("memory://").execute().await.unwrap(); + let source = conn + .create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let mut builder = conn + .create_materialized_view("view", "src") + .select([("id", "id"), ("score", "score")]); + if let Some(filter) = shape.filter() { + builder = builder.only_if(filter); + } + if let Some(limit) = shape.limit() { + builder = builder.limit(limit as u64); + } + let view = builder.execute().await.unwrap(); + Self { + conn, + source, + view, + shape, + next_id: 100, + added_columns: 0, + } + } + + async fn apply(&mut self, op: SrcOp) { + match op { + SrcOp::AppendNew => { + // Mixed parity: the middle id is odd, so UpdateOddScore always + // has a filter-matching appended row to evict. + let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202]; + self.next_id += 303; + self.source.add(rows_batch(&ids)).execute().await.unwrap(); + } + SrcOp::DeleteEven => { + self.source.delete("id % 2 = 0").await.unwrap(); + } + SrcOp::UpdateOddScore => { + self.source + .update() + .column("score", "-1.0") + .only_if("id % 2 = 1") + .execute() + .await + .unwrap(); + } + SrcOp::Compact => { + self.source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + SrcOp::MergeDropLargest => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + ids.pop(); + if ids.is_empty() { + return; + } + let batch = rows_batch(&ids); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::MergeUpsert => { + let mut ids = self.source_ids().await; + ids.sort_unstable(); + // One row that exists (updated in place) and one that does not. + let existing = ids.first().copied().unwrap_or(self.next_id); + let fresh = self.next_id; + self.next_id += 1; + let batch = merge_batch(&[existing, fresh]); + let reader = + arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = self.source.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + } + SrcOp::AddColumn => { + self.added_columns += 1; + let field = ArrowField::new( + format!("extra_{}", self.added_columns), + DataType::Int32, + true, + ); + self.source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![field], + )))) + .execute() + .await + .unwrap(); + } + } + } + + async fn source_ids(&self) -> Vec { + read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .map(|(id, _)| id) + .collect() + } + + /// The definition's result, read independently of the refresh path: + /// plain column scan, filter applied here, sorted. + async fn oracle(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.source + .query() + .select(Select::columns(&["id", "score"])), + ) + .await + .into_iter() + .filter(|(_, score)| self.shape.matches(*score as f32)) + .collect::>(); + rows.sort_unstable(); + rows + } + + async fn view_rows(&self) -> Vec<(i32, i32)> { + let mut rows = read_rows( + self.view + .table() + .query() + .select(Select::columns(&["id", "score"])), + ) + .await; + rows.sort_unstable(); + rows + } + + async fn check(&self, label: &str) -> Result<(), String> { + let expected = self.oracle().await; + let actual = self.view_rows().await; + let Some(cap) = self.shape.limit() else { + if expected != actual { + return Err(format!( + "{label}: view diverged from oracle\n expected: {expected:?}\n actual: {actual:?}" + )); + } + return Ok(()); + }; + // A capped view holds some subset of the definition's result, never + // more than the cap, and never the same row twice. + if actual.len() > cap { + return Err(format!( + "{label}: view holds {} rows, over its cap of {cap}: {actual:?}", + actual.len() + )); + } + let mut unique = actual.clone(); + unique.dedup(); + if unique.len() != actual.len() { + return Err(format!("{label}: view holds a row twice: {actual:?}")); + } + if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) { + return Err(format!( + "{label}: view holds {stray:?}, which the definition does not select: {expected:?}" + )); + } + // Below the cap the view must be complete, or a row was lost. + if actual.len() < cap.min(expected.len()) { + return Err(format!( + "{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}", + actual.len(), + expected.len() + )); + } + Ok(()) + } +} + +async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> { + let batches = query + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let ids = batch["id"].as_any().downcast_ref::().unwrap(); + let scores = batch["score"] + .as_any() + .downcast_ref::() + .unwrap(); + // Scores are integer-valued by construction; compare exactly. + (0..batch.num_rows()) + .map(|i| (ids.value(i), scores.value(i) as i32)) + .collect::>() + }) + .collect() +} + +/// Drive one mutation sequence: refresh + oracle-check after every step, +/// then a forced rebuild checked against the same oracle. +async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> { + let label = format!("{shape:?} {ops:?}"); + let mut case = Case::new(shape).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: initial refresh failed: {e}"))?; + case.check(&format!("{label} (initial)")).await?; + + for (step, op) in ops.iter().enumerate() { + case.apply(*op).await; + case.view + .refresh() + .execute() + .await + .map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?; + case.check(&format!("{label} (step {step}, {op:?})")) + .await?; + } + + case.view + .refresh() + .full(true) + .execute() + .await + .map_err(|e| format!("{label}: final full refresh failed: {e}"))?; + case.check(&format!("{label} (final rebuild)")).await?; + // Silence the unused-connection lint without dropping it mid-case. + let _ = &case.conn; + Ok(()) +} + +/// Every op sequence up to `max_len`. +fn all_sequences(max_len: u32) -> Vec> { + let mut sequences = Vec::new(); + for len in 1..=max_len { + for mut index in 0..ALL_OPS.len().pow(len) { + let mut ops = Vec::with_capacity(len as usize); + for _ in 0..len { + ops.push(ALL_OPS[index % ALL_OPS.len()]); + index /= ALL_OPS.len(); + } + sequences.push(ops); + } + } + sequences +} + +async fn run_exhaustive(max_len: u32) { + let mut cases = Vec::new(); + for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] { + for ops in all_sequences(max_len) { + cases.push((ops, shape)); + } + } + let failures: Vec = futures::stream::iter(cases) + .map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() }) + .buffer_unordered(8) + .filter_map(|failure| async move { failure }) + .collect() + .await; + assert!( + failures.is_empty(), + "{} sequences diverged; first: {}", + failures.len(), + failures[0] + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn differential_exhaustive() { + run_exhaustive(3).await; +} + +#[tokio::test(flavor = "multi_thread")] +#[ignore = "longer sweep; run manually"] +async fn differential_exhaustive_deep() { + run_exhaustive(4).await; +} + +/// Named interleavings that double as repro handles. The mode assertions pin +/// the classifier, which value comparison alone cannot: a wrongly rebuilt +/// view still matches the oracle. +#[tokio::test(flavor = "multi_thread")] +async fn differential_named_regressions() { + // An append is the one op that must stay incremental. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + case.check("append stays incremental").await.unwrap(); + + // A column the view does not read must not force a rebuild. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AddColumn).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + // Compaction rearranges rows without changing them: the watermark + // advances and nothing rebuilds. + let mut case = Case::new(Shape::Identity).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + case.apply(SrcOp::Compact).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + case.check("compaction alone").await.unwrap(); + + // Fragment bookkeeping stays coherent across the compaction: the next + // append is separable and computed alone. + case.apply(SrcOp::AppendNew).await; + let result = case.view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 3); + case.check("compact then append").await.unwrap(); + + // A row updated to no longer match the filter must leave the view -- + // and the fixture must prove the eviction happened, not merely that the + // end state matches: an update that never touched a view-resident row + // would also "match". + let mut case = Case::new(Shape::Filtered).await; + case.apply(SrcOp::AppendNew).await; + case.view.refresh().execute().await.unwrap(); + let before = case.view_rows().await.len(); + case.apply(SrcOp::UpdateOddScore).await; + case.view.refresh().execute().await.unwrap(); + let after = case.view_rows().await.len(); + assert!( + after < before, + "no view-resident row was evicted ({before} -> {after}); the fixture \ + no longer exercises the filtered-update transition" + ); + case.check("update crosses the filter").await.unwrap(); +} + +// --------------------------------------------------------------------------- +// Concurrency +// --------------------------------------------------------------------------- +// +// The sequential cases above cannot observe a cross-process race: the +// per-view refresh lock is process-local, so a second refresh in this +// process queues behind the first. What is missing is not more op +// sequences but a second process. These cases add one, and assert the same +// property the harness always asserts -- the view holds each row once. + +/// Rows the definition selects from the source: every id but the first, +/// read straight from the source, sharing nothing with the refresh path. +async fn concurrency_oracle(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("src") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + if column.value(i) > 1 { + ids.push(column.value(i)); + } + } + } + ids.sort_unstable(); + ids +} + +/// The view's ids, sorted. +async fn concurrency_view_ids(conn: &Connection) -> Vec { + let batches: Vec = conn + .open_table("mv") + .execute() + .await + .unwrap() + .query() + .select(Select::columns(&["id"])) + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let mut ids = Vec::new(); + for batch in &batches { + let column = batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + for i in 0..batch.num_rows() { + ids.push(column.value(i)); + } + } + ids.sort_unstable(); + ids +} + +/// One refresh of the view at `MV_RACE_DIR`, in its own process. +/// +/// Setup happens before the start barrier so warm-up does not stagger the +/// two processes. What makes the race certain rather than likely is the +/// second barrier inside `refresh()` itself, which holds every participant +/// between staging and commit. +#[tokio::test] +#[ignore = "spawned as a child process by the concurrency cases"] +async fn cross_process_refresh_child() { + let Ok(dir) = std::env::var("MV_RACE_DIR") else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let tag = std::env::var("MV_RACE_TAG").unwrap(); + + let conn = connect(dir.to_str().unwrap()).execute().await.unwrap(); + let table = conn.open_table("mv").execute().await.unwrap(); + let _ = table.schema().await.unwrap(); + let _ = table.count_rows(None).await.unwrap(); + let source = conn.open_table("src").execute().await.unwrap(); + let _ = source.count_rows(None).await.unwrap(); + let view = MaterializedView::from_table(table).await.unwrap(); + + std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap(); + while !dir.join("START").exists() { + std::thread::sleep(std::time::Duration::from_millis(2)); + } + + let outcome = match view.refresh().execute().await { + Ok(result) => format!("committed rows={}", result.rows_written), + Err(err) if is_commit_conflict(&err) => "conflicted".to_string(), + Err(err) => format!("failed {err}"), + }; + std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap(); +} + +/// Whether a refresh lost its commit to a concurrent one, as opposed to +/// failing for any other reason. +fn is_commit_conflict(err: &crate::Error) -> bool { + let text = err.to_string(); + text.contains("Retryable commit conflict") || text.contains("preempted by concurrent") +} + +/// Two processes refreshing one view concurrently must leave the view +/// equal to the oracle: each selected row present exactly once. +/// +/// Both plan the same incremental delta from one watermark. A refresh is +/// meant to land on the generation it planned or leave nothing behind, so +/// at most one of them may write. +#[tokio::test] +async fn concurrent_refreshes_hold_each_row_once() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap().to_string(); + let conn = connect(&path).execute().await.unwrap(); + conn.create_table("src", rows_batch(&[1, 2, 3, 4])) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("mv", "src") + .select([("id", "id"), ("score", "score")]) + .only_if("id > 1") + .execute() + .await + .unwrap(); + // Seed the watermark so the racing refreshes are both incremental. + view.refresh().execute().await.unwrap(); + + // Large enough that a refresh is real work rather than a formality. + let ids: Vec = (100..200_100).collect(); + conn.open_table("src") + .execute() + .await + .unwrap() + .add(rows_batch(&ids)) + .execute() + .await + .unwrap(); + + let tags = ["a", "b"]; + let exe = std::env::current_exe().unwrap(); + let children: Vec = tags + .iter() + .map(|tag| { + std::process::Command::new(&exe) + .args([ + "--exact", + "materialized_view::differential::cross_process_refresh_child", + "--ignored", + "--nocapture", + ]) + .env("MV_RACE_DIR", dir.path()) + .env("MV_RACE_SYNC", dir.path()) + .env("MV_RACE_PEERS", "2") + .env("MV_RACE_TAG", tag) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .unwrap() + }) + .collect(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180); + while tags + .iter() + .any(|tag| !dir.path().join(format!("ready-{tag}")).exists()) + { + assert!( + std::time::Instant::now() < deadline, + "children never became ready" + ); + std::thread::sleep(std::time::Duration::from_millis(10)); + } + std::fs::write(dir.path().join("START"), b"1").unwrap(); + for (tag, mut child) in tags.iter().zip(children) { + let status = loop { + match child.try_wait().unwrap() { + Some(status) => break status, + None if std::time::Instant::now() >= deadline => { + child.kill().unwrap(); + panic!("child {tag} never finished"); + } + None => std::thread::sleep(std::time::Duration::from_millis(10)), + } + }; + assert!(status.success(), "child {tag} exited {status}"); + } + + // Both refreshes reached the commit boundary before either committed -- + // the in-refresh barrier guarantees it -- so exactly one may win. + let outcomes: Vec = tags + .iter() + .map(|tag| { + std::fs::read_to_string(dir.path().join(format!("outcome-{tag}"))) + .unwrap_or_else(|_| panic!("child {tag} recorded no outcome")) + }) + .collect(); + for tag in tags { + assert!( + dir.path().join(format!("planned-{tag}")).exists(), + "child {tag} never reached the commit boundary, so nothing was synchronized" + ); + } + let committed = outcomes.iter().filter(|o| o.contains("committed")).count(); + let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count(); + assert_eq!( + (committed, conflicted), + (1, 1), + "exactly one refresh may win the generation both planned: {outcomes:?}" + ); + + let expected = concurrency_oracle(&conn).await; + let actual = concurrency_view_ids(&conn).await; + assert_eq!( + actual.len(), + expected.len(), + "the view holds {} rows, the oracle {}: a losing refresh left rows behind", + actual.len(), + expected.len() + ); + assert_eq!(actual, expected, "the view does not match the oracle"); +} diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs new file mode 100644 index 000000000..735751c27 --- /dev/null +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -0,0 +1,2996 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Refreshing materialized views. +//! +//! A refresh pins one source version and brings the view to exactly the +//! definition's result at that version: added rows are computed and appended, +//! removed or changed rows are evicted by provenance id and recomputed in the +//! same pass. Compaction outputs cost nothing, which is only sound while +//! [`SOURCE_ROW_ID_COLUMN`] stays valid across the rewrite. Anything the +//! classifier cannot prove intact rebuilds; an indexed rebuild swaps all +//! fragments in one commit that retains index definitions. +//! +//! The watermark ([`SOURCE_VERSION_META_KEY`]) lands in a follow-up commit; a +//! crash or race between the two leaves the view visibly unstamped and the +//! next refresh rebuilds. In-process refreshes serialize on a per-view lock; +//! across processes the commit's inserted-rows filter carries a shared token, +//! so two refreshes of one view conflict and only one lands. + +use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex as StdMutex, OnceLock}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use arrow_array::cast::AsArray; +use arrow_array::types::UInt64Type; +use arrow_array::{RecordBatch, UInt64Array}; +use arrow_schema::{Schema as ArrowSchema, SchemaRef}; +use datafusion::common::ScalarValue; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::prelude::{col, lit}; +use futures::{StreamExt, TryStreamExt}; +use lance::Dataset; +use lance::dataset::mem_wal::DatasetMemWalExt; +use lance::dataset::transaction::{Operation, Transaction}; +use lance::dataset::write::delete::DeleteBuilder; +use lance::dataset::write::merge_insert::inserted_rows::{ + KeyExistenceFilter, KeyExistenceFilterBuilder, KeyValue, +}; +use lance::dataset::{CommitBuilder, InsertBuilder, WriteDestination, WriteMode, WriteParams}; +use lance_core::{ROW_CREATED_AT_VERSION, ROW_ID, ROW_LAST_UPDATED_AT_VERSION}; +use lance_file::version::ConcreteFileVersion; +use lance_table::format::Fragment; +use serde::{Deserialize, Serialize}; + +use super::{ + INCARNATION_META_KEY, MaterializedViewDefinition, REFRESHED_AT_MS_META_KEY, + SOURCE_ROW_ID_COLUMN, SOURCE_VERSION_META_KEY, +}; +use crate::database::OpenTableRequest; +use crate::table::{NativeTable, NativeTableExt, Table}; +use crate::{Error, Result}; + +/// How a refresh brought the view up to date. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RefreshMode { + /// The view was recomputed from scratch. + Rebuild, + /// Rows from source fragments added since the last refresh were appended. + Incremental, + /// The view was already at the requested source version. + NoOp, +} + +/// The result of refreshing a materialized view. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RefreshMaterializedViewResult { + /// How the view was brought up to date. + pub mode: RefreshMode, + /// Rows written to the view: everything on a rebuild, and on an + /// incremental refresh both the rows added and the rows recomputed in + /// place of ones the source changed. + pub rows_written: u64, + /// The source table version the view now reflects. + pub source_version: u64, + /// The view table version after the refresh. + pub version: u64, +} + +/// Schema metadata key holding the view table version a successful refresh +/// left behind. Any other commit on the view is drift, and refresh rebuilds. +pub const VIEW_VERSION_META_KEY: &str = "mv.view_version"; + +/// Schema metadata key holding the commit timestamp of the watermark's source +/// manifest. A dropped and recreated source reuses version numbers but never +/// their timestamps, so a mismatch means the watermark describes a different +/// incarnation and refresh rebuilds. +pub const SOURCE_VERSION_TS_META_KEY: &str = "mv.source_version_ts"; + +/// One refresh per view at a time within this process. +fn refresh_lock(uri: &str) -> Arc> { + static LOCKS: OnceLock>>>> = + OnceLock::new(); + LOCKS + .get_or_init(Default::default) + .lock() + .expect("refresh lock registry poisoned") + .entry(uri.to_string()) + .or_default() + .clone() +} + +/// Internal implementation of the refresh logic. +pub(crate) async fn execute_refresh( + view: &Table, + full: bool, + pinned: Option, + expected_incarnation: Option<&str>, +) -> Result { + let view_native = view.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + view_native.dataset.ensure_mutable()?; + let lock = refresh_lock(view_native.dataset.get().await?.uri()); + let _guard = lock.lock().await; + // Force-load the latest view state under the lock: each handle caches + // lazily, and a second handle would otherwise plan from a snapshot taken + // before another handle's commit -- appending the same rows again or + // reporting NoOp over a mutated view. + view_native.dataset.reload().await?; + let view_ds = view_native.dataset.get().await?.as_ref().clone(); + + ensure_incarnation(&view_ds, expected_incarnation, view.name()).await?; + + // The definition a handle cached at open may since have been replaced; + // what refresh executes and what it stamps must be one generation. + let definition = match super::materialized_view_kind(&view_ds.schema().metadata)? { + Some(super::MaterializedViewKind::Select(definition)) => definition, + Some(super::MaterializedViewKind::Unrecognized { kind }) => { + return Err(Error::NotSupported { + message: format!( + "materialized view '{}' is defined by '{kind}', which this \ + version of lancedb cannot refresh", + view.name() + ), + }); + } + None => { + return Err(Error::NotAMaterializedView { + name: view.name().to_string(), + }); + } + }; + let definition = &definition; + ensure_no_mem_wal(&view_ds, "materialized view", view.name()).await?; + + let source_ds = open_source(view, definition).await?; + let source_ds = match pinned { + Some(version) => source_ds.checkout_version(version).await?, + None => source_ds, + }; + ensure_no_mem_wal(&source_ds, "source table", &definition.source_table).await?; + let source_version = source_ds.version().version; + let source_ts = source_ds.manifest.timestamp_nanos; + + // Re-plan the persisted definition against the current source schema and + // require its planned output to be exactly the view's physical schema: a + // definition the stored table cannot represent must not be certified. + let source_schema = Arc::new(ArrowSchema::from(source_ds.schema())); + let projections: Vec<(String, String)> = definition + .projections + .iter() + .map(|p| (p.output.clone(), p.expression.clone())) + .collect(); + validate_inputs(&source_ds, definition)?; + let (replanned, mut planned_fields, _renames) = super::plan( + source_schema, + &definition.source_table, + &projections, + definition.filter.as_deref(), + definition.limit, + )?; + planned_fields.push(arrow_schema::Field::new( + SOURCE_ROW_ID_COLUMN, + arrow_schema::DataType::UInt64, + false, + )); + let physical = ArrowSchema::from(view_ds.schema()); + let planned_shape: Vec<_> = planned_fields + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + let physical_shape: Vec<_> = physical + .fields() + .iter() + .map(|f| (f.name().clone(), f.data_type().clone(), f.is_nullable())) + .collect(); + if planned_shape != physical_shape { + return Err(Error::Schema { + message: format!( + "the stored definition of view '{}' does not produce this \ + view's schema; recreate the view", + view.name() + ), + }); + } + let definition = &replanned; + + let metadata = &view_ds.schema().metadata; + let watermark: Option = metadata + .get(SOURCE_VERSION_META_KEY) + .and_then(|raw| raw.parse().ok()); + let recorded_ts: Option = metadata + .get(SOURCE_VERSION_TS_META_KEY) + .and_then(|raw| raw.parse().ok()); + // The watermark speaks only for the view state its refresh left behind; + // any other commit on the view since then is drift. + let view_intact = metadata + .get(VIEW_VERSION_META_KEY) + .and_then(|raw| raw.parse::().ok()) + == Some(view_ds.version().version); + + if !full && watermark == Some(source_version) && view_intact && recorded_ts == Some(source_ts) { + return Ok(RefreshMaterializedViewResult { + mode: RefreshMode::NoOp, + rows_written: 0, + source_version, + version: view_ds.version().version, + }); + } + + let watermark = watermark.filter(|_| view_intact); + match plan_increment( + &source_ds, + source_version, + watermark, + recorded_ts, + full, + definition, + ) + .await + { + Some(increment) => { + let reconciled = incremental( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + increment, + definition, + watermark, + expected_incarnation, + ) + .await?; + match reconciled { + Some(result) => Ok(result), + // The delta was too large to reconcile in bounded memory. + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + expected_incarnation, + ) + .await + } + } + } + None => { + rebuild( + view_native, + &view_ds, + &source_ds, + source_version, + source_ts, + definition, + expected_incarnation, + ) + .await + } + } +} + +/// The source fragments whose rows are new since the watermark, or `None` +/// where the view has to rebuild. Two tiers: the transaction walk is exact +/// where it applies; the fragment-signature check is the fallback for deltas +/// the walk cannot read, and under it any fragment churn rebuilds. +async fn plan_increment( + source_ds: &Dataset, + source_version: u64, + watermark: Option, + recorded_ts: Option, + full: bool, + definition: &MaterializedViewDefinition, +) -> Option { + if full { + return None; + } + let watermark = watermark?; + if watermark > source_version { + return None; + } + let old = source_ds.checkout_version(watermark).await.ok()?; + // A recreated source reuses version numbers, never their timestamps: a + // mismatch means the watermark describes a different incarnation. + if recorded_ts != Some(old.manifest.timestamp_nanos) { + return None; + } + let old_ids: HashSet = old.get_fragments().iter().map(|f| f.id() as u64).collect(); + let live: Vec = source_ds + .get_fragments() + .iter() + .map(|f| f.metadata().clone()) + .collect(); + + if let Some(delta) = appends_and_rewrites(source_ds, watermark, source_version).await { + // A rewrite that consumed a fragment neither present at the watermark + // nor produced by an earlier rewrite swallowed a mid-delta append; + // its rows cannot be told apart from already-materialized ones. + let folded = delta + .rewritten + .iter() + .any(|id| !old_ids.contains(id) && !delta.produced.contains(id)); + if folded { + return None; + } + // An update rewrites a whole fragment. If it touched one the watermark + // never saw, that fragment holds rows appended since -- new rows the + // update did not change, which the recompute does not cover and which + // this fragment's exclusion from the append set would drop. + if delta + .updated_in_place + .iter() + .any(|id| !old_ids.contains(id)) + { + return None; + } + // Rows past the cap left every delta when the watermark advanced, so + // a capped view cannot reconcile a removal incrementally. The rebuild + // is cheap for the same reason it is capped: the scan stops there. + if definition.limit.is_some() && (delta.deleted_rows || delta.updated_rows) { + return None; + } + // Legacy storage cannot serve the row-version columns update + // discovery scans; deletes and appends need none of them. + if delta.updated_rows + && source_ds.manifest.data_storage_format.lance_file_format() == ConcreteFileVersion::V1 + { + return None; + } + // Every other fragment new at head is an append: appends and rewrites + // are the only operations in the delta that add fragments, and the + // rewrite outputs are already-materialized rows rearranged. + return Some(Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id) && !delta.produced.contains(&f.id)) + .collect(), + evict_deleted: delta.deleted_rows, + replace_updated: delta.updated_rows, + }); + } + + is_pure_append(&old, source_ds, &relevant_field_ids(source_ds, definition)).then(|| Increment { + appended: live + .into_iter() + .filter(|f| !old_ids.contains(&f.id)) + .collect(), + evict_deleted: false, + replace_updated: false, + }) +} + +/// Version gap beyond which per-version transaction reads stop being cheaper +/// than one fragment scan. +const MAX_TRANSACTION_WALK: u64 = 512; + +/// Fragment ids moved by the `Rewrite` operations of the delta. Only rewrite +/// ids are real in transaction files (an Append's are placeholders assigned +/// at commit), so appends are derived as new-at-head minus rewrite outputs. +/// What an incremental refresh must do to bring the view up to date. +struct Increment { + /// Source fragments whose rows are not in the view yet. + appended: Vec, + /// Source rows left the range, so the view holds rows to evict. + evict_deleted: bool, + /// Source rows changed in the range, so the view holds rows to replace. + replace_updated: bool, +} + +struct TxnDelta { + /// Fragments consumed by `Rewrite` operations. + rewritten: HashSet, + /// Fragments produced by `Rewrite` operations. + produced: HashSet, + /// The delta removed source rows, so the view holds rows to evict. + deleted_rows: bool, + /// The delta changed source rows in place, so the view holds rows to + /// recompute. + updated_rows: bool, + /// Fragments an update modified in place, as opposed to produced. + updated_in_place: HashSet, +} + +/// Read the delta from the transaction log, `None` where it holds anything +/// but appends and rewrites or cannot be read; `None` only sends the caller +/// to a slower check. `ReserveFragments` moves no rows and rides along. +async fn appends_and_rewrites(cur: &Dataset, from: u64, to: u64) -> Option { + if to <= from || to - from > MAX_TRANSACTION_WALK { + return None; + } + let mut delta = TxnDelta { + rewritten: HashSet::new(), + produced: HashSet::new(), + deleted_rows: false, + updated_rows: false, + updated_in_place: HashSet::new(), + }; + for version in (from + 1)..=to { + let Ok(Some(txn)) = cur.read_transaction_by_version(version).await else { + return None; + }; + match txn.operation { + Operation::Append { .. } | Operation::ReserveFragments { .. } => {} + // A delete removes source rows without changing the ones that + // remain, so the view's other rows stay valid: the refresh + // evicts exactly the ids that left. + Operation::Delete { .. } => delta.deleted_rows = true, + // Update outputs carry no new rows, so they are excluded from + // the append set like rewrite outputs; the changed rows are + // replaced individually below. + Operation::Update { + removed_fragment_ids, + new_fragments, + updated_fragments, + .. + } => { + delta.updated_rows = true; + // merge_insert reaches here too, and its by-source arm deletes + // rows rather than changing them. + delta.deleted_rows = true; + delta.rewritten.extend(removed_fragment_ids.iter().copied()); + // Only pre-existing fragment ids are real here; created ones + // are placeholders. Rewritten rows are excluded by creation + // version below, not by fragment identity. + delta + .produced + .extend(updated_fragments.iter().map(|f| f.id)); + delta + .updated_in_place + .extend(updated_fragments.iter().map(|f| f.id)); + let _ = new_fragments; + } + Operation::Rewrite { groups, .. } => { + for group in groups { + delta + .rewritten + .extend(group.old_fragments.iter().map(|f| f.id)); + delta + .produced + .extend(group.new_fragments.iter().map(|f| f.id)); + } + } + _ => return None, + } + } + Some(delta) +} + +/// Fallback pure-append check: every old fragment still present with an +/// identical signature over the columns the view reads. Compaction, deletes +/// and updates each break it and force a rebuild; a change to a column the +/// view does not read leaves it alone, which is what lets this tier pass +/// deltas the transaction walk cannot. +fn is_pure_append(old: &Dataset, cur: &Dataset, relevant: &HashSet) -> bool { + let signature = |fragment: &lance::dataset::fragment::FileFragment| { + fragment_signature(fragment.metadata(), relevant) + }; + let current: HashSet<(u64, String)> = cur.get_fragments().iter().map(signature).collect(); + old.get_fragments() + .iter() + .all(|fragment| current.contains(&signature(fragment))) +} + +/// A fragment's identity as the view observes it: data files and overlays +/// touching the columns it reads, plus the deletion file. Overlays change no +/// file path, so they must be part of the signature. +fn fragment_signature(metadata: &Fragment, relevant: &HashSet) -> (u64, String) { + let touches_relevant = + |fields: &[i32]| relevant.is_empty() || fields.iter().any(|id| relevant.contains(id)); + let mut files: Vec<&str> = metadata + .files + .iter() + .filter(|file| touches_relevant(&file.fields)) + .map(|file| file.path.as_str()) + .collect(); + files.sort_unstable(); + let mut overlays: Vec = metadata + .overlays + .iter() + .filter(|overlay| touches_relevant(&overlay.data_file.fields)) + .map(|overlay| format!("{}@{}", overlay.data_file.path, overlay.committed_version)) + .collect(); + overlays.sort_unstable(); + ( + metadata.id, + format!( + "{}|{}|{:?}", + files.join(","), + overlays.join(","), + metadata.deletion_file + ), + ) +} + +/// Field ids (with struct descendants) of the source columns the view reads. +fn relevant_field_ids(source: &Dataset, definition: &MaterializedViewDefinition) -> HashSet { + fn collect(field: &lance_core::datatypes::Field, ids: &mut HashSet) { + ids.insert(field.id); + for child in &field.children { + collect(child, ids); + } + } + let mut ids = HashSet::new(); + for input in &definition.inputs { + if let Some(field) = source.schema().field(input) { + collect(field, &mut ids); + } + } + ids +} + +/// Error if a column the view reads no longer exists in the source. +fn validate_inputs(source: &Dataset, definition: &MaterializedViewDefinition) -> Result<()> { + for input in &definition.inputs { + if source.schema().field(input).is_none() { + return Err(Error::Schema { + message: format!( + "source column '{input}' read by the view no longer exists \ + (dropped or renamed in '{}')", + definition.source_table + ), + }); + } + } + Ok(()) +} + +/// Reject MemWAL/LSM state on a refresh participant: un-compacted tiers are +/// invisible to the fragment-planned refresh scan. An active write spec and +/// retained rows both disqualify; shard directories on storage are the +/// durable evidence of the latter. +pub(crate) async fn ensure_no_mem_wal(dataset: &Dataset, role: &str, name: &str) -> Result<()> { + let retained = !dataset.list_mem_wal_latest_shard_ids().await?.is_empty(); + if retained || dataset.mem_wal_index_details().await?.is_some() { + return Err(Error::NotSupported { + message: format!( + "{role} '{name}' has an LSM write spec or retained un-compacted \ + rows: rows in un-compacted tiers are invisible to refresh" + ), + }); + } + Ok(()) +} + +async fn open_source(view: &Table, definition: &MaterializedViewDefinition) -> Result { + let database = view.database_opt().ok_or_else(|| Error::InvalidInput { + message: "the view was not opened through a database connection".into(), + })?; + let source = database + .open_table(OpenTableRequest { + name: definition.source_table.clone(), + namespace_path: Vec::new(), + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await?; + let native = source.as_native().ok_or_else(|| Error::NotSupported { + message: "materialized views are supported only on local tables".into(), + })?; + let dataset = native.dataset.get().await?.as_ref().clone(); + if !dataset.manifest.uses_stable_row_ids() { + return Err(Error::InvalidInput { + message: format!( + "source table '{}' does not have stable row ids; it is not the \ + table this view was declared over", + definition.source_table + ), + }); + } + Ok(dataset) +} + +#[allow(clippy::too_many_arguments)] +async fn incremental( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + increment: Increment, + definition: &MaterializedViewDefinition, + watermark: Option, + expected_incarnation: Option<&str>, +) -> Result> { + let new_fragments = increment.appended; + let watermark_version = watermark.unwrap_or(0); + // Provenance ids this refresh removes: dropped rows, plus changed rows + // recomputed in the same commit. One view row per source row makes the + // eviction exact. Staged, not committed: removals ride with the rows + // that replace them, so a reader never sees the view without either. + let mut eviction = Eviction::new(view_ds, EVICTION_CHUNK); + let mut updated_rows = false; + if (increment.evict_deleted || increment.replace_updated) + && let Some(watermark) = watermark + { + let delta = source_ds + .delta() + .with_begin_version(watermark) + .with_end_version(source_version) + .build()?; + // Reconciling holds the delta's provenance ids in staged deletion + // vectors; past the cap, the streamed rebuild is the bounded path. + let cap = eviction_rebuild_cap(); + let mut evicted = 0usize; + if increment.evict_deleted { + let mut stream = delta.get_deleted_row_ids().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + eviction.push(ids).await?; + } + } + if increment.replace_updated { + // Ids only: `get_updated_rows` carries every column of every + // updated row, and discovery needs none of them. + let mut scanner = source_ds.scan(); + scanner.with_row_id().project(&[ROW_CREATED_AT_VERSION])?; + // A fixed bound: the configured default could make one discovery + // batch arbitrarily large before the fallback cap is consulted. + scanner.batch_size(8192); + scanner.filter(&format!( + "{ROW_CREATED_AT_VERSION} <= {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} > {watermark} + AND {ROW_LAST_UPDATED_AT_VERSION} <= {source_version}" + ))?; + let mut stream = scanner.try_into_stream().await?; + while let Some(batch) = stream.try_next().await? { + let ids = row_ids_of(&batch)?; + evicted += ids.len(); + if evicted > cap { + return Ok(None); + } + updated_rows |= !ids.is_empty(); + eviction.push(ids).await?; + } + } + } + let eviction = eviction.finish().await?; + + // The cap counts rows already materialized, in first-materialized order. + let remaining = match definition.limit { + Some(limit) => { + let held = view_ds.count_rows(None).await? as u64; + Some(limit.saturating_sub(held)) + } + None => None, + }; + + let mut result = RefreshMaterializedViewResult { + mode: RefreshMode::Incremental, + rows_written: 0, + source_version, + version: view_ds.version().version, + }; + let nothing_to_add = (new_fragments.is_empty() && !updated_rows) || remaining == Some(0); + if nothing_to_add && eviction.is_none() { + result.version = stamp_watermark( + view_native, + view_ds.clone(), + source_version, + source_ts, + expected_incarnation, + ) + .await?; + return Ok(Some(result)); + } + // Rows left but none arrive: the removals still have to be published. + if nothing_to_add { + let filter = refresh_filter(&empty_keys(view_ds)?)?; + let published = publish( + view_ds, + eviction, + Vec::new(), + Some(filter), + expected_incarnation, + ) + .await?; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; + return Ok(Some(result)); + } + + // Appends carry the view's schema as it stands; the watermark moves in a + // follow-up commit (see the module docs for the crash window). + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let rows_written = Arc::new(AtomicU64::new(0)); + // compute_stream counts what it produces; the truncation below can drop + // some of that, so the written count comes from the tee instead. + let computed = Arc::new(AtomicU64::new(0)); + let mut stream = compute_stream( + source_ds, + definition, + RowScope { + fragments: Some(new_fragments), + // An update rewrites whole fragments, so a fragment new at head + // can hold rows the view already has. Their creation version + // does not change, so it -- not fragment identity -- says which + // rows are new. + created_after: increment.replace_updated.then_some(watermark_version), + limit: remaining, + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + + // The updated rows' current values, computed the same way and appended + // in the same commit as the new fragments' rows. + if updated_rows { + let recomputed = compute_stream( + source_ds, + definition, + RowScope { + updated_between: Some((watermark_version, source_version)), + ..Default::default() + }, + schema.clone(), + computed.clone(), + ) + .await?; + stream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + recomputed.chain(stream), + )); + } + + // Nothing survived the filter: the watermark still has to advance or the + // same fragments would be rescanned forever, but any removals still do. + let Some(first) = stream.try_next().await? else { + let published = if eviction.is_some() { + publish( + view_ds, + eviction, + Vec::new(), + Some(refresh_filter(&empty_keys(view_ds)?)?), + expected_incarnation, + ) + .await? + } else { + view_ds.clone() + }; + result.version = stamp_watermark( + view_native, + published, + source_version, + source_ts, + expected_incarnation, + ) + .await?; + return Ok(Some(result)); + }; + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter([Ok(first)]).chain(stream), + )); + + // Any two refreshes of one view must not both commit: the filter's + // shared token makes lance reject the loser on key overlap however + // their planned rows relate. + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), rows_written.clone()); + + let ds = Arc::new(view_ds.clone()); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's new rows".into(), + }); + }; + let filter = refresh_filter(&keys)?; + let appended = publish( + view_ds, + eviction, + new_fragments, + Some(filter), + expected_incarnation, + ) + .await?; + result.rows_written = rows_written.load(Ordering::Relaxed); + result.version = stamp_watermark( + view_native, + appended, + source_version, + source_ts, + expected_incarnation, + ) + .await?; + Ok(Some(result)) +} + +async fn rebuild( + view_native: &NativeTable, + view_ds: &Dataset, + source_ds: &Dataset, + source_version: u64, + source_ts: u128, + definition: &MaterializedViewDefinition, + expected_incarnation: Option<&str>, +) -> Result { + let rows_written = Arc::new(AtomicU64::new(0)); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let stream = compute_stream( + source_ds, + definition, + RowScope { + limit: definition.limit, + ..Default::default() + }, + schema, + rows_written.clone(), + ) + .await?; + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(view_ds)?, + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + // Every rebuild is one fragment swap, indexed or not: an Update commit + // carries no schema metadata, so it cannot erase a definition update + // that raced in the way an overwrite (which adopts its stream's schema) + // durably would -- and it must land on the planned generation or abort. + let replaced = + replace_retaining_indices(view_ds.clone(), stream, keys, expected_incarnation).await?; + let version = stamp_watermark( + view_native, + replaced, + source_version, + source_ts, + expected_incarnation, + ) + .await?; + Ok(RefreshMaterializedViewResult { + mode: RefreshMode::Rebuild, + rows_written: rows_written.load(Ordering::Relaxed), + source_version, + version, + }) +} + +/// Replace all of the view's data in one commit that retains its index +/// definitions: new fragments staged uncommitted, one `Update` removing every +/// old fragment. `Update` prunes index bitmaps only for modified fields and +/// none are modified here, so readers never see the view unindexed or empty. +async fn replace_retaining_indices( + view_ds: Dataset, + stream: SendableRecordBatchStream, + keys: Arc>, + expected_incarnation: Option<&str>, +) -> Result { + let ds = Arc::new(view_ds); + let read_version = ds.version().version; + #[cfg(test)] + tests::hold_before_publish(ds.uri()).await; + ensure_incarnation(&ds, expected_incarnation, ds.uri()).await?; + let removed_fragment_ids: Vec = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); + + let write_txn = InsertBuilder::new(WriteDestination::Dataset(ds.clone())) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await?; + let Operation::Append { + fragments: new_fragments, + } = write_txn.operation + else { + return Err(Error::Runtime { + message: "expected an append when staging the view's replacement rows".into(), + }); + }; + + // Built only now: the tee fills as the staging drains the stream. + let filter = refresh_filter(&keys)?; + let transaction = Transaction::new( + read_version, + Operation::Update { + removed_fragment_ids, + updated_fragments: Vec::new(), + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + // Two refreshes that materialized the same source rows must not + // both land -- a raced first rebuild would double the view. + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + ); + let committed = CommitBuilder::new(WriteDestination::Dataset(ds)) + .execute(transaction) + .await?; + if committed.version().version != read_version + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); the + refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// Record that the view now reflects `source_version`, including the view +/// Refuse to act on a view that is not `expected`'s incarnation, judged from +/// the latest stored manifest. Not a commit condition; see +/// `RefreshMaterializedViewBuilder::expect_incarnation`. +async fn ensure_incarnation(view_ds: &Dataset, expected: Option<&str>, what: &str) -> Result<()> { + let Some(expected) = expected else { + return Ok(()); + }; + let mut latest = view_ds.clone(); + latest.checkout_latest().await?; + match latest.schema().metadata.get(INCARNATION_META_KEY) { + Some(actual) if actual == expected => Ok(()), + Some(_) => Err(Error::Runtime { + message: format!( + "materialized view '{what}' is not the incarnation this refresh was \ + requested for: it was dropped and recreated" + ), + }), + None => Err(Error::Runtime { + message: format!( + "materialized view '{what}' carries no incarnation token: its schema \ + metadata was replaced since the token was captured" + ), + }), + } +} + +/// version this very commit produces. The version is predicted and then +/// verified; on a mismatch another commit raced in between, and the stamp +/// ABORTS rather than certify that commit as the refresh's own generation. +/// The view is left visibly unstamped, so the next refresh rebuilds. +async fn stamp_watermark( + view_native: &NativeTable, + mut dataset: Dataset, + source_version: u64, + source_ts: u128, + expected_incarnation: Option<&str>, +) -> Result { + ensure_incarnation(&dataset, expected_incarnation, dataset.uri()).await?; + let predicted = dataset.version().version + 1; + // A view with no token (declared before tokens existed, or its metadata + // replaced wholesale) starts a new incarnation here. + let incarnation = dataset + .schema() + .metadata + .get(INCARNATION_META_KEY) + .cloned() + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()); + dataset + .update_schema_metadata([ + (INCARNATION_META_KEY.to_string(), Some(incarnation)), + ( + SOURCE_VERSION_META_KEY.to_string(), + Some(source_version.to_string()), + ), + ( + SOURCE_VERSION_TS_META_KEY.to_string(), + Some(source_ts.to_string()), + ), + ( + REFRESHED_AT_MS_META_KEY.to_string(), + Some(now_ms().to_string()), + ), + ( + VIEW_VERSION_META_KEY.to_string(), + Some(predicted.to_string()), + ), + ]) + .await?; + let actual = dataset.version().version; + if actual != predicted { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {actual}, \ + expected {predicted}); the refresh is unrecorded and the next \ + one will rebuild" + ), + }); + } + view_native.dataset.update(dataset); + Ok(predicted) +} + +fn now_ms() -> u128 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() +} + +/// Evaluate the definition over `source`, restricted to `fragments` when +/// given, as batches in the view's schema. The filter is pushed into the +/// scan; projections are `(name, expression)` pairs, never spliced into SQL; +/// [`SOURCE_ROW_ID_COLUMN`] is filled by the scan's row id. +/// Which source rows a compute pass reads. +#[derive(Default)] +struct RowScope { + /// Read only these fragments. + fragments: Option>, + /// Read only rows created after this version. + created_after: Option, + /// Read only rows changed in place after the first version and no later + /// than the second. + updated_between: Option<(u64, u64)>, + /// Stop after this many rows. + limit: Option, +} + +async fn compute_stream( + source: &Dataset, + definition: &MaterializedViewDefinition, + scope: RowScope, + schema: SchemaRef, + rows_written: Arc, +) -> Result { + let RowScope { + fragments, + created_after, + updated_between, + limit, + } = scope; + let mut scanner = source.scan(); + if let Some(fragments) = fragments { + scanner.with_fragments(fragments); + } + scanner.with_row_id(); + // Narrowing keeps the definition's filter, so a row updated out of the + // view simply does not come back. Changed rows are named by the predicate + // `DatasetDelta::get_updated_rows` uses, not its streamed ids: an id list + // grows with the delta, this does not. + let updated_filter = updated_between.map(|(from, to)| { + format!( + "{ROW_CREATED_AT_VERSION} <= {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} > {from} \ + AND {ROW_LAST_UPDATED_AT_VERSION} <= {to}" + ) + }); + let created_filter = + created_after.map(|version| format!("{ROW_CREATED_AT_VERSION} > {version}")); + let clauses: Vec = definition + .filter + .clone() + .map(|f| format!("({f})")) + .into_iter() + .chain(updated_filter) + .chain(created_filter) + .collect(); + if !clauses.is_empty() { + scanner.filter(&clauses.join(" AND "))?; + } + let transforms: Vec<(&str, &str)> = definition + .projections + .iter() + .map(|p| (p.output.as_str(), p.expression.as_str())) + .collect(); + scanner.project_with_transform(&transforms)?; + // A scan reads a limit of zero as no limit at all, so a view capped at + // nothing is answered without one. + if limit == Some(0) { + return Ok(Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::empty(), + ))); + } + if let Some(limit) = limit { + let limit = i64::try_from(limit).map_err(|_| Error::InvalidInput { + message: format!("view limit {limit} exceeds the maximum of {}", i64::MAX), + })?; + scanner.limit(Some(limit), None)?; + } + + let out_schema = schema.clone(); + let mapped = scanner.try_into_stream().await?.map(move |batch| { + let batch = batch.map_err(|e| DataFusionError::External(Box::new(e)))?; + let mut columns = Vec::with_capacity(out_schema.fields().len()); + for field in out_schema.fields() { + let name = if field.name() == SOURCE_ROW_ID_COLUMN { + ROW_ID + } else { + field.name() + }; + let column = batch.column_by_name(name).ok_or_else(|| { + DataFusionError::Internal(format!( + "view column '{}' is not produced by the view's definition", + field.name() + )) + })?; + columns.push(column.clone()); + } + rows_written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(RecordBatch::try_new(out_schema.clone(), columns)?) + }); + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, mapped))) +} + +/// Commit the view's removals and additions as one change, on the exact +/// generation the refresh planned from. Lance rejects an overlapping +/// provenance key, but an unrelated write to the view is not a key conflict, +/// so the generation is checked here too. +async fn publish( + view_ds: &Dataset, + eviction: Option<(Vec, Vec)>, + new_fragments: Vec, + keys: Option, + expected_incarnation: Option<&str>, +) -> Result { + let planned = view_ds.version().version; + #[cfg(test)] + tests::hold_before_publish(view_ds.uri()).await; + #[cfg(test)] + tests::hold_until_peers_planned(); + ensure_incarnation(view_ds, expected_incarnation, view_ds.uri()).await?; + let (updated_fragments, removed_fragment_ids) = eviction.unwrap_or_default(); + let committed = CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + planned, + Operation::Update { + removed_fragment_ids, + updated_fragments, + new_fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: keys, + updated_fragment_offsets: None, + }, + None, + )) + .await?; + if committed.version().version != planned + 1 { + return Err(Error::Runtime { + message: format!( + "a concurrent commit raced this refresh (view version {}); + the refresh is unrecorded and the next one will rebuild", + committed.version().version + ), + }); + } + Ok(committed) +} + +/// A delta batch's row id column, borrowed: a delta batch is as large as one +/// fragment's deletions, so copying it out would be the unbounded step the +/// chunking exists to avoid. +fn row_ids_of(batch: &RecordBatch) -> Result<&UInt64Array> { + let column = batch.column_by_name(ROW_ID).ok_or_else(|| Error::Runtime { + message: format!("'{ROW_ID}' is missing from a delta batch"), + })?; + column + .as_primitive_opt::() + .ok_or_else(|| Error::Runtime { + message: "row ids are not UInt64".into(), + }) +} + +/// A provenance id no source row can hold, carried in every refresh's +/// inserted-rows filter: any two refreshes of one view overlap on it, so +/// the loser of a race conflicts at commit whatever rows each planned. +const REFRESH_TOKEN_ID: u64 = u64::MAX; + +/// A builder with no collected keys, for publishes that only remove rows. +fn empty_keys(view_ds: &Dataset) -> Result>> { + Ok(Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new( + vec![source_row_id_field_id(view_ds)?], + )))) +} + +/// An inserted-rows filter holding the refresh token plus whatever the +/// tee collected. +fn refresh_filter(keys: &Arc>) -> Result { + let mut keys = keys.lock().map_err(|_| Error::Runtime { + message: "the provenance key filter was poisoned mid-refresh".into(), + })?; + keys.insert(KeyValue::UInt64(REFRESH_TOKEN_ID)) + .map_err(|e| Error::Runtime { + message: format!("failed to mark the refresh's filter: {e}"), + })?; + Ok(keys.build()) +} + +/// Reconciled ids past this fall back to the streamed rebuild, whose memory +/// does not grow with the delta. +fn eviction_rebuild_cap() -> usize { + #[cfg(test)] + if let Some(cap) = tests::eviction_cap_override() { + return cap; + } + 4 * 1024 * 1024 +} + +/// Provenance ids per staged eviction: the delete predicate carries one +/// literal per id, so a whole delta at once is unbounded. Each chunk costs a +/// pass over the view's provenance column, which is why it is not smaller. +const EVICTION_CHUNK: usize = 64 * 1024; + +/// Accumulates the view's removals from bounded chunks of provenance ids into +/// the one set of fragment changes the refresh publishes. +struct Eviction { + /// The view as the chunks staged so far leave it. A chunk's delete has to + /// see the deletion vectors the earlier ones wrote, or it stages a + /// fragment that drops them. + snapshot: Dataset, + chunk: usize, + updated: HashMap, + removed: Vec, + pending: Vec, + staged: bool, + /// Largest the buffer ever got, which is the bound under test. + #[cfg(test)] + peak: usize, +} + +impl Eviction { + fn new(view_ds: &Dataset, chunk: usize) -> Self { + Self { + snapshot: view_ds.clone(), + chunk, + updated: HashMap::new(), + removed: Vec::new(), + pending: Vec::with_capacity(chunk), + staged: false, + #[cfg(test)] + peak: 0, + } + } + + /// Fill a chunk at a time. Taking the batch whole and splitting it would + /// hold a fragment's worth of ids and recopy the tail per chunk. + async fn push(&mut self, ids: &UInt64Array) -> Result<()> { + for id in ids.values() { + self.pending.push(*id); + #[cfg(test)] + { + self.peak = self.peak.max(self.pending.len()); + } + if self.pending.len() == self.chunk { + self.flush().await?; + } + } + Ok(()) + } + + /// Stage what is buffered, keeping the buffer's allocation. + async fn flush(&mut self) -> Result<()> { + let mut chunk = std::mem::take(&mut self.pending); + let staged = self.stage(&chunk).await; + chunk.clear(); + self.pending = chunk; + staged + } + + /// The staged fragment changes, or `None` where nothing was evicted. + async fn finish(mut self) -> Result, Vec)>> { + if !self.pending.is_empty() { + self.flush().await?; + } + if !self.staged { + return Ok(None); + } + let mut updated: Vec = self.updated.into_values().collect(); + updated.sort_unstable_by_key(|f| f.id); + Ok(Some((updated, self.removed))) + } + + async fn stage(&mut self, ids: &[u64]) -> Result<()> { + let (updated, removed) = stage_eviction(&self.snapshot, ids).await?; + self.snapshot = advance(&self.snapshot, &updated); + for fragment in updated { + self.updated.insert(fragment.id, fragment); + } + self.removed.extend(removed); + self.staged = true; + Ok(()) + } +} + +/// The view as staged removals leave it, without committing them. Fragments +/// are replaced in place so the fragment bitmap stays in step; an emptied one +/// is left alone, since the delta names each provenance id only once. +fn advance(view_ds: &Dataset, updated: &[Fragment]) -> Dataset { + if updated.is_empty() { + return view_ds.clone(); + } + let by_id: HashMap = updated.iter().map(|f| (f.id, f)).collect(); + let fragments = view_ds + .manifest + .fragments + .iter() + .map(|f| by_id.get(&f.id).map_or_else(|| f.clone(), |u| (*u).clone())) + .collect(); + let mut manifest = view_ds.manifest.as_ref().clone(); + manifest.fragments = Arc::new(fragments); + let mut snapshot = view_ds.clone(); + snapshot.manifest = Arc::new(manifest); + snapshot +} + +/// The fragment changes that remove the view's rows for `ids`, staged rather +/// than committed so they can ride in the refresh's single data commit. +async fn stage_eviction(view_ds: &Dataset, ids: &[u64]) -> Result<(Vec, Vec)> { + // An expression rather than SQL text: the id list is a value here, not a + // predicate string that grows with the delta and has to be parsed. + let predicate = col(SOURCE_ROW_ID_COLUMN).in_list( + ids.iter() + .map(|id| lit(ScalarValue::UInt64(Some(*id)))) + .collect(), + false, + ); + let staged = DeleteBuilder::from_expr(Arc::new(view_ds.clone()), predicate) + .execute_uncommitted() + .await?; + let Operation::Delete { + updated_fragments, + deleted_fragment_ids, + .. + } = staged.transaction.operation + else { + return Err(Error::Runtime { + message: "expected a delete when staging the view's evictions".into(), + }); + }; + Ok((updated_fragments, deleted_fragment_ids)) +} + +fn source_row_id_field_id(view_ds: &Dataset) -> Result { + view_ds + .schema() + .field(SOURCE_ROW_ID_COLUMN) + .map(|f| f.id) + .ok_or_else(|| Error::Runtime { + message: format!("the view has no '{SOURCE_ROW_ID_COLUMN}' column"), + }) +} + +/// Tee the provenance ids of everything written into `keys`. +fn collect_source_row_ids( + stream: SendableRecordBatchStream, + keys: Arc>, + written: Arc, +) -> SendableRecordBatchStream { + let schema = stream.schema(); + let mapped = stream.map(move |batch| { + let batch = batch?; + let column = batch + .column_by_name(SOURCE_ROW_ID_COLUMN) + .ok_or_else(|| { + DataFusionError::Internal(format!( + "'{SOURCE_ROW_ID_COLUMN}' is missing from the rows being written" + )) + })? + .as_primitive_opt::() + .ok_or_else(|| { + DataFusionError::Internal(format!("'{SOURCE_ROW_ID_COLUMN}' is not a uint64")) + })?; + let mut keys = keys + .lock() + .map_err(|_| DataFusionError::Internal("provenance key filter poisoned".into()))?; + for id in column.values() { + keys.insert(KeyValue::UInt64(*id)) + .map_err(|e| DataFusionError::Internal(e.to_string()))?; + } + written.fetch_add(batch.num_rows() as u64, Ordering::Relaxed); + Ok(batch) + }); + Box::pin(RecordBatchStreamAdapter::new(schema, mapped)) +} + +#[cfg(test)] +mod tests { + + /// Park a refresh between planning and publication so a test can move + /// the view underneath it. Inert unless [`DRIFT_TARGET`] names this view. + pub(super) async fn hold_before_publish(uri: &str) { + { + let mut target = DRIFT_TARGET.lock().unwrap(); + if target.as_deref() != Some(uri) { + return; + } + // Take it: memory:// uris are relative and repeat across tests, so + // leaving it armed would park an unrelated refresh forever. + *target = None; + } + DRIFT_PLANNED.notify_one(); + DRIFT_RELEASED.notified().await; + } + + /// The rendezvous below is one global pair, so the cases that use it run + /// one at a time rather than trading each other's signals. + pub(super) static EVICTION_CAP: StdMutex> = StdMutex::new(None); + pub(super) fn eviction_cap_override() -> Option { + *EVICTION_CAP.lock().unwrap() + } + + pub(super) static DRIFT_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + pub(super) static DRIFT_TARGET: StdMutex> = StdMutex::new(None); + pub(super) static DRIFT_PLANNED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + pub(super) static DRIFT_RELEASED: tokio::sync::Notify = tokio::sync::Notify::const_new(); + + /// Block until every participant in a cross-process race has planned and + /// staged its write, so the commits they then attempt genuinely contend + /// rather than depending on the scheduler to overlap them. Inert unless + /// `MV_RACE_SYNC` names a directory shared by the participants. + pub(super) fn hold_until_peers_planned() { + let (Ok(dir), Ok(tag), Ok(peers)) = ( + std::env::var("MV_RACE_SYNC"), + std::env::var("MV_RACE_TAG"), + std::env::var("MV_RACE_PEERS"), + ) else { + return; + }; + let dir = std::path::PathBuf::from(dir); + let peers: usize = peers.parse().unwrap(); + std::fs::write(dir.join(format!("planned-{tag}")), b"1").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(120); + while planned_count(&dir) < peers { + assert!( + std::time::Instant::now() < deadline, + "peers never reached the commit boundary" + ); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } + + fn planned_count(dir: &std::path::Path) -> usize { + std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().starts_with("planned-")) + .count() + }) + .unwrap_or(0) + } + use arrow_array::{Int32Array, record_batch}; + use futures::TryStreamExt; + use lance::dataset::NewColumnTransform; + use lance_file::version::LanceFileVersion; + + use super::*; + use crate::connect; + use crate::connection::Connection; + use crate::index::Index; + use crate::index::scalar::BTreeIndexBuilder; + use crate::materialized_view::MaterializedView; + use crate::query::{ExecutableQuery, QueryBase, Select}; + use crate::table::{CompactionOptions, OptimizeAction}; + + async fn db_with_source(values: Vec) -> (Connection, Table) { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, values)).unwrap(); + let table = conn + .create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + (conn, table) + } + + async fn doubled_view(conn: &Connection) -> MaterializedView { + conn.create_materialized_view("doubled", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap() + } + + /// A refreshed doubled view over a source holding `values`. + async fn refreshed_doubled(values: Vec) -> (Connection, Table, MaterializedView) { + let (conn, source) = db_with_source(values).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + (conn, source, view) + } + + async fn read(table: &Table, column: &str) -> Vec { + let batches = table + .query() + .select(Select::columns(&[column])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let mut values: Vec = batches + .iter() + .flat_map(|batch| { + batch[column] + .as_any() + .downcast_ref::() + .unwrap() + .iter() + .flatten() + .collect::>() + }) + .collect(); + values.sort(); + values + } + + async fn append(table: &Table, values: Vec) { + let batch = record_batch!(("x", Int32, values)).unwrap(); + table.add(batch).execute().await.unwrap(); + } + + #[tokio::test] + async fn test_first_refresh_materializes_the_view() { + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = doubled_view(&conn).await; + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 3); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + + // The watermark survives on the stored schema, not just the handle. + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + let again = reopened.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + assert_eq!(again.rows_written, 0); + } + + #[tokio::test] + async fn test_filter_selects_the_source_rows() { + let (conn, _) = db_with_source(vec![1, 20, 3, 40]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "x").await, vec![20, 40]); + } + + #[tokio::test] + async fn test_append_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + append(&source, vec![5]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 10]); + } + + #[tokio::test] + async fn test_incremental_applies_the_filter() { + let (conn, source) = db_with_source(vec![1, 20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![3, 30]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "x").await, vec![20, 30]); + } + + /// The watermark has to advance even when no appended row matches, or the + /// same fragments would be rescanned by every later refresh. + #[tokio::test] + async fn test_incremental_with_nothing_matching_advances_the_watermark() { + let (conn, source) = db_with_source(vec![20]).await; + let view = conn + .create_materialized_view("big", "src") + .select([("x", "x")]) + .only_if("x > 10") + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![1, 2]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + + let again = view.refresh().execute().await.unwrap(); + assert_eq!(again.mode, RefreshMode::NoOp); + } + + /// Unlike a computed column, a view reflects source mutation: an update + /// rebuilds rather than going stale. + #[tokio::test] + async fn test_update_replaces_the_rows_it_changed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // An update changes rows in place, so the view replaces exactly + // those rows and leaves the rest of what it holds alone. + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1, "only the changed row is recomputed"); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 40]); + } + + /// Legacy storage cannot serve the row-version columns update discovery + /// scans; appends stay incremental, updates rebuild rather than fail. + #[tokio::test] + async fn test_a_legacy_storage_source_is_reconciled() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("x", Int32, [1, 2, 3])).unwrap(); + let source = conn + .create_table("legacy_src", batch) + .write_options(crate::table::WriteOptions { + lance_write_params: Some(lance::dataset::WriteParams { + enable_stable_row_ids: true, + data_storage_version: Some(LanceFileVersion::Legacy), + ..Default::default() + }), + }) + .execute() + .await + .unwrap(); + let view = conn + .create_materialized_view("legacy_doubled", "legacy_src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .add(record_batch!(("x", Int32, [4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + + source + .update() + .column("x", "20") + .only_if("x = 2") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 6, 8, 40]); + } + + #[tokio::test] + async fn test_delete_evicts_the_view_rows_it_removed() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + // A delete removes source rows without changing the ones that + // remain, so the view evicts exactly those rows and keeps the rest + // rather than recomputing every row it already held. + source.delete("x = 2").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + + // A delete and an append in one span: both are applied. + source.delete("x = 1").await.unwrap(); + source + .add(record_batch!(("x", Int32, vec![4])).unwrap()) + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![6, 8]); + } + + /// merge_insert commits an `Update`, and its by-source arm removes rows + /// rather than changing them, so the classifier must treat that + /// transaction form as a source of deletions. + #[tokio::test] + async fn test_merge_insert_by_source_delete_evicts_the_view_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batch = record_batch!(("x", Int32, vec![1, 3])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge.when_not_matched_by_source_delete(None); + merge.execute(Box::new(reader)).await.unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 6]); + } + + /// A refresh may only certify the generation it planned from. A write + /// that lands between planning and publication is drift the refresh did + /// not account for, so it aborts rather than stamp it as materialized. + #[tokio::test(flavor = "multi_thread")] + async fn test_refresh_aborts_rather_than_certify_view_drift() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("drifting_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + let certified = source_version_of(view.table()).await; + + // The source loses a row, so the next refresh plans an eviction. + source.delete("x = 2").await.unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Move the view once the refresh has planned against it. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let drifted = conn.open_table("drifting_view").execute().await.unwrap(); + drifted.delete("twice = 6").await.unwrap(); + DRIFT_RELEASED.notify_one(); + + // Publishing removals and additions as one change makes the drift a + // conflict lance itself rejects; a pure append, which touches no + // existing fragment, still relies on the generation check. + let err = refreshing.await.unwrap().unwrap_err(); + let message = err.to_string(); + assert!( + message.contains("raced this refresh") || message.contains("preempted by concurrent"), + "got {err:?}" + ); + // The watermark still names the generation that was actually proven. + assert_eq!(source_version_of(&drifted).await, certified); + } + + async fn source_version_of(table: &Table) -> Option { + table + .schema() + .await + .unwrap() + .metadata() + .get(SOURCE_VERSION_META_KEY) + .cloned() + } + + /// A transaction file carries placeholder ids for the fragments it + /// creates. Into an empty source those collide with the ids the commit + /// assigns, so treating them as already-materialized drops the very + /// first rows while the watermark still advances past them. + #[tokio::test] + async fn test_merge_into_an_empty_source_is_materialized() { + let (_conn, source, view) = refreshed_doubled(vec![]).await; + assert_eq!(read(view.table(), "twice").await, Vec::::new()); + + let batch = record_batch!(("x", Int32, vec![1, 2])).unwrap(); + let reader = arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema()); + let mut merge = source.merge_insert(&["x"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all(); + merge.execute(Box::new(reader)).await.unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + // A second refresh must not double them either. + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A refresh publishes what it removes and what it adds as one change, + /// so an update never exposes the view without the rows it is replacing. + #[tokio::test(flavor = "multi_thread")] + async fn test_an_update_is_never_visible_as_a_gap() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("atomic_view", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = tokio::spawn(async move { view.refresh().execute().await }); + + // Read the view while the refresh is staged but not yet published. + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + let midway = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!( + read(&midway, "twice").await, + vec![2, 4, 6], + "the pre-refresh rows must still be there in full" + ); + DRIFT_RELEASED.notify_one(); + + refreshing.await.unwrap().unwrap(); + let after = conn.open_table("atomic_view").execute().await.unwrap(); + assert_eq!(read(&after, "twice").await, vec![22, 24, 26]); + } + + async fn provenance_by_x(table: &Table) -> HashMap { + let batches = table + .query() + .select(Select::columns(&["x", SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + batches + .iter() + .flat_map(|batch| { + let xs = batch["x"].as_any().downcast_ref::().unwrap(); + let ids = batch[SOURCE_ROW_ID_COLUMN].as_primitive::(); + (0..batch.num_rows()) + .map(|i| (xs.value(i), ids.value(i))) + .collect::>() + }) + .collect() + } + + /// One delta batch is as large as a fragment's deletions, so it arrives + /// well past a chunk: it has to be drained a chunk at a time without ever + /// holding the batch, and the passes' deletion vectors have to accumulate + /// rather than replace each other. + #[tokio::test] + async fn test_a_chunked_eviction_removes_every_row_it_names() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3, 4, 5, 6]).await; + + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let provenance = provenance_by_x(view.table()).await; + + let mut eviction = Eviction::new(&view_ds, 2); + let batch = UInt64Array::from_iter_values([1, 2, 3, 4].map(|x| provenance[&x])); + eviction.push(&batch).await.unwrap(); + assert_eq!( + eviction.peak, 2, + "a batch past the chunk was buffered whole" + ); + let staged = eviction.finish().await.unwrap(); + assert!(staged.is_some(), "four ids over a chunk of two stage twice"); + publish(&view_ds, staged, Vec::new(), None, None) + .await + .unwrap(); + native.dataset.reload().await.unwrap(); + + assert_eq!(read(view.table(), "x").await, vec![5, 6]); + } + + /// Two first rebuilds materialize the same source rows; the loser must + /// key-conflict at commit and land nothing, or the view doubles. + #[tokio::test] + async fn test_a_raced_first_rebuild_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_rebuild", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + let native = view.table().as_native().unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + let uri = view_ds.uri().to_string(); + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the rebuild never reached the publication boundary"); + + // The other process's first rebuild, reduced to its commit. Its + // source rows are DISJOINT from ours -- the sentinel alone must make + // the two rebuilds conflict. + let batch = record_batch!( + ("x", Int32, [8, 9]), + ("twice", Int32, [16, 18]), + ("__source_row_id", UInt64, [7u64, 8]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = Arc::new(StdMutex::new(KeyExistenceFilterBuilder::new(vec![ + source_row_id_field_id(&view_ds).unwrap(), + ]))); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = { + let mut keys = keys.lock().unwrap(); + keys.insert(KeyValue::UInt64(super::REFRESH_TOKEN_ID)) + .unwrap(); + keys.build() + }; + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + // The loser lands nothing: only the winner's rows remain. + let raced = conn.open_table("raced_rebuild").execute().await.unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![16, 18], + "the losing rebuild must not union with the winner ({err})" + ); + } + + /// An incremental refresh racing any other refresh must land nothing, + /// even when their planned rows are disjoint: the shared token makes the + /// commits conflict. + #[tokio::test] + async fn test_a_raced_incremental_lands_nothing() { + let _guard = DRIFT_LOCK.lock().await; + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("raced_incremental", "src") + .select([("x", "x"), ("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + append(&source, vec![4]).await; + + let native = view.table().as_native().unwrap(); + native.dataset.reload().await.unwrap(); + let view_ds = native.dataset.get().await.unwrap().as_ref().clone(); + *DRIFT_TARGET.lock().unwrap() = Some(view_ds.uri().to_string()); + let racing = tokio::spawn(async move { view.refresh().execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("the refresh never reached the publication boundary"); + + // The concurrent refresh, reduced to its commit: disjoint rows, the + // shared token alone must collide. + let batch = record_batch!( + ("x", Int32, [9]), + ("twice", Int32, [18]), + ("__source_row_id", UInt64, [8u64]) + ) + .unwrap(); + let schema = Arc::new(ArrowSchema::from(view_ds.schema())); + let batch = RecordBatch::try_new( + schema.clone(), + schema + .fields() + .iter() + .map(|f| batch.column_by_name(f.name()).unwrap().clone()) + .collect(), + ) + .unwrap(); + let stream: SendableRecordBatchStream = Box::pin(RecordBatchStreamAdapter::new( + schema.clone(), + futures::stream::iter([Ok(batch)]), + )); + let keys = empty_keys(&view_ds).unwrap(); + let stream = collect_source_row_ids(stream, keys.clone(), Arc::new(AtomicU64::new(0))); + let write_txn = InsertBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .with_params(&WriteParams { + mode: WriteMode::Append, + ..Default::default() + }) + .execute_uncommitted_stream(stream) + .await + .unwrap(); + let Operation::Append { fragments } = write_txn.operation else { + panic!("expected an append"); + }; + let filter = refresh_filter(&keys).unwrap(); + CommitBuilder::new(WriteDestination::Dataset(Arc::new(view_ds.clone()))) + .execute(Transaction::new( + view_ds.version().version, + Operation::Update { + removed_fragment_ids: Vec::new(), + updated_fragments: Vec::new(), + new_fragments: fragments, + fields_modified: Vec::new(), + compacted_sstables: Vec::new(), + fields_for_preserving_frag_bitmap: Vec::new(), + update_mode: None, + inserted_rows_filter: Some(filter), + updated_fragment_offsets: None, + }, + None, + )) + .await + .unwrap(); + DRIFT_RELEASED.notify_one(); + + let err = racing.await.unwrap().unwrap_err(); + let raced = conn + .open_table("raced_incremental") + .execute() + .await + .unwrap(); + assert_eq!( + read(&raced, "twice").await, + vec![2, 4, 6, 18], + "the losing increment must not land its rows ({err})" + ); + } + + /// Past the eviction cap the refresh falls back to the streamed rebuild, + /// and the result is identical either way. + #[tokio::test] + async fn test_oversized_delta_falls_back_to_rebuild() { + let (conn, source) = db_with_source((1..=20).collect()).await; + let view = doubled_view(&conn).await; + view.refresh().execute().await.unwrap(); + + source.delete("x <= 10").await.unwrap(); + *tests::EVICTION_CAP.lock().unwrap() = Some(4); + let result = view.refresh().execute().await; + *tests::EVICTION_CAP.lock().unwrap() = None; + let result = result.unwrap(); + assert_eq!( + result.mode, + RefreshMode::Rebuild, + "ten evictions, cap of four" + ); + assert_eq!(read(view.table(), "x").await, (11..=20).collect::>()); + + // Under the cap the same shape stays incremental. + source.delete("x = 11").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "x").await, (12..=20).collect::>()); + } + + /// A cap of zero is a view that holds nothing, not a view without a cap. + #[tokio::test] + async fn test_zero_limit_holds_no_rows() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("empty", "src") + .select([("x", "x")]) + .limit(0) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + + // Still nothing after the source grows, and the watermark advances. + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// An update rewrites a whole fragment. When it lands on one appended + /// since the watermark, that fragment also holds rows the update never + /// touched -- rows the recompute does not cover and the append set no + /// longer reaches. + #[tokio::test] + async fn test_update_touching_a_new_fragment_keeps_its_untouched_rows() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + // One fragment, appended after the watermark, holding both a row the + // update will change and a row it will not. + append(&source, vec![3, 40]).await; + source + .update() + .column("x", "99") + .only_if("x = 3") + .execute() + .await + .unwrap(); + + view.refresh().execute().await.unwrap(); + assert_eq!( + read(view.table(), "twice").await, + vec![2, 4, 80, 198], + "a row appended into the updated fragment went missing" + ); + } + + async fn compact(source: &Table) { + source + .optimize(OptimizeAction::Compact { + options: CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + } + + /// A compaction rearranges rows without changing them, so it costs the + /// view nothing: the watermark advances and no row is recomputed. + #[tokio::test] + async fn test_compaction_alone_refreshes_incrementally() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + append(&source, vec![3]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Rows appended after a compaction are separable through the transaction + /// log: only the appended fragments are computed. + #[tokio::test] + async fn test_append_after_compaction_stays_incremental() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + append(&source, vec![2]).await; + view.refresh().execute().await.unwrap(); + + compact(&source).await; + append(&source, vec![3]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + } + + /// An append swallowed by a later compaction cannot be told apart from + /// the rows the view already holds, so the refresh rebuilds -- once -- + /// rather than duplicate or drop. + #[tokio::test] + async fn test_append_folded_into_compaction_rebuilds() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + compact(&source).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// The create-time gate holds across a drop-and-recreate of the source + /// under the same name. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_source_without_stable_row_ids() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [9])).unwrap(); + conn.create_table("src", batch).execute().await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("stable row ids")) + ); + } + + /// A change to a column the view does not read is not a reason to + /// rebuild: the exact signature check is scoped to the view's inputs. + #[tokio::test] + async fn test_unrelated_column_change_does_not_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2]).await; + + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "unrelated", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 0); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + #[tokio::test] + async fn test_full_forces_a_rebuild() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2]).await; + let result = view.refresh().full(true).execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// A cap and incremental reconciliation do not compose: rows skipped at + /// the cap fall behind the watermark, so room freed later cannot be + /// refilled from any delta. A capped view rebuilds instead, which its + /// own cap keeps cheap. + #[tokio::test] + async fn test_limited_view_rebuilds_rather_than_reconcile() { + let (conn, source) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(2) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + assert_eq!(read(view.table(), "x").await, vec![1, 2]); + + append(&source, vec![3, 4]).await; + source + .update() + .column("x", "11") + .only_if("x = 1") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + let held = read(view.table(), "x").await; + assert_eq!(held.len(), 2, "the cap holds: {held:?}"); + let selectable = read(&source, "x").await; + assert!( + held.iter().all(|x| selectable.contains(x)), + "{held:?} is not a subset of {selectable:?}" + ); + } + + #[tokio::test] + async fn test_limit_caps_the_view() { + let (conn, source) = db_with_source(vec![1, 2, 3]).await; + let view = conn + .create_materialized_view("capped", "src") + .select([("x", "x")]) + .limit(4) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 3); + + // The cap counts already-held rows, so only one appended row lands. + append(&source, vec![4, 5, 6]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(result.rows_written, 1); + assert_eq!(view.table().count_rows(None).await.unwrap(), 4); + + // At the cap, later appends only move the watermark. + append(&source, vec![7]).await; + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// The whole point of the fragment-swap commit: a rebuild must never + /// leave the view without its index definitions. + #[tokio::test] + async fn test_rebuild_retains_indexes() { + let (_conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + + view.table() + .create_index(&["twice"], Index::BTree(BTreeIndexBuilder::default())) + .execute() + .await + .unwrap(); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + + source + .update() + .column("x", "x + 10") + .execute() + .await + .unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(view.table().list_indices().await.unwrap().len(), 1); + assert_eq!(read(view.table(), "twice").await, vec![22, 24, 26]); + + // The swapped-in rows are reachable through an indexed query. + let batches = view + .table() + .query() + .only_if("twice = 24") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(|b| b.num_rows()).sum::(), 1); + } + + #[tokio::test] + async fn test_rebuild_of_an_empty_result_is_an_empty_view() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("none", "src") + .select([("x", "x")]) + .only_if("x > 100") + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 0); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// Provenance: every view row records the source row that produced it. + #[tokio::test] + async fn test_source_row_ids_are_recorded() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2, 3]).await; + + let batches = view + .table() + .query() + .select(Select::columns(&[SOURCE_ROW_ID_COLUMN])) + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + let total: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total, 3); + for batch in &batches { + assert_eq!(batch[SOURCE_ROW_ID_COLUMN].null_count(), 0); + } + } + + #[tokio::test] + async fn test_dropping_a_source_input_fails_the_refresh() { + let (conn, source) = db_with_source(vec![1]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("twice", "x * 2")]) + .execute() + .await + .unwrap(); + view.refresh().execute().await.unwrap(); + + append(&source, vec![2]).await; + source + .add_columns() + .transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new( + vec![arrow_schema::Field::new( + "y", + arrow_schema::DataType::Int32, + true, + )], + )))) + .execute() + .await + .unwrap(); + source.drop_columns(&["x"]).await.unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("'x'"))); + } + + /// A pinned refresh materializes the source as of `version`; catching up + /// to the appends beyond it stays incremental. + #[tokio::test] + async fn test_pinned_refresh_and_catch_up() { + let (conn, source) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let pinned = source.version().await.unwrap(); + + append(&source, vec![2]).await; + let result = view + .refresh() + .source_version(pinned) + .execute() + .await + .unwrap(); + assert_eq!(result.source_version, pinned); + assert_eq!(read(view.table(), "twice").await, vec![2]); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// Views chain: a view's stable row ids and provenance column make it a + /// source like any other, and the default projection takes its declared + /// columns without copying its provenance. + #[tokio::test] + async fn test_a_view_can_source_another_view() { + let (conn, source) = db_with_source(vec![1, 2, 30]).await; + let first = doubled_view(&conn).await; + first.refresh().execute().await.unwrap(); + + let second = conn + .create_materialized_view("second", "doubled") + .only_if("twice > 10") + .execute() + .await + .unwrap(); + assert!( + second + .definition() + .projections + .iter() + .all(|p| p.output != SOURCE_ROW_ID_COLUMN) + ); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 1); + assert_eq!(read(second.table(), "twice").await, vec![60]); + + append(&source, vec![50]).await; + first.refresh().execute().await.unwrap(); + let result = second.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Incremental); + assert_eq!(read(second.table(), "twice").await, vec![60, 100]); + } + + /// The watermark speaks only for the state a refresh left behind: a + /// direct write to the view is drift, and the next refresh rebuilds + /// rather than preserving it as current. + #[tokio::test] + async fn test_direct_view_mutation_forces_a_rebuild() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + view.table().delete("x = 1").await.unwrap(); + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + } + + /// A dropped and recreated source reuses version numbers but never their + /// timestamps; the watermark must not vouch for the replacement's rows. + #[tokio::test] + async fn test_source_recreation_forces_a_rebuild() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + + conn.drop_table("src", &[]).await.unwrap(); + let batch = record_batch!(("x", Int32, [7])).unwrap(); + conn.create_table("src", batch) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![14]); + } + + /// A refresh bound to an incarnation refuses a view dropped and recreated + /// since, even under the same name and definition; the recreated view's + /// own token is accepted, and the token survives a refresh's stamp. + #[tokio::test] + async fn test_refresh_refuses_a_recreated_view_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + view.refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(token.as_str())); + + conn.drop_table("doubled", &[]).await.unwrap(); + let recreated = doubled_view(&conn).await; + assert_ne!(recreated.incarnation(), Some(token.as_str())); + + let err = recreated + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("dropped and recreated"), "{err}"); + assert_eq!(read(recreated.table(), "twice").await, Vec::::new()); + + recreated + .refresh() + .expect_incarnation(recreated.incarnation().unwrap()) + .execute() + .await + .unwrap(); + assert_eq!(read(recreated.table(), "twice").await, vec![2]); + } + + /// A cloned declaration creates two physical tables; each gets its own + /// token. + #[tokio::test] + async fn test_cloned_declaration_mints_a_fresh_incarnation_per_create() { + let (conn, source) = db_with_source(vec![1]).await; + let prepared = crate::materialized_view::prepare_declaration( + &source, + &[("x".into(), "x".into()), ("twice".into(), "x * 2".into())], + None, + None, + ) + .await + .unwrap(); + let replacement = prepared.clone(); + let first = prepared.create("cloned").await.unwrap(); + let first_token = first.incarnation().unwrap().to_string(); + + conn.drop_table("cloned", &[]).await.unwrap(); + let second = replacement.create("cloned").await.unwrap(); + assert_ne!(second.incarnation(), Some(first_token.as_str())); + } + + /// A recreation that lands after planning but before publication is + /// caught by the pre-commit read: the stale refresh fails and the + /// replacement stays empty under its own token. + #[tokio::test(flavor = "multi_thread")] + async fn test_bound_refresh_cannot_publish_into_a_raced_recreation() { + let _serial = DRIFT_LOCK.lock().await; + let (conn, _) = db_with_source(vec![1]).await; + let view = doubled_view(&conn).await; + let token = view.incarnation().unwrap().to_string(); + let uri = view + .table() + .as_native() + .unwrap() + .dataset + .get() + .await + .unwrap() + .uri() + .to_string(); + + *DRIFT_TARGET.lock().unwrap() = Some(uri); + let refreshing = + tokio::spawn(async move { view.refresh().expect_incarnation(token).execute().await }); + tokio::time::timeout(std::time::Duration::from_secs(30), DRIFT_PLANNED.notified()) + .await + .expect("refresh never reached publication"); + + conn.drop_table("doubled", &[]).await.unwrap(); + let replacement = doubled_view(&conn).await; + let replacement_token = replacement.incarnation().unwrap().to_string(); + DRIFT_RELEASED.notify_one(); + + let result = refreshing.await.unwrap(); + assert!(result.is_err(), "the stale refresh unexpectedly succeeded"); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert_eq!(reopened.incarnation(), Some(replacement_token.as_str())); + assert_eq!(read(reopened.table(), "twice").await, Vec::::new()); + } + + /// Replacing the schema metadata wholesale drops the token. A refresh + /// bound to the old token is refused for that reason, not as a + /// recreation; an unbound refresh mints the view a fresh one. + #[tokio::test] + async fn test_a_view_whose_metadata_was_replaced_starts_a_new_incarnation() { + let (conn, _, view) = refreshed_doubled(vec![1]).await; + let token = view.incarnation().unwrap().to_string(); + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(view.definition()).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + assert_eq!( + conn.open_materialized_view("doubled") + .await + .unwrap() + .incarnation(), + None + ); + + let err = view + .refresh() + .expect_incarnation(&token) + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("no incarnation token"), "{err}"); + + view.refresh().execute().await.unwrap(); + let reopened = conn.open_materialized_view("doubled").await.unwrap(); + assert!(reopened.incarnation().is_some()); + assert_ne!(reopened.incarnation(), Some(token.as_str())); + } + + /// In-process refreshes of one view serialize: the loser of the race + /// observes the winner's watermark instead of appending the same rows. + #[tokio::test(flavor = "multi_thread")] + async fn test_concurrent_refreshes_do_not_duplicate() { + let (_conn, source, view) = refreshed_doubled(vec![1]).await; + + append(&source, vec![2, 3]).await; + let (a, b) = tokio::join!(view.refresh().execute(), view.refresh().execute()); + let (a, b) = (a.unwrap(), b.unwrap()); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6]); + let modes = [a.mode, b.mode]; + assert!(modes.contains(&RefreshMode::Incremental)); + assert!(modes.contains(&RefreshMode::NoOp)); + } + + /// A second handle's lazy cache must not defeat the lock: after another + /// handle's refresh commits, the stale handle plans from the reloaded + /// state and no-ops instead of appending the same fragments again. + #[tokio::test] + async fn test_a_second_handle_does_not_double_append() { + let (conn, source, view) = refreshed_doubled(vec![1, 2, 3]).await; + let stale = conn.open_materialized_view("doubled").await.unwrap(); + + append(&source, vec![4]).await; + view.refresh().execute().await.unwrap(); + + let result = stale.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::NoOp); + assert_eq!(read(view.table(), "twice").await, vec![2, 4, 6, 8]); + } + + /// A commit racing between a refresh's data commit and its stamp must + /// not be certified as the refresh's generation: the stamp aborts, and + /// the next refresh rebuilds from the drifted state. + #[tokio::test] + async fn test_stamp_aborts_on_a_racing_commit() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let view_native = view.table().as_native().unwrap(); + let stale = view_native.dataset.get().await.unwrap().as_ref().clone(); + view.table().delete("x = 1").await.unwrap(); + + let err = stamp_watermark(view_native, stale, 99, 99, None).await; + assert!(err.is_err()); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![2, 4]); + } + + /// What refresh executes and what it stamps must be one generation: a + /// replaced definition wins over whatever a stale handle cached. + #[tokio::test] + async fn test_refresh_uses_the_latest_persisted_definition() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let replacement = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![ + crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }, + crate::materialized_view::ViewProjection { + output: "twice".into(), + expression: "x * 3".into(), + }, + ], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&replacement).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(read(view.table(), "twice").await, vec![3, 6]); + } + + /// A persisted definition that does not produce this view's schema must + /// not refresh at all, let alone be certified. + #[tokio::test] + async fn test_definition_view_schema_mismatch_is_refused() { + let (_conn, _, view) = refreshed_doubled(vec![1, 2]).await; + + let narrower = crate::materialized_view::MaterializedViewDefinition { + source_table: "src".into(), + projections: vec![crate::materialized_view::ViewProjection { + output: "x".into(), + expression: "x".into(), + }], + filter: None, + limit: None, + inputs: vec!["x".into()], + }; + let mut metadata = HashMap::new(); + metadata.insert( + crate::materialized_view::DEFINITION_META_KEY.to_string(), + crate::materialized_view::definition_to_metadata(&narrower).unwrap(), + ); + view.table() + .as_native() + .unwrap() + .replace_schema_metadata(metadata) + .await + .unwrap(); + + let err = view.refresh().execute().await.unwrap_err(); + assert!(matches!(err, Error::Schema { message } if message.contains("does not produce")),); + } + + /// An overlay replaces cell values without changing any file path; the + /// signature must see it, scoped to the columns the view reads like + /// data files are. + #[test] + fn test_fragment_signature_sees_overlays() { + use lance_file::version::ConcreteFileVersion; + use lance_table::format::DataFile; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + + let base = Fragment::new(7); + let mut file = DataFile::new_unstarted("f0.lance", ConcreteFileVersion::V2_1); + file.fields = vec![0, 1].into(); + let mut with_file = base.clone(); + with_file.files.push(file.clone()); + + let overlay = |field: i32| { + let mut data_file = DataFile::new_unstarted("o0.lance", ConcreteFileVersion::V2_1); + data_file.fields = vec![field].into(); + DataOverlayFile { + data_file, + coverage: OverlayCoverage::PerField(Vec::new()), + committed_version: 9, + } + }; + let relevant: HashSet = [0].into_iter().collect(); + + let mut overlaid_relevant = with_file.clone(); + overlaid_relevant.overlays.push(overlay(0)); + assert_ne!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_relevant, &relevant), + ); + + let mut overlaid_unrelated = with_file.clone(); + overlaid_unrelated.overlays.push(overlay(5)); + assert_eq!( + fragment_signature(&with_file, &relevant), + fragment_signature(&overlaid_unrelated, &relevant), + ); + } + + /// An output whose name needs quoting flows through as a projection + /// alias, never spliced into SQL text. + #[tokio::test] + async fn test_output_names_needing_quotes() { + let (conn, _) = db_with_source(vec![1, 2]).await; + let view = conn + .create_materialized_view("v", "src") + .select([("double value", "x * 2")]) + .execute() + .await + .unwrap(); + + let result = view.refresh().execute().await.unwrap(); + assert_eq!(result.rows_written, 2); + assert_eq!(read(view.table(), "double value").await, vec![2, 4]); + } + + /// MemWAL tiers are visible to reads but not to the refresh scan, so LSM + /// state disqualifies every participant: the source at create, either + /// side at refresh, and the view can never accept a spec. Retained + /// un-compacted rows (the catch-up flag outlives unset) count as state. + #[tokio::test] + async fn lsm_state_disqualifies_source_and_view() { + use crate::table::LsmWriteSpec; + use arrow_array::RecordBatchIterator; + + let tmp_dir = tempfile::tempdir().unwrap(); + let conn = connect(tmp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + // Hand-rolled: the LSM primary key must be non-nullable, which + // record_batch! cannot express. + let schema = Arc::new(ArrowSchema::new(vec![ + arrow_schema::Field::new("id", arrow_schema::DataType::Int64, false), + arrow_schema::Field::new("x", arrow_schema::DataType::Int32, false), + ])); + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![1, 2])) as _, + Arc::new(Int32Array::from(vec![1, 2])) as _, + ], + ) + .unwrap(); + let table = conn + .create_table("src", batch.clone()) + .write_options(crate::materialized_view::tests::stable_row_ids()) + .execute() + .await + .unwrap(); + table.set_unenforced_primary_key(["id"]).await.unwrap(); + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + + // An active-LSM source is refused at create. + let err = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap_err(); + assert!(err.to_string().contains("un-compacted"), "{err}"); + + // Unset with nothing written clears the state: the view creates, + // and a spec can never be installed over it. + table.unset_lsm_write_spec().await.unwrap(); + let view = conn + .create_materialized_view("v", "src") + .execute() + .await + .unwrap(); + let err = view + .table() + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap_err(); + assert!(err.to_string().contains("materialized view"), "{err}"); + + // A source that acquires a spec after create fails refresh. + table + .set_lsm_write_spec(LsmWriteSpec::unsharded()) + .await + .unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + + // Retained rows outlive unset: write through the WAL, unset, and + // refresh still refuses. + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_insert_all() + .use_lsm(true); + merge + .execute(Box::new(RecordBatchIterator::new( + vec![Ok(batch.clone())], + batch.schema(), + ))) + .await + .unwrap(); + table.unset_lsm_write_spec().await.unwrap(); + let err = view.refresh().execute().await.unwrap_err(); + assert!(err.to_string().contains("source table 'src'"), "{err}"); + } +} diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 169d0fda5..3f4216bc8 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -344,6 +344,62 @@ impl RemoteDatabase { self.table_cache.remove(&cache_key).await; Ok((request_id, resp)) } + + /// Collect the tables of a namespace in name order, for `table_names`. + /// + /// `table_names` promises name order and resumes after a table name, but the namespace + /// route's `page_token` is opaque -- it belongs to the store the listing walks, and a + /// token this client invented would resume from the wrong place. So the whole namespace is + /// walked by handing each response's token straight back, and the name semantics are + /// applied here. Constructing no token is what makes this work against a server on either + /// side of the change: it only ever repeats what the server said. + /// + /// This is the cost `table_names` already paid -- the server used to enumerate and sort the + /// namespace on every request -- and it is why `list_tables` replaces it. + async fn table_names_in_namespace( + &self, + request: &TableNamesRequest, + ) -> Result<(Vec, ServerVersion)> { + let namespace_id = + build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); + let path = format!("/v1/namespace/{}/table/list", namespace_id); + + let mut names = Vec::new(); + // Every page reports the same server, so keep the first page's version. + let mut version: Option = None; + let mut page_token: Option = None; + loop { + let mut req = self.client.get(&path); + if let Some(ref token) = page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + if version.is_none() { + version = Some(parse_server_version(&request_id, &rsp)?); + } + let response: ListTablesResponse = rsp.json().await.err_to_http(request_id)?; + names.extend(response.tables); + // An empty token is the end of the listing, not a token to send back: a server + // that reads an empty token as "start from the beginning" would hand back the + // first page again. + match response.page_token.filter(|token| !token.is_empty()) { + // A server that repeated a token would never finish; treat that as the end + // rather than looping on it. + Some(token) if Some(&token) != page_token.as_ref() => page_token = Some(token), + _ => break, + } + } + + names.sort(); + if let Some(ref start_after) = request.start_after { + names.retain(|name| name > start_after); + } + if let Some(limit) = request.limit { + names.truncate(limit as usize); + } + Ok((names, version.unwrap_or_default())) + } } #[cfg(all(test, feature = "remote"))] @@ -513,7 +569,7 @@ impl Database for RemoteDatabase { async fn get_function(&self, name: &str, version: &str) -> Result { let req = self .client - .post("/v1/functions/get") + .post("/v1/functions/describe") .json(&serde_json::json!({ "name": name, "version": version, @@ -621,29 +677,29 @@ impl Database for RemoteDatabase { } async fn table_names(&self, request: TableNamesRequest) -> Result> { - let mut req = if !request.namespace_path.is_empty() { - let namespace_id = - build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); - self.client - .get(&format!("/v1/namespace/{}/table/list", namespace_id)) + let (tables, version) = if request.namespace_path.is_empty() { + // The flat route resumes after a table name and orders by name, which is exactly + // what `start_after` means, so the server does the paging. + let mut req = self.client.get("/v1/table/"); + if let Some(limit) = request.limit { + req = req.query(&[("limit", limit)]); + } + if let Some(ref start_after) = request.start_after { + req = req.query(&[("page_token", start_after)]); + } + let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; + let rsp = self.client.check_response(&request_id, rsp).await?; + let version = parse_server_version(&request_id, &rsp)?; + let tables = rsp + .json::() + .await + .err_to_http(request_id)? + .tables; + (tables, version) } else { - self.client.get("/v1/table/") + self.table_names_in_namespace(&request).await? }; - if let Some(limit) = request.limit { - req = req.query(&[("limit", limit)]); - } - if let Some(start_after) = request.start_after { - req = req.query(&[("page_token", start_after)]); - } - let (request_id, rsp) = self.client.send_with_retry(req, None, true).await?; - let rsp = self.client.check_response(&request_id, rsp).await?; - let version = parse_server_version(&request_id, &rsp)?; - let tables = rsp - .json::() - .await - .err_to_http(request_id)? - .tables; for table in &tables { let table_identifier = build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter); @@ -1227,6 +1283,101 @@ mod tests { assert_eq!(names, vec!["table1", "table2"]); } + #[tokio::test] + async fn test_table_names_in_a_namespace_never_invents_a_page_token() { + // The namespace route's token belongs to the store, so `table_names` cannot build one + // from `start_after`. It walks the namespace on the server's own tokens and applies the + // name semantics itself, which is what keeps it working either side of the change. + let page = Arc::new(AtomicUsize::new(0)); + let conn = Connection::new_with_handler(move |request| { + assert_eq!(request.url().path(), "/v1/namespace/ns/table/list"); + let query = request.url().query().unwrap_or(""); + assert!( + !query.contains("page_token=users"), + "a table name must never be sent as a page token: {query}" + ); + match page.fetch_add(1, Ordering::SeqCst) { + 0 => { + assert!( + !query.contains("page_token"), + "the walk starts with no token" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["users", "orders"], "page_token": "opaque-1"}"#) + .unwrap() + } + _ => { + assert!(query.contains("page_token=opaque-1")); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["widgets"]}"#) + .unwrap() + } + } + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .start_after("users") + .execute() + .await + .unwrap(); + // Name order, resumed after "users": "orders" sorts before it and is dropped. + assert_eq!(names, vec!["widgets"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_a_repeated_token() { + // A server that handed back the token it was given would never finish the walk. + let conn = Connection::new_with_handler(|_request| { + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": "same"}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + // The guard bounds the walk instead of letting it run forever. The repeat is the + // server breaking the token contract and is not papered over here. + assert_eq!(names, vec!["a", "a"]); + } + + #[tokio::test] + async fn test_table_names_in_a_namespace_stops_on_an_empty_token() { + // An empty token ends the listing. Sending it back would ask a server that reads it + // as "start from the beginning" for the first page a second time, and every name on + // that page would be collected twice. + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let conn = Connection::new_with_handler(move |request| { + seen.fetch_add(1, Ordering::SeqCst); + assert!( + !request.url().query().unwrap_or("").contains("page_token"), + "an empty token must never be sent back" + ); + http::Response::builder() + .status(200) + .body(r#"{"tables": ["a"], "page_token": ""}"#) + .unwrap() + }); + + let names = conn + .table_names() + .namespace(vec!["ns".to_string()]) + .execute() + .await + .unwrap(); + assert_eq!(names, vec!["a"]); + assert_eq!(requests.load(Ordering::SeqCst), 1); + } + #[tokio::test] async fn test_table_names_pagination() { let conn = Connection::new_with_handler(|request| { @@ -2520,7 +2671,7 @@ mod tests { ); let conn = Connection::new_with_handler(|request| { assert_eq!(request.method(), &reqwest::Method::POST); - assert_eq!(request.url().path(), "/v1/functions/get"); + assert_eq!(request.url().path(), "/v1/functions/describe"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); assert_eq!( diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index f98d4c8dd..35ac56970 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob; use crate::table::AddColumnsResult; use crate::table::AddResult; use crate::table::BranchDiff; +use crate::table::CherryPickResult; use crate::table::DeleteResult; use crate::table::DropColumnsResult; use crate::table::LsmStats; use crate::table::LsmWriteSpec; -use crate::table::MergeBranchResult; use crate::table::MergeResult; use crate::table::Tags; use crate::table::UpdateResult; @@ -1775,6 +1775,18 @@ impl BaseTable for RemoteTable { Ok(()) } + async fn snapshot_at_current_version(&self) -> Result>> { + // A checked-out handle already names its snapshot. Otherwise resolve + // latest exactly once before creating the independent pinned handle. + let version = match self.current_version().await { + Some(version) => version, + None => self.describe().await?.version, + }; + + let snapshot = self.with_branch(self.branch.clone()); + *snapshot.version.write().await = Some(version); + Ok(Some(Arc::new(snapshot))) + } async fn restore(&self) -> Result<()> { let mut request = self .client @@ -2031,7 +2043,7 @@ impl BaseTable for RemoteTable { async fn diff_branch(&self, from_branch: &str) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self @@ -2058,20 +2070,23 @@ impl BaseTable for RemoteTable { }) } - async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result { + async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { if from_branch.trim().is_empty() { return Err(Error::InvalidInput { - message: "from_branch must be a non-empty string".into(), + message: "Branch name cannot be empty.".into(), }); } let request = self .client - .post(&format!("/v1/table/{}/branches/merge/", self.identifier)) + .post(&format!( + "/v1/table/{}/branches/cherry_pick/", + self.identifier + )) .json(&serde_json::json!({ "from_branch": from_branch, "dry_run": dry_run, })); - // No retry. 409 rejected merge is final and carries a body. + // No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error. let (request_id, response) = self.send(request, false).await?; let status = response.status(); if status == StatusCode::NOT_FOUND { @@ -2080,11 +2095,11 @@ impl BaseTable for RemoteTable { source: format!("branch '{}' does not exist", from_branch).into(), }); } - // 200 and 409 both carry MergeBranchResult. + // 200 and 409 both carry CherryPickResult. if status != StatusCode::OK && status != StatusCode::CONFLICT { let body = response.text().await.unwrap_or_default(); return Err(Error::Http { - source: format!("unexpected status {status} from merge_branch: {body}").into(), + source: format!("unexpected status {status} from cherry_pick: {body}").into(), request_id, status_code: Some(status), }); @@ -2092,7 +2107,7 @@ impl BaseTable for RemoteTable { let body = response.text().await.err_to_http(request_id.clone())?; serde_json::from_str(&body).map_err(|err| Error::Http { source: format!( - "Failed to parse merge_branch response: {}, body: {}", + "Failed to parse cherry_pick response: {}, body: {}", err, body ) .into(), @@ -2861,7 +2876,10 @@ impl BaseTable for RemoteTable { }) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { self.check_mutable().await?; let mut body = serde_json::json!({ "column": column }); self.apply_branch_body(&mut body); @@ -2882,7 +2900,7 @@ impl BaseTable for RemoteTable { status_code: None, })?; - Ok(Job::new(Box::new(FreshnessJob { + Ok(Job::new_typed(Box::new(FreshnessJob { inner: RemoteJob::new(self.client.clone(), response.job_id), freshness: self.freshness.clone(), version: self.version.clone(), @@ -3277,6 +3295,21 @@ mod tests { }, }; + fn refresh_done(job_id: &str) -> String { + json!({ + "job_id": job_id, + "job_state": "DONE", + "result": { + "rows_assigned": 12, + "rows_failed": 0, + "rows_remaining": 0, + "source_version": 7, + "published_version": 8, + } + }) + .to_string() + } + #[tokio::test] async fn test_not_found() { let table = Table::new_with_handler("my_table", |_| { @@ -4316,6 +4349,43 @@ mod tests { assert!(!table.base_table().scan_order_is_deterministic()); } + #[tokio::test] + async fn test_checkout_branch_pins_without_touching_the_original() { + let seen = Arc::new(std::sync::Mutex::new(Vec::new())); + let recorder = seen.clone(); + let table = Table::new_with_handler_version( + "my_table", + semver::Version::new(0, 5, 0), + move |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(br#"{"version": 42, "schema": {"fields": []}}"#.to_vec()) + .unwrap(), + "/v1/table/my_table/count_rows/" => { + let body = request_body_json(&request); + recorder.lock().unwrap().push(body["version"].clone()); + http::Response::builder() + .status(200) + .body(b"0".to_vec()) + .unwrap() + } + path => panic!("unexpected request path: {path}"), + }, + ); + + let pinned = table.checkout_branch("main", Some(42)).await.unwrap(); + pinned.count_rows(None).await.unwrap(); + table.count_rows(None).await.unwrap(); + + let seen = seen.lock().unwrap(); + assert_eq!(seen[0], 42, "the pinned handle must send its version"); + assert!( + seen[1].is_null(), + "the original handle must still track latest, got {:?}", + seen[1] + ); + } + #[tokio::test] async fn test_fetch_blobs_sends_the_checked_out_version() { let ipc = one_row_blob_ipc_stream("image"); @@ -6732,6 +6802,53 @@ mod tests { assert_eq!(result.version, 8); } + #[tokio::test] + async fn test_add_fixed_size_list_function_column_declares_the_vector_type() { + let table = Table::new_with_handler("my_table", |request| { + match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body( + r#"{"version":1,"schema":{"fields":[{"name":"description","nullable":true,"type":{"type":"string"}}]}}"#, + ) + .unwrap(), + "/v1/table/my_table/add_columns/" => { + let actual: serde_json::Value = serde_json::from_slice( + request.body().unwrap().as_bytes().unwrap(), + ) + .unwrap(); + let expected: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json" + )) + .unwrap(); + assert_eq!(actual, expected); + http::Response::builder() + .status(200) + .body(r#"{"version":8}"#) + .unwrap() + } + path => panic!("Unexpected path: {path}"), + } + }); + let application = crate::function::FunctionApplication::from_json( + r#"{ + "function":{"name":"embed","version":"fv_01K3EXACT"}, + "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], + "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false}, + "group_id":"fg_fixed" + }"#, + ) + .unwrap(); + + let result = table + .add_columns() + .function_as("embedding", application) + .execute() + .await + .unwrap(); + assert_eq!(result.version, 8); + } + #[tokio::test] async fn test_add_named_struct_function_expands_one_atomic_sibling_group() { let table = Table::new_with_handler("my_table", |request| match request.url().path() { @@ -6889,7 +7006,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-7")) .unwrap(), "/v1/table/my_table/count_rows/" => { saw.store( @@ -6905,7 +7022,9 @@ mod tests { }); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 12); + assert_eq!(result.published_version, Some(8)); table.count_rows(None).await.unwrap(); assert!( saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst), @@ -6927,7 +7046,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-8")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -6973,7 +7092,7 @@ mod tests { .unwrap(), "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-9")) .unwrap(), "/v1/table/my_table/tags/version/" => http::Response::builder() .status(200) @@ -7037,7 +7156,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-10")) .unwrap(), "/v1/table/my_table/describe/" => { let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]); @@ -7110,7 +7229,7 @@ mod tests { } "/v1/jobs/describe" => http::Response::builder() .status(200) - .body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string()) + .body(refresh_done("j-11")) .unwrap(), "/v1/table/my_table/count_rows/" => { *saw.lock().unwrap() = request @@ -10410,6 +10529,20 @@ mod tests { ); } + #[tokio::test] + async fn test_materialized_view_refused_without_a_request() { + // Materialized views are local-only. The table-level entry the + // bindings use must refuse a remote table before reading its schema, + // so the panicking handler is the assertion. + let table = Table::new_with_handler("my_table", |request| -> http::Response { + panic!("unexpected request: {}", request.url().path()) + }); + let err = crate::MaterializedView::from_table(table) + .await + .unwrap_err(); + assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); + } + #[tokio::test] async fn test_create_branch_empty_name_rejected_client_side() { use lance::dataset::refs::Ref; @@ -10499,8 +10632,7 @@ mod tests { "changedColumns":[], "addedIndexes":[], "removedIndexes":[], - "mergeable":true, - "mergeBlockers":[] + "errors":[] }"# } @@ -10518,15 +10650,18 @@ mod tests { }); let diff = table.diff_branch("exp").await.unwrap(); assert_eq!(diff.from_branch, "exp"); - assert!(diff.mergeable); + assert!(diff.errors.is_empty()); assert_eq!(diff.added_columns.len(), 1); assert_eq!(diff.added_columns[0].name, "tag"); } #[tokio::test] - async fn test_merge_branch_dry_run() { + async fn test_cherry_pick_dry_run() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["from_branch"], "exp"); assert_eq!(body["dry_run"], true); @@ -10536,27 +10671,29 @@ mod tests { ); http::Response::builder().status(200).body(resp).unwrap() }); - let result = table.merge_branch("exp", true).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Ready); + let result = table.cherry_pick("exp", true).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Ready); assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]); assert!(result.main_version_after.is_none()); } #[tokio::test] - async fn test_merge_branch_rejected_returns_ok_with_body() { + async fn test_cherry_pick_failed_returns_ok_with_body() { let table = Table::new_with_handler("my_table", |request| { - assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/"); + assert_eq!( + request.url().path(), + "/v1/table/my_table/branches/cherry_pick/" + ); let body = request_body_json(&request); assert_eq!(body["dry_run"], false); let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "baseMoved", "message": "main has advanced" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "promotedColumns": [] } }); @@ -10565,24 +10702,23 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); - assert!(!result.diff.mergeable); - assert_eq!(result.diff.merge_blockers.len(), 1); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); + assert!(!result.diff.errors.is_empty()); + assert_eq!(result.diff.errors.len(), 1); } #[tokio::test] - async fn test_merge_branch_unknown_blocker_code_parses() { + async fn test_cherry_pick_unknown_error_code_parses() { let table = Table::new_with_handler("my_table", |_| { let mut diff: serde_json::Value = serde_json::from_str(sample_branch_diff_json()).unwrap(); - diff["mergeable"] = serde_json::json!(false); - diff["mergeBlockers"] = serde_json::json!([{ + diff["errors"] = serde_json::json!([{ "code": "multipleCommits", "message": "branch has more than one data commit" }]); let resp = serde_json::json!({ - "status": "rejected", + "status": "failed", "diff": diff, "preview": { "operation": "append", "rowsAdded": 2 } }); @@ -10591,24 +10727,24 @@ mod tests { .body(resp.to_string()) .unwrap() }); - let result = table.merge_branch("exp", false).await.unwrap(); - assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected); + let result = table.cherry_pick("exp", false).await.unwrap(); + assert_eq!(result.status, crate::table::CherryPickStatus::Failed); assert_eq!( - result.diff.merge_blockers[0].code, - crate::table::MergeBlockerCode::Unknown + result.diff.errors[0].code, + crate::table::CherryPickErrorCode::Unknown ); assert!(result.preview.promoted_columns.is_empty()); } #[tokio::test] - async fn test_merge_branch_unexpected_2xx_is_error() { + async fn test_cherry_pick_unexpected_2xx_is_error() { let table = Table::new_with_handler("my_table", |_| { http::Response::builder() .status(204) .body(String::new()) .unwrap() }); - let err = table.merge_branch("exp", false).await.unwrap_err(); + let err = table.cherry_pick("exp", false).await.unwrap_err(); match err { Error::Http { status_code: Some(code), diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 096d60345..356c87bbe 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder; pub mod add_columns; mod add_data; -pub mod branch_merge; pub mod checkpoint; +pub mod cherry_pick; pub mod computed_columns; mod create_index; pub mod datafusion; @@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder; #[cfg(feature = "remote")] pub(crate) use add_data::PreprocessingOutput; pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior}; -pub use branch_merge::{ - BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode, - MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary, +pub use cherry_pick::{ + BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult, + CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary, }; pub use chrono::Duration; pub use computed_columns::{ @@ -771,7 +771,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { } /// Fill a computed column's unfilled rows, returning a [`Job`] tracking /// the operation. - async fn refresh_column_async(&self, _column: &str) -> Result { + async fn refresh_column_async( + &self, + _column: &str, + ) -> Result> { Err(Error::NotSupported { message: "computed columns are supported only on local tables".into(), }) @@ -789,6 +792,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { async fn checkout_tag(&self, tag: &str) -> Result<()>; /// Checkout the latest version of the table. async fn checkout_latest(&self) -> Result<()>; + /// Return an independent handle pinned to the version currently selected. + /// + /// Backends that can advance between requests should override this for + /// multi-request operations that need snapshot consistency. Backends whose + /// existing handles already provide the desired behavior return `None`. + async fn snapshot_at_current_version(&self) -> Result>> { + Ok(None) + } /// Whether repeated identical scans return rows in the same order. /// /// Callers that assign meaning to a row's position must order the results @@ -832,14 +843,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Diff a branch against main. Remote only. async fn diff_branch(&self, _from_branch: &str) -> Result { Err(Error::NotSupported { - message: "diff_branch is only supported on remote tables".into(), + message: "Branch diffs are only supported on Enterprise tables.".into(), }) } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result { + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result { Err(Error::NotSupported { - message: "merge_branch is only supported on remote tables".into(), + message: "Cherry-picking branches is only supported on Enterprise tables.".into(), }) } /// The branch this handle is scoped to, or `None` for `main`. @@ -1066,6 +1077,11 @@ impl Table { self.database.as_ref().unwrap() } + /// The database this handle was opened through, when it was. + pub fn database_opt(&self) -> Option<&Arc> { + self.database.as_ref() + } + pub fn embedding_registry(&self) -> &Arc { &self.embedding_registry } @@ -1703,7 +1719,9 @@ impl Table { /// operation instead of blocking until it completes. /// /// The job may already be complete when returned, and callers must not - /// assume the column is filled until [`Job::wait`] returns. Invalid input + /// assume the column is filled until [`Job::wait`] returns. A successful + /// wait returns the durable [`crate::function::RefreshColumnResult`] for + /// both expression-backed and Function-backed columns. Invalid input /// -- an unknown column, or one that is not computed -- is reported by /// this call rather than by the job. On local tables the job runs as an /// in-process task; on LanceDB Cloud and Enterprise it is the server's @@ -1714,11 +1732,15 @@ impl Table { /// # async fn refresh_in_background(table: &Table) -> Result<(), Box> { /// let job = table.refresh_column_async("doubled").await?; /// println!("refresh running: {:?}", job.status().await?); - /// job.wait().await?; + /// let result = job.wait().await?; + /// println!("assigned {} rows", result.rows_assigned); /// # Ok(()) /// # } /// ``` - pub async fn refresh_column_async(&self, column: impl AsRef) -> Result { + pub async fn refresh_column_async( + &self, + column: impl AsRef, + ) -> Result> { self.inner.refresh_column_async(column.as_ref()).await } @@ -2258,14 +2280,10 @@ impl Table { self.inner.diff_branch(from_branch).await } - /// Merge a branch into main, or dry-run. Remote only. - /// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`]. - pub async fn merge_branch( - &self, - from_branch: &str, - dry_run: bool, - ) -> Result { - self.inner.merge_branch(from_branch, dry_run).await + /// Cherry-pick a branch onto main, or dry-run. Remote only. + /// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`]. + pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result { + self.inner.cherry_pick(from_branch, dry_run).await } /// The branch this handle is scoped to, or `None` for `main`. @@ -3424,7 +3442,10 @@ impl BaseTable for NativeTable { Ok(result) } - async fn refresh_column_async(&self, column: &str) -> Result { + async fn refresh_column_async( + &self, + column: &str, + ) -> Result> { refresh::execute_refresh_column_async(self, column).await } diff --git a/rust/lancedb/src/table/branch_merge.rs b/rust/lancedb/src/table/cherry_pick.rs similarity index 86% rename from rust/lancedb/src/table/branch_merge.rs rename to rust/lancedb/src/table/cherry_pick.rs index ad81ab64c..93bc76637 100644 --- a/rust/lancedb/src/table/branch_merge.rs +++ b/rust/lancedb/src/table/cherry_pick.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -//! Types for remote branch diff / merge against main. +//! Types for remote branch diff / cherry-pick onto main. use serde::{Deserialize, Serialize}; @@ -44,13 +44,13 @@ pub struct RowCountSummary { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBlockerCode { +pub enum CherryPickErrorCode { BaseMoved, RowCountMismatch, RowsChanged, ColumnRemoved, ColumnChanged, - NoMergeableChanges, + NothingToApply, NoColumnChanges, InputColumnDependency, ParentNotMain, @@ -60,8 +60,8 @@ pub enum MergeBlockerCode { #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBlocker { - pub code: MergeBlockerCode, +pub struct CherryPickError { + pub code: CherryPickErrorCode, pub message: String, } @@ -81,34 +81,33 @@ pub struct BranchDiff { pub changed_columns: Vec, pub added_indexes: Vec, pub removed_indexes: Vec, - pub mergeable: bool, - pub merge_blockers: Vec, + pub errors: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergePreview { +pub struct CherryPickPreview { #[serde(default)] pub promoted_columns: Vec, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub enum MergeBranchStatus { +pub enum CherryPickStatus { Ready, - Rejected, + Failed, NotImplemented, - Merged, + CherryPicked, #[serde(other)] Unknown, } #[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct MergeBranchResult { - pub status: MergeBranchStatus, +pub struct CherryPickResult { + pub status: CherryPickStatus, pub diff: BranchDiff, - pub preview: MergePreview, + pub preview: CherryPickPreview, #[serde(default, skip_serializing_if = "Option::is_none")] pub main_version_after: Option, } diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index b1c0b8370..133f37a55 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -586,6 +586,25 @@ fn canonical_input_arrow_type(field: &JsonArrowField) -> Result { } } +/// `fixed_size_list` -> (`item`, `size`); the comma must sit outside +/// any nested `<...>`. +fn split_fixed_size_list(raw: &str) -> Option<(&str, i32)> { + let inner = raw.strip_prefix("fixed_size_list<")?.strip_suffix('>')?; + let mut depth = 0_u32; + let mut separator = None; + for (index, byte) in inner.bytes().enumerate() { + match byte { + b'<' => depth += 1, + b'>' => depth = depth.checked_sub(1)?, + b',' if depth == 0 => separator = Some(index), + _ => {} + } + } + let (item, size) = inner.split_at(separator?); + let size: i32 = size[1..].trim().parse().ok()?; + (size > 0).then_some((item.trim(), size)) +} + fn parse_output_arrow_type(raw: &str) -> Result { fn parse(raw: &str) -> Result { let raw = raw.trim(); @@ -618,6 +637,16 @@ fn parse_output_arrow_type(raw: &str) -> Result { )]); return Ok(data_type); } + if let Some((inner, size)) = split_fixed_size_list(raw) { + let mut data_type = JsonArrowDataType::new("fixed_size_list".to_string()); + data_type.fields = Some(vec![JsonArrowField::new( + "item".to_string(), + false, + parse(inner)?, + )]); + data_type.length = Some(i64::from(size)); + return Ok(data_type); + } let normalized = match raw { "boolean" => "bool", "string" => "utf8", @@ -1322,6 +1351,32 @@ pub(super) async fn add_foreign_kind(table: &crate::Table, name: &str, kind: &st #[cfg(test)] mod tests { + #[test] + fn output_arrow_type_grammar_matches_the_shared_golden() { + let golden: serde_json::Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/arrow_types.json" + )) + .unwrap(); + let valid = golden["valid"].as_array().unwrap().iter(); + for case in valid.chain(golden["server_only"].as_array().unwrap()) { + let raw = case["arrow_type"].as_str().unwrap(); + let parsed = super::parse_output_arrow_type(raw) + .unwrap_or_else(|error| panic!("{raw}: {error}")); + assert_eq!( + serde_json::to_value(&parsed).unwrap(), + case["json"], + "{raw}" + ); + } + for raw in golden["invalid"].as_array().unwrap() { + let raw = raw.as_str().unwrap(); + assert!( + super::parse_output_arrow_type(raw).is_err(), + "{raw:?} should be rejected" + ); + } + } + use arrow_array::record_batch; use arrow_schema::DataType; use futures::TryStreamExt; diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 144c6dbfb..e373522bc 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -12,6 +12,7 @@ use arrow_schema::{DataType, Field}; use lance::index::DatasetIndexExt; use lance::index::vector::VectorIndexParams; use lance::index::vector::utils::infer_vector_dim; +use lance_arrow::json::is_json_field; use lance_index::IndexType; use lance_index::scalar::{BuiltinIndexType, ScalarIndexParams}; use lance_index::vector::bq::RQBuildParams; @@ -219,6 +220,14 @@ impl NativeTable { ))) } Index::Bitmap(_) => { + if is_json_field(field) { + return Err(Error::Schema { + message: format!( + "A BITMAP index cannot be created on the whole-document lance.json field `{}`. Create a JSON-path scalar index for structured equality or range predicates, or use FTS for document search", + field.name() + ), + }); + } Self::validate_index_type(field, "Bitmap", supported_bitmap_data_type)?; Ok(Box::new(ScalarIndexParams::for_builtin( BuiltinIndexType::Bitmap, @@ -1465,6 +1474,35 @@ mod tests { assert_eq!(stats.distance_type, None); } + #[tokio::test] + async fn test_create_bitmap_index_rejects_lance_json() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![lance_arrow::json::json_field( + "metadata", true, + )])); + let table = conn + .create_empty_table("json_bitmap", schema) + .execute() + .await + .unwrap(); + + let err = table + .create_index(&["metadata"], Index::Bitmap(Default::default())) + .execute() + .await + .expect_err("a whole-document lance.json field must not support a bitmap index"); + let message = err.to_string(); + assert!( + message.contains("lance.json"), + "unexpected error: {message}" + ); + assert!( + message.contains("JSON-path scalar index"), + "unexpected error: {message}" + ); + assert!(message.contains("FTS"), "unexpected error: {message}"); + } + #[tokio::test] async fn test_create_label_list_index() { let conn = connect("memory://").execute().await.unwrap(); diff --git a/rust/lancedb/src/table/merge.rs b/rust/lancedb/src/table/merge.rs index ea68e99df..b9dd5732b 100644 --- a/rust/lancedb/src/table/merge.rs +++ b/rust/lancedb/src/table/merge.rs @@ -321,7 +321,8 @@ pub(crate) async fn execute_merge_insert( mod tests { use arrow_array::builder::FixedSizeBinaryBuilder; use arrow_array::{ - Int32Array, RecordBatch, RecordBatchIterator, RecordBatchReader, StringArray, UInt64Array, + FixedSizeListArray, Int32Array, NullArray, RecordBatch, RecordBatchIterator, + RecordBatchReader, StringArray, UInt32Array, UInt64Array, }; use arrow_schema::{DataType, Field, Schema}; use std::sync::Arc; @@ -529,6 +530,74 @@ mod tests { assert_eq!(result.num_deleted_rows, 5); assert_eq!(table.count_rows(None).await.unwrap(), 5); } + + #[tokio::test] + async fn test_merge_insert_fixed_size_list_above_u32_child_count() { + // Arrow's FixedSizeList take kernel uses u32 child indices. Previously, + // delete-by-source materialized the target payload in a full outer join, + // causing the final list below to overflow those indices and panic. + // A Null child keeps this boundary test small in memory. + const LIST_SIZE: i32 = 65_536; + const ROW_COUNT: usize = (u32::MAX as usize / LIST_SIZE as usize) + 1; + const BATCH_SIZE: usize = 8_192; + + let item = Arc::new(Field::new("item", DataType::Null, true)); + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new( + "vector", + DataType::FixedSizeList(item.clone(), LIST_SIZE), + false, + ), + ])); + let batch = |start: usize, len: usize| { + RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(UInt32Array::from_iter_values( + start as u32..(start + len) as u32, + )), + Arc::new(FixedSizeListArray::new( + item.clone(), + LIST_SIZE, + Arc::new(NullArray::new(len * LIST_SIZE as usize)), + None, + )), + ], + ) + .unwrap() + }; + + let target_batches = (0..ROW_COUNT) + .step_by(BATCH_SIZE) + .map(|start| { + let len = (ROW_COUNT - start).min(BATCH_SIZE); + Ok(batch(start, len)) + }) + .collect::>(); + let target_data: Box = + Box::new(RecordBatchIterator::new(target_batches, schema.clone())); + let conn = connect("memory://").execute().await.unwrap(); + let table = conn + .create_table("fixed_size_list_overflow", target_data) + .execute() + .await + .unwrap(); + + let source = batch(ROW_COUNT - 1, 1); + let mut merge = table.merge_insert(&["id"]); + merge + .when_matched_update_all(None) + .when_not_matched_by_source_delete(None); + let result = merge + .execute(Box::new(RecordBatchIterator::new([Ok(source)], schema))) + .await + .unwrap(); + + assert_eq!(result.num_updated_rows, 1); + assert_eq!(result.num_deleted_rows, (ROW_COUNT - 1) as u64); + assert_eq!(table.count_rows(None).await.unwrap(), 1); + } } #[cfg(test)] diff --git a/rust/lancedb/src/table/merge/lsm.rs b/rust/lancedb/src/table/merge/lsm.rs index 44a2874de..a06507ba0 100644 --- a/rust/lancedb/src/table/merge/lsm.rs +++ b/rust/lancedb/src/table/merge/lsm.rs @@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec) .into(), }); } + if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() { + return Err(Error::NotSupported { + message: "an LSM write spec cannot be installed on a materialized view: \ + rows in un-compacted tiers are invisible to refresh" + .into(), + }); + } let mut builder = dataset.initialize_mem_wal(); let writer_config_defaults = match spec { LsmWriteSpec::Bucket { diff --git a/rust/lancedb/src/table/query.rs b/rust/lancedb/src/table/query.rs index 9feb9d5ab..9b81786cd 100644 --- a/rust/lancedb/src/table/query.rs +++ b/rust/lancedb/src/table/query.rs @@ -70,6 +70,9 @@ async fn can_execute_namespace_query(table: &NativeTable, query: &AnyQuery) -> R .contains(&NamespaceClientPushdownOperation::QueryTable) && table.namespace_client.is_some() && table.dataset.current_branch().is_none() + // NsQueryTableRequest has no version field, so a pushed-down query would + // read latest and ignore the pin. + && table.dataset.time_travel_version().is_none() && !requires_local_namespace_execution(query)) { return Ok(false); diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index a3f3da92f..35f883411 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -49,11 +49,25 @@ pub struct RefreshColumnResult { pub version: u64, } +struct RefreshExecution { + result: RefreshColumnResult, + source_version: u64, +} + /// Internal implementation of the refresh logic. pub(crate) async fn execute_refresh_column( table: &NativeTable, column: &str, ) -> Result { + Ok(execute_refresh_column_with_source(table, column) + .await? + .result) +} + +async fn execute_refresh_column_with_source( + table: &NativeTable, + column: &str, +) -> Result { table.dataset.ensure_mutable()?; ensure_no_lsm_write_spec(table).await?; let dataset = table.dataset.get().await?; @@ -87,9 +101,13 @@ pub(crate) async fn execute_refresh_column( } if replacements.is_empty() { - return Ok(RefreshColumnResult { - rows_filled: 0, - version: dataset.version().version, + let source_version = dataset.version().version; + return Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled: 0, + version: source_version, + }, + source_version, }); } @@ -110,14 +128,20 @@ pub(crate) async fn execute_refresh_column( let version = new_dataset.version().version; table.dataset.update(new_dataset); - Ok(RefreshColumnResult { - rows_filled, - version, + Ok(RefreshExecution { + result: RefreshColumnResult { + rows_filled, + version, + }, + source_version: read_version, }) } /// Run the refresh as a [`Job`] in this process. -pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &str) -> Result { +pub(crate) async fn execute_refresh_column_async( + table: &NativeTable, + column: &str, +) -> Result> { // Validate before spawning so bad input is reported by this call rather // than only by the job. table.dataset.ensure_mutable()?; @@ -129,9 +153,16 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s let table = table.clone(); let column = column.to_string(); Ok(Job::spawned(tokio::spawn(async move { - execute_refresh_column(&table, &column).await?; + let execution = execute_refresh_column_with_source(&table, &column).await?; table.bump_freshness(); - Ok(()) + Ok(crate::function::RefreshColumnResult { + rows_assigned: execution.result.rows_filled, + rows_failed: 0, + rows_remaining: 0, + source_version: execution.source_version, + published_version: (execution.result.rows_filled > 0) + .then_some(execution.result.version), + }) }))) } @@ -193,7 +224,7 @@ fn declared_expression(dataset: &Dataset, column: &str) -> Result { /// /// Lance's dialect delimits with backticks, so a double-quoted name would /// parse as a string literal rather than a column. -fn quote_identifier(name: &str) -> String { +pub(crate) fn quote_identifier(name: &str) -> String { format!("`{}`", name.replace('`', "``")) } @@ -396,6 +427,17 @@ mod tests { read(&table, "doubled").await, vec![Some(2), Some(4), Some(6)] ); + + let no_op = table + .refresh_column_async("doubled") + .await + .unwrap() + .wait() + .await + .unwrap(); + assert_eq!(no_op.rows_assigned, 0); + assert_eq!(no_op.source_version, 3); + assert_eq!(no_op.published_version, None); } /// Values written after the last refresh must be reachable by another one. @@ -597,7 +639,7 @@ mod tests { /// A fragment spanning several scan batches exercises the streamed fill: /// the probe buffers only until the first gained value and the rest flows - /// through write_column a batch at a time. + /// through write_columns a batch at a time. #[tokio::test] async fn test_refresh_streams_a_multi_batch_fragment() { let values: Vec = (0..20_000).collect(); @@ -646,7 +688,12 @@ mod tests { let job = table.refresh_column_async("doubled").await.unwrap(); assert!(job.id().is_none(), "in-process jobs have no server id"); - job.wait().await.unwrap(); + let result = job.wait().await.unwrap(); + assert_eq!(result.rows_assigned, 3); + assert_eq!(result.rows_failed, 0); + assert_eq!(result.rows_remaining, 0); + assert_eq!(result.source_version, 2); + assert_eq!(result.published_version, Some(3)); assert_eq!(job.status().await.unwrap(), "finished"); assert_eq!( read(&table, "doubled").await, @@ -672,9 +719,9 @@ mod tests { declare_doubled(&table).await.unwrap(); let job = table.refresh_column_async("doubled").await.unwrap(); - job.wait().await.unwrap(); + let first = job.wait().await.unwrap(); // A second wait after completion observes the same outcome. - job.wait().await.unwrap(); + assert_eq!(job.wait().await.unwrap(), first); assert_eq!(job.status().await.unwrap(), "finished"); } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json new file mode 100644 index 000000000..ff26e4c08 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/arrow_types.json @@ -0,0 +1,333 @@ +{ + "valid": [ + { + "arrow_type": "bool", + "json": { + "type": "bool" + } + }, + { + "arrow_type": "int8", + "json": { + "type": "int8" + } + }, + { + "arrow_type": "int16", + "json": { + "type": "int16" + } + }, + { + "arrow_type": "int32", + "json": { + "type": "int32" + } + }, + { + "arrow_type": "int64", + "json": { + "type": "int64" + } + }, + { + "arrow_type": "uint8", + "json": { + "type": "uint8" + } + }, + { + "arrow_type": "uint16", + "json": { + "type": "uint16" + } + }, + { + "arrow_type": "uint32", + "json": { + "type": "uint32" + } + }, + { + "arrow_type": "uint64", + "json": { + "type": "uint64" + } + }, + { + "arrow_type": "float16", + "json": { + "type": "float16" + } + }, + { + "arrow_type": "float32", + "json": { + "type": "float32" + } + }, + { + "arrow_type": "float64", + "json": { + "type": "float64" + } + }, + { + "arrow_type": "utf8", + "json": { + "type": "utf8" + } + }, + { + "arrow_type": "binary", + "json": { + "type": "binary" + } + }, + { + "arrow_type": "date32", + "json": { + "type": "date32" + } + }, + { + "arrow_type": "date64", + "json": { + "type": "date64" + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int64" + } + } + ] + } + }, + { + "arrow_type": "list", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "large_list", + "json": { + "type": "large_list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "utf8" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 384, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 8, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float16" + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list", + "json": { + "type": "fixed_size_list", + "length": 1, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "uint8" + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "fixed_size_list, 2>", + "json": { + "type": "fixed_size_list", + "length": 2, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "int32" + } + } + ] + } + } + ] + } + }, + { + "arrow_type": "list>", + "json": { + "type": "list", + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + } + } + ], + "server_only": [ + { + "arrow_type": "null", + "json": { + "type": "null" + } + } + ], + "invalid": [ + "", + "list<>", + "list[3]", + "fixed_size_list", + "fixed_size_list", + "fixed_size_list", + "map", + "decimal128(10, 2)", + "timestamp[us]", + "struct" + ] +} \ No newline at end of file diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json new file mode 100644 index 000000000..7d92f1454 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json @@ -0,0 +1,79 @@ +{ + "new_columns": [ + { + "name": "embedding", + "all_null": true + } + ], + "function": { + "application": { + "function": { + "name": "embed", + "version": "fv_01K3EXACT" + }, + "inputs": [ + { + "parameter": "text", + "kind": "column", + "value": { + "path": "description" + } + } + ], + "output": { + "kind": "scalar", + "arrow_type": "fixed_size_list", + "nullable": false + }, + "group_id": "fg_fixed" + }, + "binding_metadata_version": 1, + "input_bindings": [ + { + "parameter": "text", + "field_path": "description", + "arrow_type": "utf8", + "nullable": true + } + ], + "input_schema": { + "fields": [ + { + "name": "text", + "nullable": true, + "type": { + "type": "utf8" + } + } + ] + }, + "output_schema": { + "fields": [ + { + "name": "embedding", + "nullable": true, + "type": { + "type": "fixed_size_list", + "length": 3, + "fields": [ + { + "name": "item", + "nullable": false, + "type": { + "type": "float32" + } + } + ] + } + } + ] + }, + "outputs": [ + { + "result_field": "$value", + "output_name": "embedding", + "output_ordinal": 0 + } + ] + } +} \ No newline at end of file