Compare commits

...
Author SHA1 Message Date
Lance Release 3a11f4e97c Bump version: 0.38.0-beta.5 → 0.38.0-beta.6 2026-08-23 17:32:40 +00:00
LanceDB Robot 000e3b506b chore: update lance dependency to v11.0.0-beta.22 (#4036)
Updates the Rust workspace Lance dependencies and Java lance-core
dependency to v11.0.0-beta.22, including the refreshed Cargo lockfile.
No compatibility fixes were required; see the [Lance
tag](https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.22).
2026-08-23 10:31:32 -07:00
Lance Release 1b950188c3 Bump version: 0.38.0-beta.4 → 0.38.0-beta.5 2026-08-23 08:07:48 +00:00
LanceDB Robot 6cc77b573c chore: update lance dependency to v11.0.0-beta.21 (#4029)
Updates the Rust workspace and Java `lance-core` dependency to Lance
v11.0.0-beta.21.

No compatibility fixes were required; workspace Clippy passes with
warnings denied. Triggering tag:
https://github.com/lance-format/lance/releases/tag/v11.0.0-beta.21
2026-08-23 00:53:14 -07:00
Weston PaceandClaude Sonnet 4.6 1f1d03f306 feat(python): add backpressure to StreamingDataset post-transform queue (#3897)
Rename prefetch_batches → io_queue_depth and introduce
transform_queue_depth as a symmetric pair: both express "number of
batches to buffer per split at this pipeline stage." The old names are
still accepted as keyword arguments but log a deprecation warning
redirecting callers to the new names.

transform_queue_depth caps how many transform-result batches can
accumulate per split in the post-transform queue. Without this limit a
slow consumer (e.g. a GPU training step) causes cooked rows to pile up
unboundedly. The backpressure check in _try_submit_tx counts both
already-cooked rows and rows expected from in-flight transforms; it
skips proactive transform submission when the combined total reaches the
limit. The reactive _ensure_cooked path bypasses the check so the
consumer never stalls.

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-08-23 15:28:07 +08:00
Lance Release 45cd053478 Bump version: 0.38.0-beta.3 → 0.38.0-beta.4 2026-08-22 16:38:47 +00:00
Wyatt Alt 68749ecfa3 feat(nodejs): materialized view bindings (#3935)
Exposes materialized views to TypeScript: createMaterializedView,
openMaterializedView and listMaterializedViews on Connection, and a
MaterializedView handle carrying the parsed definition and
refresh({full, sourceVersion}), which returns the typed refresh result.
select accepts column names, [alias, expression] pairs, or a record of
the
same; the definition reads back off the stored schema, so a reopened
handle
needs no side channel. Remote connections surface the core's
not-supported
error up front.

The napi crate needed the same recursion-limit raise as the core crate:
the
refresh future's type graph overflows the default trait-recursion depth.


<sub>Stack created with <a
href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a
href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
2026-08-21 23:48:43 -07:00
37 changed files with 1187 additions and 95 deletions
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.3"
current_version = "0.38.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Generated
+49 -45
View File
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5222,7 +5222,11 @@ dependencies = [
"pin-project",
"prost",
"rand 0.9.5",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-google",
"serde",
"serde_json",
"tempfile",
"tokio",
"tracing",
@@ -5232,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5247,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"async-trait",
@@ -5260,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5314,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5329,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5370,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5384,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.19"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.19#3128c0024427cb5bf8c04d492893ae45e78b0511"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5398,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.5"
dependencies = [
"ahash",
"anyhow",
@@ -5486,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0-beta.3"
version = "0.38.0-beta.5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5511,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.5"
dependencies = [
"arrow",
"async-trait",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.19", default-features = false, "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.19", "tag" = "v11.0.0-beta.19", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
</dependency>
```
+75 -5
View File
@@ -169,6 +169,45 @@ Creates a new empty Table
***
### createMaterializedView()
```ts
abstract createMaterializedView(
name,
source,
options?): Promise<MaterializedView>
```
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
#### Parameters
* **name**: `string`
* **source**: `string`
* **options?**
* **options.limit?**: `number`
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
* **options.where?**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### createNamespace()
```ts
@@ -499,6 +538,22 @@ List server-side jobs across the database's tables.
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### listNamespaces()
```ts
@@ -529,6 +584,26 @@ Child namespace names and
***
### openMaterializedView()
```ts
abstract openMaterializedView(name): Promise<MaterializedView>
```
Open the materialized view named `name`.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### openTable()
```ts
@@ -538,18 +613,13 @@ abstract openTable(
options?): Promise<Table>
```
Open a table in the database.
#### Parameters
* **name**: `string`
The name of the table
* **namespacePath?**: `string`[]
The namespace path of the table (defaults to root namespace)
* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
Additional options
#### Returns
+101
View File
@@ -0,0 +1,101 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedView
# Class: MaterializedView
A handle on a materialized view: its table plus its definition.
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
## Constructors
### new MaterializedView()
```ts
new MaterializedView(table): MaterializedView
```
#### Parameters
* **table**: [`Table`](Table.md)
#### Returns
[`MaterializedView`](MaterializedView.md)
## Accessors
### name
```ts
get name(): string
```
#### Returns
`string`
## Methods
### definition()
```ts
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
#### Returns
`Promise`&lt;[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)&gt;
***
### refresh()
```ts
refresh(options?): Promise<RefreshMaterializedViewResult>
```
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
`Promise`&lt;[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)&gt;
***
### table()
```ts
table(): Table
```
The view, as the table it is.
#### Returns
[`Table`](Table.md)
+4
View File
@@ -28,6 +28,7 @@
- [Job](classes/Job.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
@@ -101,6 +102,7 @@
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
@@ -110,6 +112,7 @@
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
- [RenameTableOptions](interfaces/RenameTableOptions.md)
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
@@ -142,6 +145,7 @@
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
- [MultiVector](type-aliases/MultiVector.md)
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
- [SchemaLike](type-aliases/SchemaLike.md)
@@ -0,0 +1,59 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
## Properties
### filter?
```ts
optional filter: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
# Interface: RefreshMaterializedViewResult
## Properties
### mode
```ts
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
***
### rowsWritten
```ts
rowsWritten: number;
```
***
### sourceVersion
```ts
sourceVersion: number;
```
***
### version
```ts
version: number;
```
@@ -0,0 +1,14 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
# Type Alias: MaterializedViewSelect
```ts
type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
```
The view's columns: column names, `[alias, SQL expression]` pairs, or a
record of the same. A bare name projects itself.
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
<relativePath>../pom.xml</relativePath>
</parent>
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>11.0.0-beta.19</lance-core.version>
<lance-core.version>11.0.0-beta.22</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
publish = false
license.workspace = true
description.workspace = true
+48
View File
@@ -487,4 +487,52 @@ describe("embedding functions", () => {
expect(stringSchema3).toEqual(stringExpectedSchema);
},
);
test("parses one function writing several vector columns", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const registry = getRegistry();
registry.register("multi_output_mock")(MockEmbeddingFunction);
// A materialized view can project one source vector column under two
// names, so a table's configuration names the same function twice.
const parsed = await registry.parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_a",
model: {},
},
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_b",
model: {},
},
]),
],
]),
);
expect(
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
).toEqual(["vector_a", "vector_b"]);
});
});
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { Connection, connect } from "../lancedb";
import {
DEFINITION_META_KEY,
definitionFromMetadata,
} from "../lancedb/materialized_view";
describe("materialized views", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
await db.createTable(
"people",
[
{ name: "ada", age: 36 },
{ name: "kid", age: 7 },
{ name: "grace", age: 85 },
],
{ storageOptions: { newTableEnableStableRowIds: "true" } },
);
});
afterEach(() => tmpDir.removeCallback());
it("rejects a stored limit a number cannot carry", () => {
const big = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":9007199254740993}',
],
]);
expect(() => definitionFromMetadata(big, "v")).toThrow(
/too large to represent exactly/,
);
const safe = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":42}',
],
]);
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(0);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
});
it("round-trips the definition", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
const view = await db.openMaterializedView("adults");
const definition = await view.definition();
expect(definition.sourceTable).toBe("people");
expect(definition.filter).toBe("age >= 18");
expect(definition.projections).toEqual([
["name", "`name`"],
["age", "`age`"],
]);
expect(definition.inputs).toEqual(["age", "name"]);
});
it("refreshes incrementally after an append", async () => {
const view = await db.createMaterializedView("copy", "people");
await view.refresh();
const people = await db.openTable("people");
await people.add([{ name: "alan", age: 41 }]);
const result = await view.refresh();
expect(result.mode).toBe("incremental");
expect(Number(result.rowsWritten)).toBe(1);
expect(await view.table().countRows()).toBe(4);
expect((await view.refresh()).mode).toBe("no_op");
});
it("lists views and rejects non-views", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
expect(await db.listMaterializedViews()).toEqual(["adults"]);
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
});
it("rejects an invalid expression at create time", async () => {
await expect(
db.createMaterializedView("bad", "people", {
select: [["x", "missing + 1"]],
}),
).rejects.toThrow("missing");
});
it("rejects invalid numeric options before creating anything", async () => {
for (const limit of [-5, 1.5, Infinity, NaN]) {
await expect(
db.createMaterializedView("bad", "people", { limit }),
).rejects.toThrow("non-negative integer");
}
expect(await db.listMaterializedViews()).toEqual([]);
const view = await db.createMaterializedView("copy", "people");
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
"non-negative integer",
);
}
});
it("quotes bare select names", async () => {
await db.createTable("odd_names", [{ "order item": "widget" }], {
storageOptions: { newTableEnableStableRowIds: "true" },
});
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
});
it("requires stable row ids on the source", async () => {
await db.createTable("plain", [{ x: 1 }]);
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
"stable row ids",
);
});
});
+19
View File
@@ -75,6 +75,25 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("refuses materialized views before issuing any request", async () => {
const paths: string[] = [];
await withMockDatabase(
(req, res) => {
paths.push(req.url ?? "");
res.writeHead(404).end();
},
async (db) => {
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
/only on local databases/,
);
await expect(db.listMaterializedViews()).rejects.toThrow(
/only on local databases/,
);
expect(paths).toEqual([]);
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
+1 -1
View File
@@ -2953,7 +2953,7 @@ describe("column name options", () => {
.limit(10)
.toArray();
expect(results2.length).toBe(10);
});
}, 30_000);
});
describe("when creating an empty table", () => {
+70
View File
@@ -16,6 +16,12 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import {
MaterializedView,
MaterializedViewSelect,
normalizeSelect,
validateNonNegativeInteger,
} from "./materialized_view";
import { Connection as LanceDbConnection } from "./native";
import type {
CreateNamespaceResponse,
@@ -247,6 +253,41 @@ export abstract class Connection {
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
* @param {Partial<OpenTableOptions>} options - Additional options
*/
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is created empty, with the query recorded in its schema
* metadata; `view.refresh()` computes the rows. The view is a normal
* table: it can be queried, indexed and searched, and it appears in
* `tableNames`. The source table must have stable row ids (create it with
* the `newTableEnableStableRowIds` storage option); they keep the view's
* provenance valid across source compactions and cannot be enabled after
* a table exists. Local databases only.
*/
abstract createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView>;
/**
* Open the materialized view named `name`.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract openMaterializedView(name: string): Promise<MaterializedView>;
/**
* The names of the materialized views in this database.
*
* Found by reading every table's schema, so this costs an open per table.
*/
abstract listMaterializedViews(): Promise<string[]>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -531,6 +572,35 @@ export class LocalConnection extends Connection {
);
}
async createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
const innerTable = await this.inner.createMaterializedView(
name,
source,
normalizeSelect(options?.select),
options?.where,
options?.limit,
);
return new MaterializedView(new LocalTable(innerTable));
}
async openMaterializedView(name: string): Promise<MaterializedView> {
const innerTable = await this.inner.openMaterializedView(name);
return new MaterializedView(new LocalTable(innerTable));
}
async listMaterializedViews(): Promise<string[]> {
return await this.inner.listMaterializedViews();
}
async openTable(
name: string,
namespacePath?: string[],
+6
View File
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
import type { FtsToken } from "./table";
// Re-export native header provider for use with connectWithHeaderProvider
export {
MaterializedView,
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -51,6 +56,7 @@ export {
AddResult,
AddColumnsResult,
RefreshColumnResult,
RefreshMaterializedViewResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
DeleteResult,
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { RefreshMaterializedViewResult } from "./native";
import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/** The query that defines a materialized view. */
export interface MaterializedViewDefinition {
/** Name of the source table, in the same database as the view. */
sourceTable: string;
/** `[output column, SQL expression]` pairs, in view schema order. */
projections: [string, string][];
/** SQL predicate selecting the source rows the view holds. */
filter?: string;
/** Cap on the number of rows the view holds. */
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
}
/**
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
* record of the same. A bare name projects itself.
*/
export type MaterializedViewSelect =
| (string | [string, string])[]
| Record<string, string>;
/**
* @internal Reject a numeric option N-API would otherwise silently coerce:
* `Infinity` reaches Rust as 0, `1.5` as 1.
*/
export function validateNonNegativeInteger(
value: number | undefined,
name: string,
): void {
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
throw new Error(`${name} must be a non-negative integer`);
}
}
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
function quoteIdentifier(name: string): string {
return "`" + name.replace(/`/g, "``") + "`";
}
/**
* @internal Normalize a select argument into `[alias, expression]` pairs.
* A bare name projects itself and is quoted, so any valid column name works;
* pair and record entries are kept verbatim because their right side is an
* expression.
*/
export function normalizeSelect(
select?: MaterializedViewSelect,
): [string, string][] | undefined {
if (select === undefined) {
return undefined;
}
if (Array.isArray(select)) {
return select.map((item) =>
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
);
}
return Object.entries(select);
}
/** @internal Parse a definition off a table's stored schema metadata. */
export function definitionFromMetadata(
metadata: Map<string, string>,
name: string,
): MaterializedViewDefinition {
const raw = metadata.get(DEFINITION_META_KEY);
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
if (value.kind !== "select") {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
);
}
const limit = value.limit ?? undefined;
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
// integer and every rounded one does not, so this rejects precisely the
// values a number cannot carry.
if (limit !== undefined && !Number.isSafeInteger(limit)) {
throw new Error(
`materialized view '${name}' has a stored limit too large to represent exactly`,
);
}
return {
sourceTable: value.source_table,
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
projections: (value.projections ?? []).map((p: any) => [
p.output,
p.expression,
]),
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
};
}
/**
* A handle on a materialized view: its table plus its definition.
*
* Obtained from {@link Connection#createMaterializedView} or
* {@link Connection#openMaterializedView}. The view is a normal table --
* queries, indexes and search all apply through {@link MaterializedView#table}
* -- whose contents are maintained by {@link MaterializedView#refresh}.
*/
export class MaterializedView {
private readonly inner: Table;
constructor(table: Table) {
this.inner = table;
}
get name(): string {
return this.inner.name;
}
/** The view, as the table it is. */
table(): Table {
return this.inner;
}
/** The query that defines the view, read from its stored schema. */
async definition(): Promise<MaterializedViewDefinition> {
const schema = await this.inner.schema();
return definitionFromMetadata(schema.metadata, this.name);
}
/**
* Recompute the view from its source.
*
* The refresh is incremental when the source's changes can be reconciled
* into the view -- rows added, changed or removed since the last one --
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
* refreshes to that source version instead of the latest.
*
* Concurrent refreshes of one view do not duplicate its rows. Two that
* plan the same source rows conflict on commit, and the loser throws
* rather than writing them a second time.
*/
async refresh(options?: {
full?: boolean;
sourceVersion?: number;
}): Promise<RefreshMaterializedViewResult> {
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
return await this.inner.refreshMaterializedView(
options?.full,
options?.sourceVersion,
);
}
}
+20
View File
@@ -35,6 +35,7 @@ import {
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
RefreshMaterializedViewResult,
TableStatistics,
Tags,
UpdateFieldMetadataResult,
@@ -602,6 +603,18 @@ export abstract class Table {
*/
abstract refreshColumnAsync(column: string): Promise<Job>;
/**
* Recompute this table's contents from its materialized-view definition.
*
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
* it: rejects tables that carry no view definition. Local tables only.
* @ignore
*/
abstract refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult>;
/**
* Alter the name or nullability of columns.
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -1264,6 +1277,13 @@ export class LocalTable extends Table {
return await this.inner.refreshColumnAsync(column);
}
async refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult> {
return await this.inner.refreshMaterializedView(full, sourceVersion);
}
async alterColumns(
columnAlterations: ColumnAlteration[],
): Promise<AlterColumnsResult> {
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": [
"win32"
],
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.5",
"cpu": [
"x64",
"arm64"
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
+52
View File
@@ -266,6 +266,58 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn create_materialized_view(
&self,
name: String,
source: String,
projections: Option<Vec<Vec<String>>>,
filter: Option<String>,
limit: Option<i64>,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.create_materialized_view(name, source);
if let Some(projections) = projections {
let mut pairs = Vec::with_capacity(projections.len());
for pair in projections {
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
napi::Error::from_reason("each projection must be an [output, expression] pair")
})?;
pairs.push((output, expression));
}
builder = builder.select(pairs);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
let limit = u64::try_from(limit)
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
builder = builder.limit(limit);
}
let view = builder.execute().await.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
let view = self
.get_inner()?
.open_materialized_view(&name)
.await
.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
let views = self
.get_inner()?
.list_materialized_views()
.await
.default_error()?;
Ok(views.into_iter().map(|v| v.name).collect())
}
#[napi(catch_unwind)]
pub async fn open_table(
&self,
+4
View File
@@ -1,6 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// The materialized-view refresh future deepens the type graph past the
// default trait-recursion depth; same raise as the core crate applies.
#![recursion_limit = "256"]
use std::collections::HashMap;
use env_logger::Env;
+45
View File
@@ -381,6 +381,26 @@ impl Table {
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn refresh_materialized_view(
&self,
full: Option<bool>,
source_version: Option<i64>,
) -> napi::Result<RefreshMaterializedViewResult> {
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
.await
.default_error()?;
let mut builder = view.refresh().full(full.unwrap_or(false));
if let Some(version) = source_version {
let version = u64::try_from(version).map_err(|_| {
napi::Error::from_reason("sourceVersion must be a non-negative integer")
})?;
builder = builder.source_version(version);
}
let result = builder.execute().await.default_error()?;
Ok(result.into())
}
#[napi(catch_unwind)]
pub async fn add_columns_with_schema(
&self,
@@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult {
pub version: i64,
}
#[napi(object)]
pub struct RefreshMaterializedViewResult {
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
pub mode: String,
pub rows_written: i64,
pub source_version: i64,
pub version: i64,
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match value.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
rows_written: value.rows_written as i64,
source_version: value.source_version as i64,
version: value.version as i64,
}
}
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
Self {
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
+55 -10
View File
@@ -61,7 +61,7 @@ class StreamingDataset(IterableDataset):
Internally ``__iter__`` runs a two-stage pipeline:
- **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches``
- **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth``
workers fetches raw ``RecordBatch`` objects from LanceDB in parallel
across all splits and places them in a per-split raw-batch queue.
- **Stage 2 (transform)**: a second thread pool with
@@ -104,11 +104,11 @@ class StreamingDataset(IterableDataset):
call. Larger values amortise per-request overhead (critical on object
storage) at the cost of higher memory usage per split buffer. Defaults
to ``DEFAULT_READ_BATCH_SIZE`` (64).
prefetch_batches:
io_queue_depth:
Number of I/O batches to keep in flight per split. Higher values
overlap storage latency with transform and training compute at the cost
of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES``
(4).
of more memory and threads. Must be greater than zero. Defaults to
``DEFAULT_PREFETCH_BATCHES`` (4).
columns:
Optional list of column names to read. When set, only those columns
are fetched from storage; all others are omitted. ``None`` (the
@@ -175,6 +175,16 @@ class StreamingDataset(IterableDataset):
Prefer the ``filter`` parameter when bad rows can be expressed as a
SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
splits are built, so every guarantee is fully preserved.
transform_queue_depth:
Number of transform-result batches to buffer per split in the
post-transform queue before backpressure is applied to the transform
stage. When the combined count of in-flight transform futures and
already-buffered rows for a split reaches
``transform_queue_depth * read_batch_size``, no new transforms are
submitted for that split until the consumer catches up. Useful for
capping peak memory when the consumer (e.g. a GPU training step) is
slower than the transform stage. Must be greater than zero.
``None`` (the default) imposes no limit.
worker_info_override:
If set, used in place of ``torch.utils.data.get_worker_info()`` to
determine the DataLoader worker assignment. Intended for unit tests
@@ -194,17 +204,26 @@ class StreamingDataset(IterableDataset):
rank: int = 0,
world_size: int = 1,
read_batch_size: int = DEFAULT_READ_BATCH_SIZE,
prefetch_batches: int = DEFAULT_PREFETCH_BATCHES,
io_queue_depth: int = DEFAULT_PREFETCH_BATCHES,
columns: Optional[list[str]] = None,
shuffle_clump_size: Optional[int] = None,
filter: Optional[str] = None,
transform: Optional[Callable] = None,
transform_parallelism: Optional[int] = None,
on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
transform_queue_depth: Optional[int] = None,
connection_factory: Optional[Callable[[str], Any]] = None,
worker_info_override=None,
# Deprecated; use io_queue_depth instead.
prefetch_batches: Optional[int] = None,
):
super().__init__()
if prefetch_batches is not None:
logger.warning(
"prefetch_batches is deprecated and will be removed in a future "
"version; use io_queue_depth instead"
)
io_queue_depth = prefetch_batches
if num_splits is None:
num_splits = world_size
if shuffle_seed is None:
@@ -214,6 +233,8 @@ class StreamingDataset(IterableDataset):
f"num_splits ({num_splits}) must be divisible by "
f"world_size ({world_size})"
)
if io_queue_depth <= 0:
raise ValueError("io_queue_depth must be greater than 0")
if transform_parallelism is not None and transform_parallelism <= 0:
raise ValueError("transform_parallelism must be greater than 0")
if on_transform_error not in ("raise", "skip", "warn") and not callable(
@@ -223,6 +244,8 @@ class StreamingDataset(IterableDataset):
"on_transform_error must be 'raise', 'skip', 'warn', or a "
f"callable, got {on_transform_error!r}"
)
if transform_queue_depth is not None and transform_queue_depth <= 0:
raise ValueError("transform_queue_depth must be greater than 0")
self._table = table
self._num_splits = num_splits
@@ -232,13 +255,14 @@ class StreamingDataset(IterableDataset):
self._rank = rank
self._world_size = world_size
self._read_batch_size = read_batch_size
self._prefetch_batches = prefetch_batches
self._io_queue_depth = io_queue_depth
self._columns = columns
self._shuffle_clump_size = shuffle_clump_size
self._filter = filter
self._transform = transform
self._transform_parallelism = transform_parallelism
self._on_transform_error = on_transform_error
self._transform_queue_depth = transform_queue_depth
self._connection_factory = connection_factory
self._worker_info_override = worker_info_override
@@ -365,7 +389,7 @@ class StreamingDataset(IterableDataset):
pos_consumed = list(initial_positions)
batch_size = self._read_batch_size
max_prefetch = self._prefetch_batches
io_queue_depth = self._io_queue_depth
transform_workers = (
self._transform_parallelism
if self._transform_parallelism is not None
@@ -374,6 +398,13 @@ class StreamingDataset(IterableDataset):
final_transform = (
self._transform if self._transform is not None else Transforms.arrow2python
)
# None means no limit; otherwise cap rows per split to
# transform_queue_depth batches worth (including in-flight transforms).
max_cooked_rows = (
self._transform_queue_depth * batch_size
if self._transform_queue_depth is not None
else None
)
# Per-split pipeline state. Batches are paired with the absolute
# permutation position of their first row so that skipped rows can be
@@ -409,7 +440,9 @@ class StreamingDataset(IterableDataset):
io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
def _fill_io(i: int) -> None:
while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
while (
len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i]
):
_submit_io(i)
def _drain_io(i: int) -> None:
@@ -487,7 +520,19 @@ class StreamingDataset(IterableDataset):
def _try_submit_tx(i: int) -> None:
"""Submit transforms for raw_batches[i] up to available capacity."""
while raw_batches[i] and tx_semaphore.acquire(blocking=False):
while raw_batches[i]:
# Backpressure: only submit a new transform when there is room
# for a full batch in the post-transform queue. Checking for
# a full batch prevents submitting a transform that would
# overflow the limit mid-batch (e.g. 990 rows queued with a
# capacity of 1000 and a batch_size of 128 must wait until
# 128 rows have been consumed, not just 1).
if max_cooked_rows is not None:
in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size
if in_pipeline + batch_size > max_cooked_rows:
break
if not tx_semaphore.acquire(blocking=False):
break
abs_start, batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
@@ -531,7 +576,7 @@ class StreamingDataset(IterableDataset):
# ── Main loop ─────────────────────────────────────────────────────────
with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool:
with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool:
with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool:
self._raw_batches_ref = raw_batches
self._cooked_ref = cooked
+184 -2
View File
@@ -1374,6 +1374,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli
)
# ---------------------------------------------------------------------------
# Backpressure / transform_queue_depth tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("transform_queue_depth", [0, -1])
def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth):
"""transform_queue_depth=0 or negative must raise ValueError."""
with pytest.raises(
ValueError, match="transform_queue_depth must be greater than 0"
):
StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
transform_queue_depth=transform_queue_depth,
)
@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4])
def test_transform_queue_depth_correctness(lance_table, transform_queue_depth):
"""With backpressure enabled, every row is still yielded exactly once."""
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=transform_queue_depth,
read_batch_size=8,
)
items = list(ds)
assert sorted(item["id"] for item in items) == list(range(NUM_ROWS))
def test_transform_queue_depth_matches_no_backpressure(lance_table):
"""With backpressure enabled the same samples are produced as without it."""
ds_unlimited = StreamingDataset(
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
)
ds_limited = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=1,
)
assert [item["id"] for item in ds_unlimited] == [
item["id"] for item in ds_limited
], "transform_queue_depth must not affect the sample ordering or set"
def test_transform_queue_depth_bounds_cooked_rows(lance_table):
"""prefetch_queue_depth stays within transform_queue_depth * read_batch_size
per split when observed from the main thread during iteration."""
n_splits = 4
batch_size = 8
cooked_depth = 2
# max cooked rows across all 4 splits: 4 * 2 * 8 = 64
max_allowed = n_splits * cooked_depth * batch_size
ds = StreamingDataset(
lance_table,
num_splits=n_splits,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=cooked_depth,
read_batch_size=batch_size,
transform_parallelism=1,
world_size=1,
)
peak = 0
for _ in ds:
depth = ds.prefetch_queue_depth
if depth > peak:
peak = depth
# The main thread observes depth *after* popping a row, so the peak is at
# most max_allowed (one row already popped from the split just served).
assert peak <= max_allowed, (
f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}"
)
def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path):
"""Admission requires a full read_batch_size of free space, not just one slot.
The test intercepts ThreadPoolExecutor.submit to make I/O calls execute
synchronously on the main thread. This ensures all raw batches land in
raw_batches (via _drain_io) before _try_submit_tx evaluates the admission
predicate for the first time. Without this, the I/O future for batch N+1
might still be in io_pending at the capacity-minus-one transition, leaving
raw_batches empty and causing _try_submit_tx to skip the admission check
entirely — so both the correct and the broken predicate produce depth=0
observations and the test cannot distinguish them.
With all raw batches pre-loaded in raw_batches the 4→3 cooked transition
(consuming one row from a full cooked queue) always triggers _try_submit_tx
against a non-empty raw_batches.
With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4.
A transform may only be submitted when in_pipeline + batch_size <= 4, i.e.
when in_pipeline == 0 (cooked is completely empty). Under the old broken
predicate (in_pipeline >= max_cooked_rows) the second transform would be
admitted with cooked containing batch_size-1 rows still unconsumed.
"""
import concurrent.futures as cf
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import patch
db = lancedb.connect(tmp_path)
batch_size = 4
# Four full batches → four transform submissions to observe.
table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))}))
cooked_at_submit: list[int] = []
original_submit = ThreadPoolExecutor.submit
def tracking_submit(self, fn, *args, **kwargs):
name = getattr(fn, "__name__", "")
if name == "_io_call":
# Run I/O synchronously on the calling (main) thread and return an
# already-completed Future. _drain_io checks fut.done(), so a
# completed Future is moved to raw_batches immediately on the next
# _advance call — making raw-batch readiness deterministic at the
# capacity-minus-one transition instead of depending on I/O thread
# scheduling.
fut = cf.Future()
try:
fut.set_result(fn(*args, **kwargs))
except Exception as exc:
fut.set_exception(exc)
return fut
if name == "_tx_call_guarded":
# Capture cooked depth synchronously on the main thread before the
# transform worker can drain the queue.
ref = ds._cooked_ref
cooked_at_submit.append(len(ref[0]) if ref is not None else -1)
return original_submit(self, fn, *args, **kwargs)
with patch.object(ThreadPoolExecutor, "submit", tracking_submit):
ds = StreamingDataset(
table,
num_splits=1,
shuffle_seed=42,
read_batch_size=batch_size,
transform_queue_depth=1,
transform_parallelism=1,
)
list(ds)
assert len(cooked_at_submit) == 4, (
f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}"
)
# With full-batch backpressure each transform is only admitted when the
# cooked queue is completely empty (depth == 0). The old broken predicate
# would admit at depth == batch_size - 1 == 3.
assert all(depth == 0 for depth in cooked_at_submit), (
"Transform admitted with non-empty cooked queue; full-batch backpressure "
"requires in_pipeline + batch_size <= max_cooked_rows before admission. "
f"Cooked depths at each submission: {cooked_at_submit}"
)
# ---------------------------------------------------------------------------
# Deprecated parameter name tests
# ---------------------------------------------------------------------------
def test_prefetch_batches_deprecated_warns(lance_table, caplog):
"""prefetch_batches logs a deprecation warning and behaves like io_queue_depth."""
with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
prefetch_batches=2,
)
messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING]
assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), (
f"Expected deprecation warning mentioning io_queue_depth; got: {messages}"
)
assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS))
def test_filter_limits_rows(tmp_path):
"""A filter expression is applied to the permutation so only matching rows
are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded."""
@@ -1954,7 +2136,7 @@ def test_doc_example_basic(tmp_path):
def test_doc_example_prefetch_params(tmp_path):
"""doc: Prefetching — read_batch_size and prefetch_batches still cover all rows."""
"""doc: Prefetching — read_batch_size and io_queue_depth still cover all rows."""
db = lancedb.connect(tmp_path)
table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))}))
@@ -1963,7 +2145,7 @@ def test_doc_example_prefetch_params(tmp_path):
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
read_batch_size=8,
prefetch_batches=2,
io_queue_depth=2,
)
assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true