rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-09-04 04:28:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2820-1

Browse Source
This commit is contained in:
Gatefixer
2026-08-24 17:41:48 +00:00
parent d87d69c181 b0dae5eb0b
commit 241d0604d3
77 changed files with 6837 additions and 544 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bumpversion.toml
+1 -1
View File
@@ -1,5 +1,5 @@
[tool.bumpversion]
current_version = "0.38.0-beta.3"
current_version = "0.38.0-beta.6"
parse = """(?x)
(?P<major>0|[1-9]\\d*)\\.
(?P<minor>0|[1-9]\\d*)\\.
Cargo.lock
Generated
+49 -45
View File
@@ -3455,8 +3455,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4815,8 +4815,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -4888,8 +4888,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4911,7 +4911,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4925,7 +4925,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4934,8 +4934,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrayref",
"crunchy",
@@ -4945,8 +4945,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4983,8 +4983,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5013,8 +5013,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5031,8 +5031,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"proc-macro2",
"quote",
@@ -5041,8 +5041,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5075,8 +5075,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5107,8 +5107,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arc-swap",
"arrow",
@@ -5172,8 +5172,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5195,8 +5195,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5222,7 +5222,11 @@ dependencies = [
"pin-project",
"prost",
"rand 0.9.5",
"reqsign-core",
"reqsign-file-read-tokio",
"reqsign-google",
"serde",
"serde_json",
"tempfile",
"tokio",
"tracing",
@@ -5232,8 +5236,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5247,8 +5251,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"async-trait",
@@ -5260,8 +5264,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5314,8 +5318,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5329,8 +5333,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow",
"arrow-array",
@@ -5370,8 +5374,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5384,8 +5388,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "11.0.0-beta.18"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.18#7b6e2d3586e9c1b99326313533aba2150557ed65"
version = "11.0.0-beta.22"
source = "git+https://github.com/lance-format/lance.git?tag=v11.0.0-beta.22#ea3cb4d799c468232735e9bcb43959487aca5c20"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5398,7 +5402,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
dependencies = [
"ahash",
"anyhow",
@@ -5486,7 +5490,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5511,7 +5515,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
dependencies = [
"arrow",
"async-trait",
Cargo.toml
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.18", default-features = false, "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.18", "tag" = "v11.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=11.0.0-beta.22", default-features = false, "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=11.0.0-beta.22", "tag" = "v11.0.0-beta.22", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
docs/src/java/java.md
+1 -1
View File
@@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`:
<dependency>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-core</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
</dependency>
```
docs/src/js/classes/Branches.md
+25 -25
View File
@@ -37,6 +37,31 @@ latest and stays writable.
***
### cherryPick()
```ts
cherryPick(fromBranch, dryRun): Promise<CherryPickResult>
```
Cherry-pick a branch onto main.
Set `dryRun` to `true` to preview. A failed cherry-pick resolves
with `status: "failed"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to cherry-pick from.
* **dryRun**: `boolean` = `false`
When true, only preview. Defaults to false.
#### Returns
`Promise`&lt;[`CherryPickResult`](../interfaces/CherryPickResult.md)&gt;
***
### create()
```ts
@@ -112,28 +137,3 @@ List all branches, mapping name to branch metadata.
#### Returns
`Promise`&lt;`Record`&lt;`string`, [`BranchContents`](BranchContents.md)&gt;&gt;
***
### merge()
```ts
merge(fromBranch, dryRun): Promise<MergeBranchResult>
```
Merge a branch into main.
Set `dryRun` to `true` to preview the merge. A rejected merge resolves
with `status: "rejected"` instead of throwing.
#### Parameters
* **fromBranch**: `string`
Branch to merge from.
* **dryRun**: `boolean` = `false`
When true, only preview the merge. Defaults to false.
#### Returns
`Promise`&lt;[`MergeBranchResult`](../interfaces/MergeBranchResult.md)&gt;
docs/src/js/classes/Connection.md
+75 -5
View File
@@ -169,6 +169,45 @@ Creates a new empty Table
***
### createMaterializedView()
```ts
abstract createMaterializedView(
name,
source,
options?): Promise<MaterializedView>
```
Define a materialized view named `name` over the table `source`.
The view is created empty, with the query recorded in its schema
metadata; `view.refresh()` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
`tableNames`. The source table must have stable row ids (create it with
the `newTableEnableStableRowIds` storage option); they keep the view's
provenance valid across source compactions and cannot be enabled after
a table exists. Local databases only.
#### Parameters
* **name**: `string`
* **source**: `string`
* **options?**
* **options.limit?**: `number`
* **options.select?**: [`MaterializedViewSelect`](../type-aliases/MaterializedViewSelect.md)
* **options.where?**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### createNamespace()
```ts
@@ -499,6 +538,22 @@ List server-side jobs across the database's tables.
***
### listMaterializedViews()
```ts
abstract listMaterializedViews(): Promise<string[]>
```
The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per table.
#### Returns
`Promise`&lt;`string`[]&gt;
***
### listNamespaces()
```ts
@@ -529,6 +584,26 @@ Child namespace names and
***
### openMaterializedView()
```ts
abstract openMaterializedView(name): Promise<MaterializedView>
```
Open the materialized view named `name`.
Rejects a table that exists but is not a materialized view.
#### Parameters
* **name**: `string`
#### Returns
`Promise`&lt;[`MaterializedView`](MaterializedView.md)&gt;
***
### openTable()
```ts
@@ -538,18 +613,13 @@ abstract openTable(
options?): Promise<Table>
```
Open a table in the database.
#### Parameters
* **name**: `string`
The name of the table
* **namespacePath?**: `string`[]
The namespace path of the table (defaults to root namespace)
* **options?**: `Partial`&lt;[`OpenTableOptions`](../interfaces/OpenTableOptions.md)&gt;
Additional options
#### Returns
docs/src/js/classes/MaterializedView.md
+101
View File
@@ -0,0 +1,101 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedView
# Class: MaterializedView
A handle on a materialized view: its table plus its definition.
Obtained from [Connection#createMaterializedView](Connection.md#creatematerializedview) or
[Connection#openMaterializedView](Connection.md#openmaterializedview). The view is a normal table --
queries, indexes and search all apply through [MaterializedView#table](MaterializedView.md#table)
-- whose contents are maintained by [MaterializedView#refresh](MaterializedView.md#refresh).
## Constructors
### new MaterializedView()
```ts
new MaterializedView(table): MaterializedView
```
#### Parameters
* **table**: [`Table`](Table.md)
#### Returns
[`MaterializedView`](MaterializedView.md)
## Accessors
### name
```ts
get name(): string
```
#### Returns
`string`
## Methods
### definition()
```ts
definition(): Promise<MaterializedViewDefinition>
```
The query that defines the view, read from its stored schema.
#### Returns
`Promise`&lt;[`MaterializedViewDefinition`](../interfaces/MaterializedViewDefinition.md)&gt;
***
### refresh()
```ts
refresh(options?): Promise<RefreshMaterializedViewResult>
```
Recompute the view from its source.
The refresh is incremental when the source's changes can be reconciled
into the view -- rows added, changed or removed since the last one --
and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
refreshes to that source version instead of the latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser throws
rather than writing them a second time.
#### Parameters
* **options?**
* **options.full?**: `boolean`
* **options.sourceVersion?**: `number`
#### Returns
`Promise`&lt;[`RefreshMaterializedViewResult`](../interfaces/RefreshMaterializedViewResult.md)&gt;
***
### table()
```ts
table(): Table
```
The view, as the table it is.
#### Returns
[`Table`](Table.md)
docs/src/js/globals.md
+7 -3
View File
@@ -28,6 +28,7 @@
- [Job](classes/Job.md)
- [MakeArrowTableOptions](classes/MakeArrowTableOptions.md)
- [MatchQuery](classes/MatchQuery.md)
- [MaterializedView](classes/MaterializedView.md)
- [MergeInsertBuilder](classes/MergeInsertBuilder.md)
- [MultiMatchQuery](classes/MultiMatchQuery.md)
- [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md)
@@ -59,6 +60,9 @@
- [BranchIndexSummary](interfaces/BranchIndexSummary.md)
- [BranchRowCountSummary](interfaces/BranchRowCountSummary.md)
- [BucketStats](interfaces/BucketStats.md)
- [CherryPickError](interfaces/CherryPickError.md)
- [CherryPickPreview](interfaces/CherryPickPreview.md)
- [CherryPickResult](interfaces/CherryPickResult.md)
- [ClientConfig](interfaces/ClientConfig.md)
- [ColumnAlteration](interfaces/ColumnAlteration.md)
- [ColumnOrdering](interfaces/ColumnOrdering.md)
@@ -98,10 +102,8 @@
- [ListNamespacesResponse](interfaces/ListNamespacesResponse.md)
- [LsmStats](interfaces/LsmStats.md)
- [LsmWriteSpec](interfaces/LsmWriteSpec.md)
- [MaterializedViewDefinition](interfaces/MaterializedViewDefinition.md)
- [MemtableStats](interfaces/MemtableStats.md)
- [MergeBlocker](interfaces/MergeBlocker.md)
- [MergeBranchResult](interfaces/MergeBranchResult.md)
- [MergePreview](interfaces/MergePreview.md)
- [MergeResult](interfaces/MergeResult.md)
- [NativeOAuthConfig](interfaces/NativeOAuthConfig.md)
- [OAuthConfig](interfaces/OAuthConfig.md)
@@ -110,6 +112,7 @@
- [OptimizeStats](interfaces/OptimizeStats.md)
- [QueryExecutionOptions](interfaces/QueryExecutionOptions.md)
- [RefreshColumnResult](interfaces/RefreshColumnResult.md)
- [RefreshMaterializedViewResult](interfaces/RefreshMaterializedViewResult.md)
- [RemovalStats](interfaces/RemovalStats.md)
- [RenameTableOptions](interfaces/RenameTableOptions.md)
- [RestNamespaceConfig](interfaces/RestNamespaceConfig.md)
@@ -142,6 +145,7 @@
- [FieldLike](type-aliases/FieldLike.md)
- [IntoSql](type-aliases/IntoSql.md)
- [IntoVector](type-aliases/IntoVector.md)
- [MaterializedViewSelect](type-aliases/MaterializedViewSelect.md)
- [MultiVector](type-aliases/MultiVector.md)
- [RecordBatchLike](type-aliases/RecordBatchLike.md)
- [SchemaLike](type-aliases/SchemaLike.md)
docs/src/js/interfaces/BranchDiff.md
+8 -16
View File
@@ -50,6 +50,14 @@ changedColumns: BranchColumnChange[];
***
### errors
```ts
errors: CherryPickError[];
```
***
### fromBranch
```ts
@@ -66,22 +74,6 @@ mainVersion: number;
***
### mergeBlockers
```ts
mergeBlockers: MergeBlocker[];
```
***
### mergeable
```ts
mergeable: boolean;
```
***
### parentVersion
```ts
docs/src/js/interfaces/MergeBlocker.md → docs/src/js/interfaces/CherryPickError.md
+3 -3
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBlocker
[@lancedb/lancedb](../globals.md) / CherryPickError
# Interface: MergeBlocker
# Interface: CherryPickError
A reason why a branch cannot currently be merged.
A reason why a cherry-pick cannot currently land.
## Properties
docs/src/js/interfaces/CherryPickPreview.md
+17
View File
@@ -0,0 +1,17 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / CherryPickPreview
# Interface: CherryPickPreview
Changes that would be, or were, promoted by a cherry-pick.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/MergeBranchResult.md → docs/src/js/interfaces/CherryPickResult.md
+6 -6
View File
@@ -2,11 +2,11 @@
***
[@lancedb/lancedb](../globals.md) / MergeBranchResult
[@lancedb/lancedb](../globals.md) / CherryPickResult
# Interface: MergeBranchResult
# Interface: CherryPickResult
Result of previewing or attempting a branch merge.
Result of previewing or attempting a cherry-pick.
## Properties
@@ -29,7 +29,7 @@ optional mainVersionAfter: number;
### preview
```ts
preview: MergePreview;
preview: CherryPickPreview;
```
***
@@ -38,9 +38,9 @@ preview: MergePreview;
```ts
status:
| "failed"
| "unknown"
| "rejected"
| "ready"
| "notImplemented"
| "merged";
| "cherryPicked";
```
docs/src/js/interfaces/MaterializedViewDefinition.md
+59
View File
@@ -0,0 +1,59 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewDefinition
# Interface: MaterializedViewDefinition
The query that defines a materialized view.
## Properties
### filter?
```ts
optional filter: string;
```
SQL predicate selecting the source rows the view holds.
***
### inputs
```ts
inputs: string[];
```
Source columns the projections and filter read.
***
### limit?
```ts
optional limit: number;
```
Cap on the number of rows the view holds.
***
### projections
```ts
projections: [string, string][];
```
`[output column, SQL expression]` pairs, in view schema order.
***
### sourceTable
```ts
sourceTable: string;
```
Name of the source table, in the same database as the view.
docs/src/js/interfaces/MergePreview.md
-17
View File
@@ -1,17 +0,0 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MergePreview
# Interface: MergePreview
Changes that would be, or were, promoted by a branch merge.
## Properties
### promotedColumns
```ts
promotedColumns: string[];
```
docs/src/js/interfaces/RefreshMaterializedViewResult.md
+41
View File
@@ -0,0 +1,41 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / RefreshMaterializedViewResult
# Interface: RefreshMaterializedViewResult
## Properties
### mode
```ts
mode: string;
```
How the view was brought up to date: "rebuild", "incremental" or "no_op".
***
### rowsWritten
```ts
rowsWritten: number;
```
***
### sourceVersion
```ts
sourceVersion: number;
```
***
### version
```ts
version: number;
```
docs/src/js/type-aliases/MaterializedViewSelect.md
+14
View File
@@ -0,0 +1,14 @@
[**@lancedb/lancedb**](../README.md) • **Docs**
***
[@lancedb/lancedb](../globals.md) / MaterializedViewSelect
# Type Alias: MaterializedViewSelect
```ts
type MaterializedViewSelect: (string | [string, string])[] | Record<string, string>;
```
The view's columns: column names, `[alias, SQL expression]` pairs, or a
record of the same. A bare name projects itself.
docs/src/python/python.md
+10
View File
@@ -102,6 +102,12 @@ listing a storage directory.
::: lancedb.job.AsyncJob
## Materialized Views (Synchronous)
::: lancedb.materialized_view.MaterializedView
::: lancedb.materialized_view.MaterializedViewDefinition
## Expressions
Type-safe expression builder for filters and projections. Use these instead
@@ -295,6 +301,10 @@ Table hold your actual data as a collection of records / rows.
::: lancedb.table.AsyncBranches
## Materialized Views (Asynchronous)
::: lancedb.materialized_view.AsyncMaterializedView
## Indices (Asynchronous)
Indices can be created on a table to speed up queries. This section
java/lancedb-core/pom.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<parent>
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
<relativePath>../pom.xml</relativePath>
</parent>
java/pom.xml
+2 -2
View File
@@ -6,7 +6,7 @@
<groupId>com.lancedb</groupId>
<artifactId>lancedb-parent</artifactId>
<version>0.38.0-beta.3</version>
<version>0.38.0-beta.6</version>
<packaging>pom</packaging>
<name>${project.artifactId}</name>
<description>LanceDB Java SDK Parent POM</description>
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>11.0.0-beta.18</lance-core.version>
<lance-core.version>11.0.0-beta.22</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
nodejs/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "lancedb-nodejs"
edition.workspace = true
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
publish = false
license.workspace = true
description.workspace = true
nodejs/__test__/embedding.test.ts
+48
View File
@@ -487,4 +487,52 @@ describe("embedding functions", () => {
expect(stringSchema3).toEqual(stringExpectedSchema);
},
);
test("parses one function writing several vector columns", async () => {
class MockEmbeddingFunction extends EmbeddingFunction<string> {
ndims() {
return 3;
}
embeddingDataType(): Float {
return new Float32();
}
async computeQueryEmbeddings(_data: string) {
return [1, 2, 3];
}
async computeSourceEmbeddings(data: string[]) {
return Array.from({ length: data.length }).fill([
1, 2, 3,
]) as number[][];
}
}
const registry = getRegistry();
registry.register("multi_output_mock")(MockEmbeddingFunction);
// A materialized view can project one source vector column under two
// names, so a table's configuration names the same function twice.
const parsed = await registry.parseFunctions(
new Map([
[
"embedding_functions",
JSON.stringify([
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_a",
model: {},
},
{
name: "multi_output_mock",
sourceColumn: "text",
vectorColumn: "vector_b",
model: {},
},
]),
],
]),
);
expect(
[...parsed.values()].map(({ vectorColumn }) => vectorColumn).sort(),
).toEqual(["vector_a", "vector_b"]);
});
});
nodejs/__test__/materialized_view.test.ts
+147
View File
@@ -0,0 +1,147 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import * as tmp from "tmp";
import { Connection, connect } from "../lancedb";
import {
DEFINITION_META_KEY,
definitionFromMetadata,
} from "../lancedb/materialized_view";
describe("materialized views", () => {
let tmpDir: tmp.DirResult;
let db: Connection;
beforeEach(async () => {
tmpDir = tmp.dirSync({ unsafeCleanup: true });
db = await connect(tmpDir.name);
await db.createTable(
"people",
[
{ name: "ada", age: 36 },
{ name: "kid", age: 7 },
{ name: "grace", age: 85 },
],
{ storageOptions: { newTableEnableStableRowIds: "true" } },
);
});
afterEach(() => tmpDir.removeCallback());
it("rejects a stored limit a number cannot carry", () => {
const big = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":9007199254740993}',
],
]);
expect(() => definitionFromMetadata(big, "v")).toThrow(
/too large to represent exactly/,
);
const safe = new Map([
[
DEFINITION_META_KEY,
'{"kind":"select","source_table":"people","limit":42}',
],
]);
expect(definitionFromMetadata(safe, "v").limit).toBe(42);
});
it("creates, refreshes and queries a view", async () => {
const view = await db.createMaterializedView("adults", "people", {
select: ["name", ["shout", "upper(name)"]],
where: "age >= 18",
});
expect(view.name).toBe("adults");
expect(await view.table().countRows()).toBe(0);
const result = await view.refresh();
expect(result.mode).toBe("rebuild");
expect(Number(result.rowsWritten)).toBe(2);
const rows = await view.table().query().toArray();
expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]);
});
it("round-trips the definition", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
const view = await db.openMaterializedView("adults");
const definition = await view.definition();
expect(definition.sourceTable).toBe("people");
expect(definition.filter).toBe("age >= 18");
expect(definition.projections).toEqual([
["name", "`name`"],
["age", "`age`"],
]);
expect(definition.inputs).toEqual(["age", "name"]);
});
it("refreshes incrementally after an append", async () => {
const view = await db.createMaterializedView("copy", "people");
await view.refresh();
const people = await db.openTable("people");
await people.add([{ name: "alan", age: 41 }]);
const result = await view.refresh();
expect(result.mode).toBe("incremental");
expect(Number(result.rowsWritten)).toBe(1);
expect(await view.table().countRows()).toBe(4);
expect((await view.refresh()).mode).toBe("no_op");
});
it("lists views and rejects non-views", async () => {
await db.createMaterializedView("adults", "people", {
where: "age >= 18",
});
expect(await db.listMaterializedViews()).toEqual(["adults"]);
await expect(db.openMaterializedView("people")).rejects.toThrow(
"not a materialized view",
);
});
it("rejects an invalid expression at create time", async () => {
await expect(
db.createMaterializedView("bad", "people", {
select: [["x", "missing + 1"]],
}),
).rejects.toThrow("missing");
});
it("rejects invalid numeric options before creating anything", async () => {
for (const limit of [-5, 1.5, Infinity, NaN]) {
await expect(
db.createMaterializedView("bad", "people", { limit }),
).rejects.toThrow("non-negative integer");
}
expect(await db.listMaterializedViews()).toEqual([]);
const view = await db.createMaterializedView("copy", "people");
for (const sourceVersion of [-1, 1.5, Infinity, NaN]) {
await expect(view.refresh({ sourceVersion })).rejects.toThrow(
"non-negative integer",
);
}
});
it("quotes bare select names", async () => {
await db.createTable("odd_names", [{ "order item": "widget" }], {
storageOptions: { newTableEnableStableRowIds: "true" },
});
const view = await db.createMaterializedView("quoted", "odd_names", {
select: ["order item"],
});
const result = await view.refresh();
expect(Number(result.rowsWritten)).toBe(1);
});
it("requires stable row ids on the source", async () => {
await db.createTable("plain", [{ x: 1 }]);
await expect(db.createMaterializedView("v", "plain")).rejects.toThrow(
"stable row ids",
);
});
});
nodejs/__test__/remote.test.ts
+31 -14
View File
@@ -75,6 +75,25 @@ async function withMockDatabase(
}
describe("remote connection", () => {
it("refuses materialized views before issuing any request", async () => {
const paths: string[] = [];
await withMockDatabase(
(req, res) => {
paths.push(req.url ?? "");
res.writeHead(404).end();
},
async (db) => {
await expect(db.openMaterializedView("secret_table")).rejects.toThrow(
/only on local databases/,
);
await expect(db.listMaterializedViews()).rejects.toThrow(
/only on local databases/,
);
expect(paths).toEqual([]);
},
);
});
it("should accept partial connection options", async () => {
await connect("db://test", {
apiKey: "fake",
@@ -311,7 +330,7 @@ describe("remote connection", () => {
expect(createIndexBody?.["custom_stop_words"]).toEqual(["the"]);
});
it("diffs and merges remote branches", async () => {
it("diffs and cherry-picks remote branches", async () => {
const sampleDiff = {
fromBranch: "exp",
parentVersion: 1,
@@ -333,10 +352,9 @@ describe("remote connection", () => {
changedColumns: [],
addedIndexes: [],
removedIndexes: [],
mergeable: true,
mergeBlockers: [],
errors: [],
};
const mergeBodies: Record<string, unknown>[] = [];
const cherryPickBodies: Record<string, unknown>[] = [];
await withMockDatabase(
(req, res) => {
@@ -366,17 +384,16 @@ describe("remote connection", () => {
.end(JSON.stringify(sampleDiff));
return;
}
if (path.endsWith("/branches/merge/")) {
mergeBodies.push(body);
if (path.endsWith("/branches/cherry_pick/")) {
cherryPickBodies.push(body);
const dryRun = body["dry_run"] === true;
const response = {
status: dryRun ? "ready" : "rejected",
status: dryRun ? "ready" : "failed",
diff: dryRun
? sampleDiff
: {
...sampleDiff,
mergeable: false,
mergeBlockers: [
errors: [
{ code: "baseMoved", message: "main has advanced" },
],
},
@@ -398,19 +415,19 @@ describe("remote connection", () => {
await expect(branches.diff("exp")).resolves.toEqual(sampleDiff);
const rejected = await branches.merge("exp");
expect(rejected.status).toBe("rejected");
expect(rejected.diff.mergeBlockers).toEqual([
const failed = await branches.cherryPick("exp");
expect(failed.status).toBe("failed");
expect(failed.diff.errors).toEqual([
{ code: "baseMoved", message: "main has advanced" },
]);
const preview = await branches.merge("exp", true);
const preview = await branches.cherryPick("exp", true);
expect(preview.status).toBe("ready");
expect(preview.preview.promotedColumns).toEqual(["tag"]);
},
);
expect(mergeBodies).toEqual([
expect(cherryPickBodies).toEqual([
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
{ from_branch: "exp", dry_run: false },
// biome-ignore lint/style/useNamingConvention: snake_case mandated by the server wire format
nodejs/__test__/table.test.ts
+1 -1
View File
@@ -2953,7 +2953,7 @@ describe("column name options", () => {
.limit(10)
.toArray();
expect(results2.length).toBe(10);
});
}, 30_000);
});
describe("when creating an empty table", () => {
nodejs/lancedb/connection.ts
+70
View File
@@ -16,6 +16,12 @@ import {
makeEmptyTable,
} from "./arrow";
import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry";
import {
MaterializedView,
MaterializedViewSelect,
normalizeSelect,
validateNonNegativeInteger,
} from "./materialized_view";
import { Connection as LanceDbConnection } from "./native";
import type {
CreateNamespaceResponse,
@@ -247,6 +253,41 @@ export abstract class Connection {
* @param {string[]} namespacePath - The namespace path of the table (defaults to root namespace)
* @param {Partial<OpenTableOptions>} options - Additional options
*/
/**
* Define a materialized view named `name` over the table `source`.
*
* The view is created empty, with the query recorded in its schema
* metadata; `view.refresh()` computes the rows. The view is a normal
* table: it can be queried, indexed and searched, and it appears in
* `tableNames`. The source table must have stable row ids (create it with
* the `newTableEnableStableRowIds` storage option); they keep the view's
* provenance valid across source compactions and cannot be enabled after
* a table exists. Local databases only.
*/
abstract createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView>;
/**
* Open the materialized view named `name`.
*
* Rejects a table that exists but is not a materialized view.
*/
abstract openMaterializedView(name: string): Promise<MaterializedView>;
/**
* The names of the materialized views in this database.
*
* Found by reading every table's schema, so this costs an open per table.
*/
abstract listMaterializedViews(): Promise<string[]>;
abstract openTable(
name: string,
namespacePath?: string[],
@@ -531,6 +572,35 @@ export class LocalConnection extends Connection {
);
}
async createMaterializedView(
name: string,
source: string,
options?: {
select?: MaterializedViewSelect;
where?: string;
limit?: number;
},
): Promise<MaterializedView> {
validateNonNegativeInteger(options?.limit, "limit");
const innerTable = await this.inner.createMaterializedView(
name,
source,
normalizeSelect(options?.select),
options?.where,
options?.limit,
);
return new MaterializedView(new LocalTable(innerTable));
}
async openMaterializedView(name: string): Promise<MaterializedView> {
const innerTable = await this.inner.openMaterializedView(name);
return new MaterializedView(new LocalTable(innerTable));
}
async listMaterializedViews(): Promise<string[]> {
return await this.inner.listMaterializedViews();
}
async openTable(
name: string,
namespacePath?: string[],
nodejs/lancedb/index.ts
+9 -3
View File
@@ -21,6 +21,11 @@ import type { BaseTokenizer } from "./indices";
import type { FtsToken } from "./table";
// Re-export native header provider for use with connectWithHeaderProvider
export {
MaterializedView,
MaterializedViewDefinition,
MaterializedViewSelect,
} from "./materialized_view";
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
// OpenTelemetry metrics bridge. Only the high-level entry point is public; the
@@ -51,6 +56,7 @@ export {
AddResult,
AddColumnsResult,
RefreshColumnResult,
RefreshMaterializedViewResult,
AlterColumnsResult,
UpdateFieldMetadataResult,
DeleteResult,
@@ -135,10 +141,10 @@ export {
BranchColumnChange,
BranchIndexSummary,
BranchRowCountSummary,
MergeBlocker,
CherryPickError,
BranchDiff,
MergePreview,
MergeBranchResult,
CherryPickPreview,
CherryPickResult,
AddDataOptions,
UpdateOptions,
OptimizeOptions,
nodejs/lancedb/materialized_view.ts
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
import { RefreshMaterializedViewResult } from "./native";
import { Table } from "./table";
/** Schema metadata key holding a materialized view's definition. */
export const DEFINITION_META_KEY = "mv.definition";
/** The query that defines a materialized view. */
export interface MaterializedViewDefinition {
/** Name of the source table, in the same database as the view. */
sourceTable: string;
/** `[output column, SQL expression]` pairs, in view schema order. */
projections: [string, string][];
/** SQL predicate selecting the source rows the view holds. */
filter?: string;
/** Cap on the number of rows the view holds. */
limit?: number;
/** Source columns the projections and filter read. */
inputs: string[];
}
/**
* The view's columns: column names, `[alias, SQL expression]` pairs, or a
* record of the same. A bare name projects itself.
*/
export type MaterializedViewSelect =
| (string | [string, string])[]
| Record<string, string>;
/**
* @internal Reject a numeric option N-API would otherwise silently coerce:
* `Infinity` reaches Rust as 0, `1.5` as 1.
*/
export function validateNonNegativeInteger(
value: number | undefined,
name: string,
): void {
if (value !== undefined && !(Number.isSafeInteger(value) && value >= 0)) {
throw new Error(`${name} must be a non-negative integer`);
}
}
/** @internal Quote a column name as a Lance SQL identifier (backticks). */
function quoteIdentifier(name: string): string {
return "`" + name.replace(/`/g, "``") + "`";
}
/**
* @internal Normalize a select argument into `[alias, expression]` pairs.
* A bare name projects itself and is quoted, so any valid column name works;
* pair and record entries are kept verbatim because their right side is an
* expression.
*/
export function normalizeSelect(
select?: MaterializedViewSelect,
): [string, string][] | undefined {
if (select === undefined) {
return undefined;
}
if (Array.isArray(select)) {
return select.map((item) =>
typeof item === "string" ? [item, quoteIdentifier(item)] : item,
);
}
return Object.entries(select);
}
/** @internal Parse a definition off a table's stored schema metadata. */
export function definitionFromMetadata(
metadata: Map<string, string>,
name: string,
): MaterializedViewDefinition {
const raw = metadata.get(DEFINITION_META_KEY);
if (raw === undefined) {
throw new Error(`Table '${name}' is not a materialized view`);
}
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
const value: any = JSON.parse(raw);
if (value.kind !== "select") {
throw new Error(
`materialized view '${name}' is defined by '${value.kind}', which this ` +
"version of lancedb cannot refresh",
);
}
const limit = value.limit ?? undefined;
// JSON.parse rounds integers past 2^53; every exact u64 parses to a safe
// integer and every rounded one does not, so this rejects precisely the
// values a number cannot carry.
if (limit !== undefined && !Number.isSafeInteger(limit)) {
throw new Error(
`materialized view '${name}' has a stored limit too large to represent exactly`,
);
}
return {
sourceTable: value.source_table,
// biome-ignore lint/suspicious/noExplicitAny: raw JSON
projections: (value.projections ?? []).map((p: any) => [
p.output,
p.expression,
]),
filter: value.filter ?? undefined,
limit,
inputs: value.inputs ?? [],
};
}
/**
* A handle on a materialized view: its table plus its definition.
*
* Obtained from {@link Connection#createMaterializedView} or
* {@link Connection#openMaterializedView}. The view is a normal table --
* queries, indexes and search all apply through {@link MaterializedView#table}
* -- whose contents are maintained by {@link MaterializedView#refresh}.
*/
export class MaterializedView {
private readonly inner: Table;
constructor(table: Table) {
this.inner = table;
}
get name(): string {
return this.inner.name;
}
/** The view, as the table it is. */
table(): Table {
return this.inner;
}
/** The query that defines the view, read from its stored schema. */
async definition(): Promise<MaterializedViewDefinition> {
const schema = await this.inner.schema();
return definitionFromMetadata(schema.metadata, this.name);
}
/**
* Recompute the view from its source.
*
* The refresh is incremental when the source's changes can be reconciled
* into the view -- rows added, changed or removed since the last one --
* and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
* refreshes to that source version instead of the latest.
*
* Concurrent refreshes of one view do not duplicate its rows. Two that
* plan the same source rows conflict on commit, and the loser throws
* rather than writing them a second time.
*/
async refresh(options?: {
full?: boolean;
sourceVersion?: number;
}): Promise<RefreshMaterializedViewResult> {
validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
return await this.inner.refreshMaterializedView(
options?.full,
options?.sourceVersion,
);
}
}
nodejs/lancedb/table.ts
+38 -19
View File
@@ -35,6 +35,7 @@ import {
Branches as NativeBranches,
OptimizeStats,
RefreshColumnResult,
RefreshMaterializedViewResult,
TableStatistics,
Tags,
UpdateFieldMetadataResult,
@@ -602,6 +603,18 @@ export abstract class Table {
*/
abstract refreshColumnAsync(column: string): Promise<Job>;
/**
* Recompute this table's contents from its materialized-view definition.
*
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
* it: rejects tables that carry no view definition. Local tables only.
* @ignore
*/
abstract refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult>;
/**
* Alter the name or nullability of columns.
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -1264,6 +1277,13 @@ export class LocalTable extends Table {
return await this.inner.refreshColumnAsync(column);
}
async refreshMaterializedView(
full?: boolean,
sourceVersion?: number,
): Promise<RefreshMaterializedViewResult> {
return await this.inner.refreshMaterializedView(full, sourceVersion);
}
async alterColumns(
columnAlterations: ColumnAlteration[],
): Promise<AlterColumnsResult> {
@@ -1557,8 +1577,8 @@ export interface BranchRowCountSummary {
deltaAvailable: boolean;
}
/** A reason why a branch cannot currently be merged. */
export interface MergeBlocker {
/** A reason why a cherry-pick cannot currently land. */
export interface CherryPickError {
code: string;
message: string;
}
@@ -1578,20 +1598,19 @@ export interface BranchDiff {
changedColumns: BranchColumnChange[];
addedIndexes: BranchIndexSummary[];
removedIndexes: BranchIndexSummary[];
mergeable: boolean;
mergeBlockers: MergeBlocker[];
errors: CherryPickError[];
}
/** Changes that would be, or were, promoted by a branch merge. */
export interface MergePreview {
/** Changes that would be, or were, promoted by a cherry-pick. */
export interface CherryPickPreview {
promotedColumns: string[];
}
/** Result of previewing or attempting a branch merge. */
export interface MergeBranchResult {
status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
/** Result of previewing or attempting a cherry-pick. */
export interface CherryPickResult {
status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
diff: BranchDiff;
preview: MergePreview;
preview: CherryPickPreview;
mainVersionAfter?: number;
}
@@ -1654,21 +1673,21 @@ export class Branches {
}
/**
* Merge a branch into main.
* Cherry-pick a branch onto main.
*
* Set `dryRun` to `true` to preview the merge. A rejected merge resolves
* with `status: "rejected"` instead of throwing.
* Set `dryRun` to `true` to preview. A failed cherry-pick resolves
* with `status: "failed"` instead of throwing.
*
* @param fromBranch Branch to merge from.
* @param dryRun When true, only preview the merge. Defaults to false.
* @param fromBranch Branch to cherry-pick from.
* @param dryRun When true, only preview. Defaults to false.
*/
async merge(
async cherryPick(
fromBranch: string,
dryRun: boolean = false,
): Promise<MergeBranchResult> {
return (await this.#inner.merge(
): Promise<CherryPickResult> {
return (await this.#inner.cherryPick(
fromBranch,
dryRun,
)) as unknown as MergeBranchResult;
)) as unknown as CherryPickResult;
}
}
nodejs/npm/darwin-arm64/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-darwin-arm64",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["darwin"],
"cpu": ["arm64"],
"main": "lancedb.darwin-arm64.node",
nodejs/npm/linux-arm64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-gnu.node",
nodejs/npm/linux-arm64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-arm64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["arm64"],
"main": "lancedb.linux-arm64-musl.node",
nodejs/npm/linux-x64-gnu/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-gnu",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-gnu.node",
nodejs/npm/linux-x64-musl/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-linux-x64-musl",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["linux"],
"cpu": ["x64"],
"main": "lancedb.linux-x64-musl.node",
nodejs/npm/win32-arm64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-arm64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": [
"win32"
],
nodejs/npm/win32-x64-msvc/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "@lancedb/lancedb-win32-x64-msvc",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"os": ["win32"],
"cpu": ["x64"],
"main": "lancedb.win32-x64-msvc.node",
nodejs/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@lancedb/lancedb",
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"cpu": [
"x64",
"arm64"
nodejs/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"ann"
],
"private": false,
"version": "0.38.0-beta.3",
"version": "0.38.0-beta.6",
"main": "dist/index.js",
"exports": {
".": "./dist/index.js",
nodejs/src/connection.rs
+52
View File
@@ -266,6 +266,58 @@ impl Connection {
Ok(Table::new(tbl))
}
#[napi(catch_unwind)]
pub async fn create_materialized_view(
&self,
name: String,
source: String,
projections: Option<Vec<Vec<String>>>,
filter: Option<String>,
limit: Option<i64>,
) -> napi::Result<Table> {
let mut builder = self.get_inner()?.create_materialized_view(name, source);
if let Some(projections) = projections {
let mut pairs = Vec::with_capacity(projections.len());
for pair in projections {
let [output, expression]: [String; 2] = pair.try_into().map_err(|_| {
napi::Error::from_reason("each projection must be an [output, expression] pair")
})?;
pairs.push((output, expression));
}
builder = builder.select(pairs);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
let limit = u64::try_from(limit)
.map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?;
builder = builder.limit(limit);
}
let view = builder.execute().await.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn open_materialized_view(&self, name: String) -> napi::Result<Table> {
let view = self
.get_inner()?
.open_materialized_view(&name)
.await
.default_error()?;
Ok(Table::new(view.table().clone()))
}
#[napi(catch_unwind)]
pub async fn list_materialized_views(&self) -> napi::Result<Vec<String>> {
let views = self
.get_inner()?
.list_materialized_views()
.await
.default_error()?;
Ok(views.into_iter().map(|v| v.name).collect())
}
#[napi(catch_unwind)]
pub async fn open_table(
&self,
nodejs/src/job.rs
+5 -2
View File
@@ -14,9 +14,12 @@ pub struct Job {
}
impl Job {
pub(crate) fn new(inner: lancedb::Job) -> Self {
pub(crate) fn new<T>(inner: lancedb::Job<T>) -> Self
where
T: Clone + Send + Sync + 'static,
{
Self {
inner: Arc::new(inner),
inner: Arc::new(inner.map(|_| ())),
}
}
}
nodejs/src/lib.rs
+4
View File
@@ -1,6 +1,10 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
// The materialized-view refresh future deepens the type graph past the
// default trait-recursion depth; same raise as the core crate applies.
#![recursion_limit = "256"]
use std::collections::HashMap;
use env_logger::Env;
nodejs/src/table.rs
+48 -3
View File
@@ -381,6 +381,26 @@ impl Table {
Ok(crate::job::Job::new(job))
}
#[napi(catch_unwind)]
pub async fn refresh_materialized_view(
&self,
full: Option<bool>,
source_version: Option<i64>,
) -> napi::Result<RefreshMaterializedViewResult> {
let view = lancedb::MaterializedView::from_table(self.inner_ref()?.clone())
.await
.default_error()?;
let mut builder = view.refresh().full(full.unwrap_or(false));
if let Some(version) = source_version {
let version = u64::try_from(version).map_err(|_| {
napi::Error::from_reason("sourceVersion must be a non-negative integer")
})?;
builder = builder.source_version(version);
}
let result = builder.execute().await.default_error()?;
Ok(result.into())
}
#[napi(catch_unwind)]
pub async fn add_columns_with_schema(
&self,
@@ -1387,6 +1407,31 @@ pub struct RefreshColumnResult {
pub version: i64,
}
#[napi(object)]
pub struct RefreshMaterializedViewResult {
/// How the view was brought up to date: "rebuild", "incremental" or "no_op".
pub mode: String,
pub rows_written: i64,
pub source_version: i64,
pub version: i64,
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(value: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match value.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
rows_written: value.rows_written as i64,
source_version: value.source_version as i64,
version: value.version as i64,
}
}
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(value: lancedb::table::RefreshColumnResult) -> Self {
Self {
@@ -1605,18 +1650,18 @@ impl Branches {
}
#[napi(ts_return_type = "Promise<Record<string, unknown>>")]
pub async fn merge(
pub async fn cherry_pick(
&self,
from_branch: String,
dry_run: Option<bool>,
) -> napi::Result<serde_json::Value> {
let result = self
.inner
.merge_branch(&from_branch, dry_run.unwrap_or(false))
.cherry_pick(&from_branch, dry_run.unwrap_or(false))
.await
.default_error()?;
serde_json::to_value(result).map_err(|err| {
napi::Error::from_reason(format!("failed to serialize branch merge result: {err}"))
napi::Error::from_reason(format!("failed to serialize cherry-pick result: {err}"))
})
}
}
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/__init__.py
+9
View File
@@ -29,9 +29,15 @@ from .functions import (
FunctionRegistrationRequest as FunctionRegistrationRequest,
FunctionVersion as FunctionVersion,
PythonRuntimeSpec as PythonRuntimeSpec,
RefreshColumnResult as RefreshColumnResult,
UdfDefinition as UdfDefinition,
udf as udf,
)
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
MaterializedViewDefinition,
)
from .table import AsyncTable, Table
from .types import BaseTokenizerType
from ._lancedb import Session
@@ -506,6 +512,9 @@ async def connect_async(
__all__ = [
"AsyncMaterializedView",
"MaterializedView",
"MaterializedViewDefinition",
"connect",
"connect_async",
"tokenize",
python/python/lancedb/_lancedb.pyi
+21 -10
View File
@@ -147,7 +147,7 @@ class Connection(object):
limit: Optional[int],
) -> list[str]: ... # Deprecated: Use list_tables instead
def job(self, job_id: str) -> Job: ...
async def create_function_async(self, request_json: str) -> FunctionJob: ...
async def create_function_async(self, request_json: str) -> Job: ...
async def get_function(self, name: str, version: str) -> str: ...
async def list_jobs(self) -> List[JobInfo]: ...
async def get_job(self, job_id: str) -> Optional[JobDescription]: ...
@@ -197,6 +197,15 @@ class Connection(object):
cur_namespace_path: Optional[List[str]] = None,
new_namespace_path: Optional[List[str]] = None,
) -> None: ...
async def create_materialized_view(
self,
name: str,
source: str,
projections: Optional[List[Tuple[str, str]]] = None,
filter: Optional[str] = None,
limit: Optional[int] = None,
) -> Table: ...
async def list_materialized_views(self) -> List[str]: ...
async def drop_table(
self, name: str, namespace_path: Optional[List[str]] = None
) -> None: ...
@@ -225,14 +234,7 @@ class Job:
@property
def id(self) -> Optional[str]: ...
async def status(self) -> str: ...
async def wait(self) -> None: ...
async def cancel(self) -> None: ...
class FunctionJob:
@property
def id(self) -> Optional[str]: ...
async def status(self) -> str: ...
async def wait(self) -> str: ...
async def wait(self) -> Optional[str]: ...
async def cancel(self) -> None: ...
class JobInfo:
@@ -355,6 +357,9 @@ class Table:
) -> AddColumnsResult: ...
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
async def refresh_column_async(self, column: str) -> Job: ...
async def refresh_materialized_view(
self, full: bool = False, source_version: Optional[int] = None
) -> RefreshMaterializedViewResult: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
async def alter_columns(
self, columns: list[dict[str, Any]]
@@ -420,7 +425,7 @@ class Branches:
async def checkout(self, name: str, version: Optional[int] = None) -> Table: ...
async def delete(self, name: str) -> None: ...
async def diff(self, from_branch: str) -> Dict[str, Any]: ...
async def merge(
async def cherry_pick(
self, from_branch: str, dry_run: bool = False
) -> Dict[str, Any]: ...
@@ -704,6 +709,12 @@ class RefreshColumnResult:
rows_filled: int
version: int
class RefreshMaterializedViewResult:
mode: str
rows_written: int
source_version: int
version: int
class AlterColumnsResult:
version: int
python/python/lancedb/db.py
+168 -2
View File
@@ -46,7 +46,13 @@ from lance_namespace.errors import NamespaceNotEmptyError, TableNotFoundError
from . import __version__
from ._lancedb import connect as lancedb_connect # type: ignore
from .functions import FunctionVersion, UdfDefinition
from .job import AsyncJob, Job, _function_job
from .job import AsyncJob, Job, _typed_job
from .materialized_view import (
AsyncMaterializedView,
MaterializedView,
SelectArg,
normalize_select,
)
from .table import (
AsyncTable,
LanceTable,
@@ -510,6 +516,70 @@ class DBConnection(EnforceOverrides):
"""
raise NotImplementedError
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
The view is created empty, with the query recorded in its schema
metadata; ``view.refresh()`` computes the rows. The view is a normal
table: it can be queried, indexed and searched, and it appears in
``table_names``. Local databases only.
The source table must have stable row ids (create it with the
``new_table_enable_stable_row_ids`` storage option): they keep the
view's provenance valid across source compactions, and cannot be
enabled after a table exists.
Parameters
----------
name: str
The name of the view.
source: str
The name of the source table, in this database.
select: list or dict, optional
The view's columns: column names, ``(alias, SQL expression)``
pairs, or a dict of the same. Omitting it selects every source
column, expanded against the source schema at creation time.
where: str, optional
SQL predicate; only matching source rows appear in the view.
limit: int, optional
Cap the view at this many rows, in materialization order.
Returns
-------
MaterializedView
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def open_materialized_view(self, name: str) -> MaterializedView:
"""Open the materialized view named ``name``.
Raises ``ValueError`` if the table exists but is not a materialized
view.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
)
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the database.
@@ -1136,6 +1206,58 @@ class LanceDBConnection(DBConnection):
tbl.checkout(version)
return tbl
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
Examples
--------
>>> import lancedb
>>> db = lancedb.connect(
... "./.lancedb",
... storage_options={"new_table_enable_stable_row_ids": "true"},
... )
>>> data = [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]
>>> table = db.create_table("people", data)
>>> view = db.create_materialized_view(
... "adults",
... "people",
... select=["name", ("shout", "upper(name)")],
... where="age >= 18",
... )
>>> result = view.refresh()
>>> result.rows_written
1
"""
LOOP.run(
self._conn.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
)
return MaterializedView(self.open_table(name))
@override
def open_materialized_view(self, name: str) -> MaterializedView:
"""Open the materialized view named ``name``."""
view = MaterializedView(self.open_table(name))
view.definition
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database."""
return LOOP.run(self._conn.list_materialized_views())
def clone_table(
self,
target_table_name: str,
@@ -1906,6 +2028,50 @@ class AsyncConnection(object):
await tbl.checkout(version)
return tbl
async def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> AsyncMaterializedView:
"""Define a materialized view named ``name`` over the table ``source``.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
"""
inner = await self._inner.create_materialized_view(
name,
source,
projections=normalize_select(select),
filter=where,
limit=limit,
)
return AsyncMaterializedView(AsyncTable(inner))
async def open_materialized_view(self, name: str) -> AsyncMaterializedView:
"""Open the materialized view named ``name``.
Raises ``ValueError`` if the table exists but is not a materialized
view.
"""
if self.uri.startswith("db://"):
raise NotImplementedError(
"materialized views are supported only on local databases"
)
view = AsyncMaterializedView(await self.open_table(name))
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in this database.
Found by reading every table's schema, so this costs an open per
table.
"""
return await self._inner.list_materialized_views()
async def clone_table(
self,
target_table_name: str,
@@ -2071,7 +2237,7 @@ class AsyncConnection(object):
inner = await self._inner.create_function_async(
definition.registration_request.to_canonical_json()
)
return _function_job(inner)
return _typed_job(inner, FunctionVersion.from_json)
async def get_function(self, name: str, *, version: str) -> FunctionVersion:
"""Open one exact immutable Function version from the remote catalog."""
python/python/lancedb/functions.py
+8 -2
View File
@@ -1,10 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Canonical values exchanged with LanceDB Enterprise Function services.
"""Canonical Function values exchanged with LanceDB Enterprise services.
These immutable models contain client/wire state only. Catalog persistence,
environment bake, secret resolution, and execution are owned by Sophon.
``RefreshColumnResult`` is also the backend-neutral result of a local
expression-backed refresh job.
"""
from __future__ import annotations
@@ -460,7 +462,11 @@ class FunctionBinding(_RemoteValue):
class RefreshColumnResult(_RemoteValue):
"""Terminal result of a remote Function-column refresh Job."""
"""Terminal result of an expression-backed or Function-backed refresh Job.
Local jobs produce this value in process. LanceDB Cloud and Enterprise
decode the same value from the durable server-job terminal payload.
"""
rows_assigned: _UInt64
rows_failed: _UInt64
python/python/lancedb/job.py
+28 -30
View File
@@ -5,12 +5,11 @@
import asyncio
from datetime import timedelta
from typing import Any, Generic, Optional, TypeVar, cast
from typing import Any, Callable, Generic, Optional, TypeVar, cast
from lancedb.background_loop import LOOP
from . import _lancedb
from .functions import FunctionVersion
T = TypeVar("T")
@@ -18,11 +17,18 @@ T = TypeVar("T")
class AsyncJob(Generic[T]):
"""A handle to an operation that may still be running.
The operation may already be complete when the handle is created.
The operation may already be complete when the handle is created. ``T``
is the endpoint's terminal result type; unit-result jobs resolve to
``None``.
"""
def __init__(self, inner: Optional[Any]):
def __init__(
self,
inner: Optional[Any],
result_decoder: Optional[Callable[[Any], T]] = None,
):
self._inner = inner
self._result_decoder = result_decoder
@property
def id(self) -> Optional[str]:
@@ -50,17 +56,21 @@ class AsyncJob(Generic[T]):
async def wait(self, timeout: Optional[timedelta] = None) -> T:
"""Wait until the operation reaches a terminal state.
Returns the endpoint's typed result, or ``None`` for a unit-result
job.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
if self._inner is None:
return cast(T, None)
if timeout is None:
return cast(T, await self._inner.wait())
return cast(
T,
await asyncio.wait_for(self._inner.wait(), timeout.total_seconds()),
)
result = await self._inner.wait()
else:
result = await asyncio.wait_for(self._inner.wait(), timeout.total_seconds())
if self._result_decoder is not None:
return self._result_decoder(result)
return cast(T, result)
async def cancel(self):
"""Request cancellation. Cancelling a finished operation is a no-op."""
@@ -70,7 +80,7 @@ class AsyncJob(Generic[T]):
class Job(Generic[T]):
"""Synchronous counterpart of `AsyncJob`."""
"""Synchronous counterpart of `AsyncJob` with the same result type."""
def __init__(self, inner: Optional[AsyncJob[T]]):
self._inner = inner
@@ -96,6 +106,9 @@ class Job(Generic[T]):
def wait(self, timeout: Optional[timedelta] = None) -> T:
"""Block until the operation reaches a terminal state.
Returns the endpoint's typed result, or ``None`` for a unit-result
job.
Raises `JobFailedError` if the operation failed, `JobCancelledError`
if it was cancelled, and `TimeoutError` if `timeout` elapses first.
"""
@@ -110,23 +123,8 @@ class Job(Generic[T]):
LOOP.run(self._inner.cancel())
class _FunctionJobAdapter:
def __init__(self, inner: "_lancedb.FunctionJob"):
self._inner = inner
@property
def id(self) -> Optional[str]:
return self._inner.id
async def status(self) -> str:
return await self._inner.status()
async def wait(self) -> FunctionVersion:
return FunctionVersion.from_json(await self._inner.wait())
async def cancel(self):
await self._inner.cancel()
def _function_job(inner: "_lancedb.FunctionJob") -> AsyncJob[FunctionVersion]:
return AsyncJob(_FunctionJobAdapter(inner))
def _typed_job(
inner: "_lancedb.Job", result_decoder: Callable[[str], T]
) -> AsyncJob[T]:
"""Bind an internal JSON-producing job to its public result model."""
return AsyncJob(inner, result_decoder)
python/python/lancedb/materialized_view.py
+178
View File
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
"""Materialized views: tables defined by a query over a source table and
maintained by refresh. See ``DBConnection.create_materialized_view``."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union
from .background_loop import LOOP
if TYPE_CHECKING:
import pyarrow as pa
from ._lancedb import RefreshMaterializedViewResult
from .table import AsyncTable, LanceTable
DEFINITION_META_KEY = b"mv.definition"
SelectArg = Union[
str,
Sequence[Union[str, Tuple[str, str]]],
Dict[str, str],
None,
]
@dataclass
class MaterializedViewDefinition:
"""The query that defines a materialized view."""
source_table: str
"""Name of the source table, in the same database as the view."""
projections: List[Tuple[str, str]]
"""``(output column, SQL expression)`` pairs, in view schema order."""
filter: Optional[str] = None
"""SQL predicate selecting the source rows the view holds."""
limit: Optional[int] = None
"""Cap on the number of rows the view holds."""
inputs: List[str] = field(default_factory=list)
"""Source columns the projections and filter read."""
def _definition_from_schema(
schema: "pa.Schema", name: str
) -> MaterializedViewDefinition:
metadata = schema.metadata or {}
raw = metadata.get(DEFINITION_META_KEY)
if raw is None:
raise ValueError(f"Table '{name}' is not a materialized view")
value = json.loads(raw)
kind = value.get("kind")
if kind != "select":
raise NotImplementedError(
f"materialized view '{name}' is defined by '{kind}', which this "
"version of lancedb cannot refresh"
)
return MaterializedViewDefinition(
source_table=value["source_table"],
projections=[
(p["output"], p["expression"]) for p in value.get("projections", [])
],
filter=value.get("filter"),
limit=value.get("limit"),
inputs=value.get("inputs", []),
)
def _quote_identifier(name: str) -> str:
"""Quote a column name as a Lance SQL identifier (backticks)."""
escaped = name.replace("`", "``")
return f"`{escaped}`"
def normalize_select(select: SelectArg) -> Optional[List[Tuple[str, str]]]:
"""``select`` items may be a column name, an ``(alias, expression)`` pair,
or a dict of the same. A bare name projects itself and is quoted, so any
valid column name works; dict and pair entries are kept verbatim because
their right side is an expression.
A lone string is one column, not a sequence of its characters."""
if select is None:
return None
if isinstance(select, str):
select = [select]
if isinstance(select, dict):
return list(select.items())
normalized = []
for item in select:
if isinstance(item, str):
normalized.append((item, _quote_identifier(item)))
else:
alias, expression = item
normalized.append((alias, expression))
return normalized
class AsyncMaterializedView:
"""A handle on a materialized view: its table plus its definition.
Obtained from ``AsyncConnection.create_materialized_view`` or
``AsyncConnection.open_materialized_view``.
"""
def __init__(self, table: "AsyncTable"):
self._table = table
def __repr__(self) -> str:
return f"AsyncMaterializedView(name={self.name!r})"
@property
def name(self) -> str:
return self._table.name
@property
def table(self) -> "AsyncTable":
"""The view, as the table it is. Queries, indexes and search all
apply; writes are not blocked, but a rebuild replaces them."""
return self._table
async def definition(self) -> MaterializedViewDefinition:
"""The query that defines the view, read from its stored schema."""
return _definition_from_schema(await self._table.schema(), self.name)
async def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source.
The refresh is incremental when the source's changes can be
reconciled into the view -- rows added, changed or removed since the
last one -- and otherwise rebuilds. ``full=True`` forces a rebuild;
``source_version`` refreshes to that source version instead of the
latest.
Concurrent refreshes of one view do not duplicate its rows. Two that
plan the same source rows conflict on commit, and the loser raises
rather than writing them a second time.
"""
return await self._table._inner.refresh_materialized_view(
full=full, source_version=source_version
)
class MaterializedView:
"""Synchronous variant of
[AsyncMaterializedView][lancedb.materialized_view.AsyncMaterializedView]."""
def __init__(self, table: "LanceTable"):
self._table = table
self._async = AsyncMaterializedView(table._table)
def __repr__(self) -> str:
return f"MaterializedView(name={self.name!r})"
@property
def name(self) -> str:
return self._table.name
@property
def table(self) -> "LanceTable":
"""The view, as the table it is."""
return self._table
@property
def definition(self) -> MaterializedViewDefinition:
"""The query that defines the view, read from its stored schema."""
return _definition_from_schema(self._table.schema, self.name)
def refresh(
self, *, full: bool = False, source_version: Optional[int] = None
) -> "RefreshMaterializedViewResult":
"""Recompute the view from its source. See
[AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh]."""
return LOOP.run(self._async.refresh(full=full, source_version=source_version))
python/python/lancedb/namespace.py
+68
View File
@@ -61,6 +61,11 @@ from lance_namespace import (
NamespaceExistsRequest,
TableExistsRequest,
)
from lancedb.materialized_view import (
AsyncMaterializedView,
MaterializedView,
SelectArg,
)
from lancedb.table import AsyncTable, LanceTable, Table
from lancedb.util import validate_table_name
from lancedb.common import DATA
@@ -619,6 +624,42 @@ class LanceNamespaceDBConnection(DBConnection):
tbl.checkout(version)
return tbl
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> "MaterializedView":
"""Define a materialized view over a table in the root namespace.
See
[DBConnection.create_materialized_view][lancedb.DBConnection.create_materialized_view].
"""
return MaterializedView(
self.open_table(
LOOP.run(
self._inner.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
).name
)
)
@override
def open_materialized_view(self, name: str) -> "MaterializedView":
"""Open the materialized view named ``name``."""
view = MaterializedView(self.open_table(name))
view.definition
return view
@override
def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return LOOP.run(self._inner.list_materialized_views())
@override
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
if namespace_path is None:
@@ -1141,6 +1182,33 @@ class AsyncLanceNamespaceDBConnection:
route_pushdown_to_rust=self._route_pushdown_to_rust,
)
async def create_materialized_view(
self,
name: str,
source: str,
*,
select: "SelectArg" = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> "AsyncMaterializedView":
"""Define a materialized view over a table in the root namespace."""
view = await self._inner.create_materialized_view(
name, source, select=select, where=where, limit=limit
)
# Reopen through the namespace so the view's table carries the
# namespace client and pushdown configuration a bare inner table lacks.
return AsyncMaterializedView(await self.open_table(view.name))
async def open_materialized_view(self, name: str) -> "AsyncMaterializedView":
"""Open the materialized view named ``name``."""
view = AsyncMaterializedView(await self.open_table(name))
await view.definition()
return view
async def list_materialized_views(self) -> List[str]:
"""The names of the materialized views in the root namespace."""
return await self._inner.list_materialized_views()
async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the namespace."""
if namespace_path is None:
python/python/lancedb/remote/db.py
+27
View File
@@ -25,6 +25,7 @@ from ..common import DATA
from ..db import DBConnection, LOOP
from ..functions import FunctionVersion, UdfDefinition
from ..job import AsyncJob, Job
from ..materialized_view import MaterializedView, SelectArg
if TYPE_CHECKING:
from .._lancedb import JobDescription, JobInfo
@@ -648,6 +649,32 @@ class RemoteDBConnection(DBConnection):
namespace_path=namespace_path,
)
@override
def create_materialized_view(
self,
name: str,
source: str,
*,
select: SelectArg = None,
where: Optional[str] = None,
limit: Optional[int] = None,
) -> MaterializedView:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def open_materialized_view(self, name: str) -> MaterializedView:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def list_materialized_views(self) -> List[str]:
raise NotImplementedError(
"materialized views are supported only on local databases"
)
@override
def drop_table(self, name: str, namespace_path: Optional[List[str]] = None):
"""Drop a table from the database.
python/python/lancedb/remote/table.py
+2 -2
View File
@@ -49,7 +49,7 @@ from lancedb.index import (
LabelList,
)
from lancedb.job import Job
from lancedb.functions import FunctionApplication
from lancedb.functions import FunctionApplication, RefreshColumnResult
from lancedb.remote.db import LOOP
from lancedb.table import IndexConfigType, KNOWN_METRICS
import pyarrow as pa
@@ -972,7 +972,7 @@ class RemoteTable(Table):
def refresh_column(self, column: str):
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]:
return Job(LOOP.run(self._table.refresh_column_async(column)))
def alter_columns(
python/python/lancedb/streaming.py
+484 -70
View File
@@ -24,11 +24,16 @@ import os
import random
import threading
import time
import warnings
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from copy import deepcopy
from multiprocessing import RawArray
from typing import Any, Callable, Iterator, Optional, Union
from typing import Any, Callable, cast, Iterator, Literal, Optional, Union
import pyarrow as pa
import pyarrow.compute as pc
import torch
from torch.utils.data import IterableDataset, get_worker_info
from .permutation import (
@@ -61,7 +66,7 @@ class StreamingDataset(IterableDataset):
Internally ``__iter__`` runs a two-stage pipeline:
- **Stage 1 (I/O)**: one thread pool with ``num_splits * prefetch_batches``
- **Stage 1 (I/O)**: one thread pool with ``num_splits * io_queue_depth``
workers fetches raw ``RecordBatch`` objects from LanceDB in parallel
across all splits and places them in a per-split raw-batch queue.
- **Stage 2 (transform)**: a second thread pool with
@@ -104,11 +109,11 @@ class StreamingDataset(IterableDataset):
call. Larger values amortise per-request overhead (critical on object
storage) at the cost of higher memory usage per split buffer. Defaults
to ``DEFAULT_READ_BATCH_SIZE`` (64).
prefetch_batches:
io_queue_depth:
Number of I/O batches to keep in flight per split. Higher values
overlap storage latency with transform and training compute at the cost
of more memory and threads. Defaults to ``DEFAULT_PREFETCH_BATCHES``
(4).
of more memory and threads. Must be greater than zero. Defaults to
``DEFAULT_PREFETCH_BATCHES`` (4).
columns:
Optional list of column names to read. When set, only those columns
are fetched from storage; all others are omitted. ``None`` (the
@@ -132,6 +137,39 @@ class StreamingDataset(IterableDataset):
Maximum number of transforms to run concurrently. Must be greater
than zero. When ``None`` (the default), uses ``os.cpu_count()`` or 1
when the CPU count is unavailable.
pack_sequences:
Sequence-packing mode: token lists from consecutive documents are
joined with ``eos_id`` and sliced into blocks of this many tokens.
Each item is then a dict of two ``(pack_sequences,)`` LongTensors —
``input_ids`` and ``doc_ids`` (per-position document index within
the block, for block-diagonal masks or position-id resets).
* Packing happens independently per owned split and preserves per-split
resume state.
* When a split cannot fill a real block for a cycle but an owned sibling
still can, or when only a short tail remains at epoch end, the short buffer
is padded to ``pack_sequences`` with ``pad_id`` so every local cycle emits
one block per owned split.
* ``eos_id``, ``pad_id``, and ``columns`` naming a single integer-list
column are required; incompatible with ``transform``.
eos_id:
Separator token id between packed documents. Required with
``pack_sequences``, ignored otherwise.
pad_id:
Padding token id used to complete blocks when a split runs out of
real tokens mid-cycle or at epoch end. Required with
``pack_sequences``, ignored otherwise. It must be reserved for padding:
padding positions retain the preceding document's ``doc_id`` (or zero
in an all-padding block), so callers must mask them separately using
``input_ids == pad_id``.
blocks_per_epoch:
Total number of packed blocks emitted globally per epoch. Required with
``pack_sequences``. An integer must be divisible by ``num_splits``.
Every logical split emits exactly ``blocks_per_epoch / num_splits``
blocks: exhausted splits emit padding, while tokens beyond the budget
are left out of the epoch. This fixed per-split budget keeps packed
iteration and checkpoints independent of rank topology.
Pass ``"auto"`` to estimate a corpus-level budget from a bounded sample
of token lists. The estimate may be inaccurate.
on_transform_error:
What to do when the transform raises an exception:
@@ -175,6 +213,16 @@ class StreamingDataset(IterableDataset):
Prefer the ``filter`` parameter when bad rows can be expressed as a
SQL predicate (e.g. ``"col IS NOT NULL"``) — filtering happens before
splits are built, so every guarantee is fully preserved.
transform_queue_depth:
Number of transform-result batches to buffer per split in the
post-transform queue before backpressure is applied to the transform
stage. When the combined count of in-flight transform futures and
already-buffered rows for a split reaches
``transform_queue_depth * read_batch_size``, no new transforms are
submitted for that split until the consumer catches up. Useful for
capping peak memory when the consumer (e.g. a GPU training step) is
slower than the transform stage. Must be greater than zero.
``None`` (the default) imposes no limit.
worker_info_override:
If set, used in place of ``torch.utils.data.get_worker_info()`` to
determine the DataLoader worker assignment. Intended for unit tests
@@ -194,17 +242,30 @@ class StreamingDataset(IterableDataset):
rank: int = 0,
world_size: int = 1,
read_batch_size: int = DEFAULT_READ_BATCH_SIZE,
prefetch_batches: int = DEFAULT_PREFETCH_BATCHES,
io_queue_depth: int = DEFAULT_PREFETCH_BATCHES,
columns: Optional[list[str]] = None,
shuffle_clump_size: Optional[int] = None,
filter: Optional[str] = None,
transform: Optional[Callable] = None,
transform_parallelism: Optional[int] = None,
pack_sequences: Optional[int] = None,
eos_id: Optional[int] = None,
pad_id: Optional[int] = None,
blocks_per_epoch: Optional[Union[int, Literal["auto"]]] = None,
on_transform_error: Union[str, Callable[[Exception], bool]] = "raise",
transform_queue_depth: Optional[int] = None,
connection_factory: Optional[Callable[[str], Any]] = None,
worker_info_override=None,
# Deprecated; use io_queue_depth instead.
prefetch_batches: Optional[int] = None,
):
super().__init__()
if prefetch_batches is not None:
logger.warning(
"prefetch_batches is deprecated and will be removed in a future "
"version; use io_queue_depth instead"
)
io_queue_depth = prefetch_batches
if num_splits is None:
num_splits = world_size
if shuffle_seed is None:
@@ -214,8 +275,59 @@ class StreamingDataset(IterableDataset):
f"num_splits ({num_splits}) must be divisible by "
f"world_size ({world_size})"
)
if io_queue_depth <= 0:
raise ValueError("io_queue_depth must be greater than 0")
if transform_parallelism is not None and transform_parallelism <= 0:
raise ValueError("transform_parallelism must be greater than 0")
if pack_sequences is not None:
if pack_sequences <= 0:
raise ValueError("pack_sequences must be greater than 0")
if eos_id is None:
raise ValueError("eos_id is required when pack_sequences is set")
if pad_id is None:
raise ValueError("pad_id is required when pack_sequences is set")
if blocks_per_epoch is None:
raise ValueError(
"blocks_per_epoch is required when pack_sequences is set"
)
if blocks_per_epoch != "auto":
if not isinstance(blocks_per_epoch, int) or isinstance(
blocks_per_epoch, bool
):
raise ValueError(
"blocks_per_epoch must be a positive integer or 'auto'"
)
if blocks_per_epoch <= 0:
raise ValueError("blocks_per_epoch must be greater than 0")
if blocks_per_epoch % num_splits != 0:
raise ValueError(
f"blocks_per_epoch ({blocks_per_epoch}) must be divisible by "
f"num_splits ({num_splits})"
)
if transform is not None:
raise ValueError("transform cannot be combined with pack_sequences")
if columns is None or len(columns) != 1:
raise ValueError(
"pack_sequences requires columns to name exactly one "
"list-typed column of token ids"
)
field = table.schema.field(columns[0])
if not (
pa.types.is_list(field.type)
or pa.types.is_large_list(field.type)
or pa.types.is_fixed_size_list(field.type)
):
raise ValueError(
f"pack_sequences requires a list-typed token column; "
f"{columns[0]} has type {field.type}"
)
if not pa.types.is_integer(field.type.value_type):
raise ValueError(
"pack_sequences requires a token column with integer values; "
f"{columns[0]} has value type {field.type.value_type}"
)
elif blocks_per_epoch is not None:
raise ValueError("blocks_per_epoch requires pack_sequences")
if on_transform_error not in ("raise", "skip", "warn") and not callable(
on_transform_error
):
@@ -223,6 +335,8 @@ class StreamingDataset(IterableDataset):
"on_transform_error must be 'raise', 'skip', 'warn', or a "
f"callable, got {on_transform_error!r}"
)
if transform_queue_depth is not None and transform_queue_depth <= 0:
raise ValueError("transform_queue_depth must be greater than 0")
self._table = table
self._num_splits = num_splits
@@ -232,16 +346,26 @@ class StreamingDataset(IterableDataset):
self._rank = rank
self._world_size = world_size
self._read_batch_size = read_batch_size
self._prefetch_batches = prefetch_batches
self._io_queue_depth = io_queue_depth
self._columns = columns
self._shuffle_clump_size = shuffle_clump_size
self._filter = filter
self._transform = transform
self._transform_parallelism = transform_parallelism
self._pack_sequences = pack_sequences
self._eos_id = eos_id
self._pad_id = pad_id
self._blocks_per_epoch = blocks_per_epoch
self._on_transform_error = on_transform_error
self._transform_queue_depth = transform_queue_depth
self._connection_factory = connection_factory
self._worker_info_override = worker_info_override
# Packing resume state: permutation positions and partial-block buffers.
self._pack_consumed: list[int] = [0] * num_splits
self._pack_buffers: dict[int, dict[str, list[int]]] = {}
self._pack_blocks_emitted: list[int] = [0] * num_splits
# Live references to pipeline state, set only while __iter__ is running
# in the same process. Used by the observability properties when the
# DataLoader runs with num_workers=0.
@@ -291,6 +415,9 @@ class StreamingDataset(IterableDataset):
else:
self._perm_table = builder.split_sequential(fixed=num_splits).execute()
if self._blocks_per_epoch == "auto":
self._blocks_per_epoch = self._estimate_blocks_per_epoch()
# Contiguous block of global split indices assigned to this rank.
splits_per_rank = num_splits // world_size
rank_start = rank * splits_per_rank
@@ -298,6 +425,71 @@ class StreamingDataset(IterableDataset):
range(rank_start, rank_start + splits_per_rank)
)
def _estimate_blocks_per_epoch(self) -> int:
"""Estimate a fixed packed-block budget from a bounded token sample."""
# TODO: Replace this fallback with Lance's dedicated exact token-count
# estimation API once it is available.
if self._pack_sequences is None or not self._columns:
raise RuntimeError(
"packing must be configured before estimating its budget"
)
pack_len = self._pack_sequences
token_column = self._columns[0]
sample_cap_per_split = max(1, 100_000 // self._num_splits)
sampled_tokens = 0
total_sampled = 0
total_rows = 0
rng = random.Random(self._shuffle_seed)
warnings.warn(
"blocks_per_epoch='auto' uses an approximate token-count sample; "
"pass an explicit value for exact epoch sizing",
)
for split in range(self._num_splits):
permutation = Permutation.from_tables(
self._table, self._perm_table, split=split
)
permutation = permutation.select_columns([token_column])
permutation = permutation.with_transform(Transforms.arrow2arrow)
split_rows = permutation.num_rows
if split_rows == 0:
raise ValueError(
"blocks_per_epoch='auto' cannot estimate an empty dataset"
)
# Sample roughly 1% from each logical split, with at least one row
# per split and a global target cap of 100,000 rows.
sample_rows = min(
split_rows,
max(1, min((split_rows + 99) // 100, sample_cap_per_split)),
)
sample_offsets = sorted(rng.sample(range(split_rows), sample_rows))
sample_batch_size = max(1, self._read_batch_size)
for start in range(0, sample_rows, sample_batch_size):
batch = permutation.__getitems__(
sample_offsets[start : start + sample_batch_size]
)
lengths = pc.list_value_length(batch.column(0))
if lengths.null_count:
raise ValueError("pack_sequences does not support null token lists")
sampled_tokens += int(pc.sum(lengths).as_py())
total_sampled += sample_rows
total_rows += split_rows
# Pool the samples into one global average. Each document contributes
# one EOS token.
estimated_tokens = (
(sampled_tokens + total_sampled) * total_rows // total_sampled
)
blocks = estimated_tokens // pack_len
return max(
self._num_splits,
blocks - blocks % self._num_splits,
)
def _resolve_my_splits(self) -> list[int]:
"""Return the split indices this instance should read in __iter__."""
torch_worker_info = get_worker_info()
@@ -348,8 +540,14 @@ class StreamingDataset(IterableDataset):
)
if self._columns is not None:
perm = perm.select_columns(self._columns)
perm = perm.with_transform(lambda batch: batch)
start_pos = self._resume_positions.get(split_idx, self._resume_offset)
perm = perm.with_transform(Transforms.arrow2arrow)
# Both modes resume from absolute permutation positions. Packing
# stores them separately because it also checkpoints partial blocks.
start_pos = (
self._pack_consumed[split_idx]
if self._pack_sequences is not None
else self._resume_positions.get(split_idx, self._resume_offset)
)
if start_pos > 0:
perm = perm.with_skip(start_pos)
initial_positions.append(start_pos)
@@ -365,14 +563,36 @@ class StreamingDataset(IterableDataset):
pos_consumed = list(initial_positions)
batch_size = self._read_batch_size
max_prefetch = self._prefetch_batches
io_queue_depth = self._io_queue_depth
transform_workers = (
self._transform_parallelism
if self._transform_parallelism is not None
else (os.cpu_count() or 1)
)
final_transform = (
self._transform if self._transform is not None else Transforms.arrow2python
final_transform: Callable[[pa.RecordBatch], Any]
if self._pack_sequences is not None:
# Packing consumes raw token lists, one per document.
def arrow_tokens(batch: pa.RecordBatch) -> list[list[int]]:
token_column = batch.column(0)
if token_column.null_count or token_column.flatten().null_count:
raise ValueError(
"pack_sequences does not support null token lists or values"
)
return cast(list[list[int]], token_column.to_pylist())
final_transform = arrow_tokens
else:
final_transform = (
self._transform
if self._transform is not None
else Transforms.arrow2python
)
# None means no limit; otherwise cap rows per split to
# transform_queue_depth batches worth (including in-flight transforms).
max_cooked_rows = (
self._transform_queue_depth * batch_size
if self._transform_queue_depth is not None
else None
)
# Per-split pipeline state. Batches are paired with the absolute
@@ -409,7 +629,9 @@ class StreamingDataset(IterableDataset):
io_pending[i].append((abs_start, io_pool.submit(_io_call, perm_i, indices)))
def _fill_io(i: int) -> None:
while len(io_pending[i]) < max_prefetch and fetch_head[i] < split_sizes[i]:
while (
len(io_pending[i]) < io_queue_depth and fetch_head[i] < split_sizes[i]
):
_submit_io(i)
def _drain_io(i: int) -> None:
@@ -487,7 +709,19 @@ class StreamingDataset(IterableDataset):
def _try_submit_tx(i: int) -> None:
"""Submit transforms for raw_batches[i] up to available capacity."""
while raw_batches[i] and tx_semaphore.acquire(blocking=False):
while raw_batches[i]:
# Backpressure: only submit a new transform when there is room
# for a full batch in the post-transform queue. Checking for
# a full batch prevents submitting a transform that would
# overflow the limit mid-batch (e.g. 990 rows queued with a
# capacity of 1000 and a batch_size of 128 must wait until
# 128 rows have been consumed, not just 1).
if max_cooked_rows is not None:
in_pipeline = len(cooked[i]) + len(tx_pending[i]) * batch_size
if in_pipeline + batch_size > max_cooked_rows:
break
if not tx_semaphore.acquire(blocking=False):
break
abs_start, batch = raw_batches[i].popleft()
tx_pending[i].append(tx_pool.submit(_tx_call_guarded, abs_start, batch))
@@ -529,19 +763,127 @@ class StreamingDataset(IterableDataset):
else:
break # split exhausted
def _update_stats(*, idle: bool = False) -> None:
"""Refresh pipeline statistics visible to the parent process."""
ws = self._worker_stats
ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
ws[1] = (
0
if idle
else sum(batch.num_rows for q in raw_batches for _, batch in q)
)
ws[2] = 0 if idle else sum(len(q) for q in cooked)
ws[3] = sum(local_consumed)
ws[4] = self._bytes_loaded
ws[5] = int(self._fetch_time * 1_000_000)
ws[6] = int(self._transform_time * 1_000_000)
ws[7] = self._rows_skipped
# Sequence-packing helpers
pack_len = cast(int, self._pack_sequences)
eos_id = cast(int, self._eos_id)
pad_id = cast(int, self._pad_id)
blocks_per_split = (
cast(int, self._blocks_per_epoch) // self._num_splits
if self._pack_sequences is not None
else 0
)
pack_consumed = list(self._pack_consumed)
pack_buffers = deepcopy(self._pack_buffers)
pack_blocks_emitted = list(self._pack_blocks_emitted)
def _pack_buffer(i: int) -> dict[str, list[int]]:
return pack_buffers.setdefault(my_splits[i], {"tokens": [], "starts": []})
def _fill_block(i: int) -> None:
"""Fill split i's buffer to one block or exhaust the split."""
buf = _pack_buffer(i)
while len(buf["tokens"]) < pack_len:
_ensure_cooked(i)
if not cooked[i]:
return
buf["starts"].append(len(buf["tokens"]))
pos, tokens = cooked[i].popleft()
buf["tokens"].extend(tokens)
buf["tokens"].append(eos_id)
pack_consumed[my_splits[i]] = pos + 1
local_consumed[i] += 1
_advance(i)
def _emit_block(i: int) -> dict[str, Any]:
buf = _pack_buffer(i)
tokens, starts = buf["tokens"], buf["starts"]
# doc_ids label document segments within the block; 0 also covers
# the continuation of a document begun in a prior block.
doc_ids = torch.zeros(pack_len, dtype=torch.int64)
doc_starts = [s for s in starts if 0 < s < pack_len]
doc_ids[doc_starts] = 1
doc_ids.cumsum_(dim=0) # cumulative sum marks document boundaries
block = {
"input_ids": torch.tensor(tokens[:pack_len], dtype=torch.int64),
"doc_ids": doc_ids,
}
del tokens[:pack_len]
# Shift start boundaries for the next call.
buf["starts"] = [s - pack_len for s in starts if s >= pack_len]
return block
def _commit_pack_state() -> None:
self._pack_consumed = list(pack_consumed)
self._pack_buffers = {
split: {
"tokens": list(buffer["tokens"]),
"starts": list(buffer["starts"]),
}
for split, buffer in pack_buffers.items()
}
self._pack_blocks_emitted = list(pack_blocks_emitted)
# ── Main loop ─────────────────────────────────────────────────────────
with ThreadPoolExecutor(max_workers=n * max_prefetch) as io_pool:
with ThreadPoolExecutor(max_workers=n * io_queue_depth) as io_pool:
with ThreadPoolExecutor(max_workers=transform_workers) as tx_pool:
self._raw_batches_ref = raw_batches
self._cooked_ref = cooked
self._fetch_head_ref = fetch_head
self._split_sizes_ref = split_sizes
self._local_consumed_ref = local_consumed
try:
for i in range(n):
_fill_io(i)
if self._pack_sequences is not None:
first_count = pack_blocks_emitted[my_splits[0]]
if any(
pack_blocks_emitted[split] != first_count
for split in my_splits[1:]
):
raise ValueError(
"Packed checkpoint is not aligned across the splits "
"owned by this iterator; merge every rank "
"state with merge_state_dicts before resuming on a "
"different topology"
)
while pack_blocks_emitted[my_splits[0]] < blocks_per_split:
# Each logical split gets one block per cycle. Exhausted
# splits are padded through the fixed global budget.
for i in range(n):
_fill_block(i)
for i in range(n):
tokens = _pack_buffer(i)["tokens"]
if len(tokens) < pack_len:
tokens.extend([pad_id] * (pack_len - len(tokens)))
block = _emit_block(i)
pack_blocks_emitted[my_splits[i]] += 1
if i == n - 1:
_commit_pack_state()
_update_stats()
yield block
return
while True:
# A cycle only runs if every split can still produce a
# row. Without skips all splits exhaust simultaneously
@@ -575,21 +917,7 @@ class StreamingDataset(IterableDataset):
self._resume_offset = initial_offset + local_consumed[i]
for j, split_idx in enumerate(my_splits):
self._resume_positions[split_idx] = pos_consumed[j]
ws = self._worker_stats
ws[0] = sum(
split_sizes[j] - fetch_head[j] for j in range(n)
)
ws[1] = sum(
batch.num_rows
for q in raw_batches
for _, batch in q
)
ws[2] = sum(len(q) for q in cooked)
ws[3] = sum(local_consumed)
ws[4] = self._bytes_loaded
ws[5] = int(self._fetch_time * 1_000_000)
ws[6] = int(self._transform_time * 1_000_000)
ws[7] = self._rows_skipped
_update_stats()
yield row
finally:
@@ -597,15 +925,7 @@ class StreamingDataset(IterableDataset):
# when iteration ends mid-cycle (e.g. a split whose rows
# were all skipped before completing a single cycle), so
# counters like rows_skipped would otherwise be stale.
ws = self._worker_stats
ws[0] = sum(split_sizes[j] - fetch_head[j] for j in range(n))
ws[1] = 0 # queue-depth properties document 0 when idle
ws[2] = 0
ws[3] = sum(local_consumed)
ws[4] = self._bytes_loaded
ws[5] = int(self._fetch_time * 1_000_000)
ws[6] = int(self._transform_time * 1_000_000)
ws[7] = self._rows_skipped
_update_stats(idle=True)
self._raw_batches_ref = None
self._cooked_ref = None
self._fetch_head_ref = None
@@ -763,21 +1083,31 @@ class StreamingDataset(IterableDataset):
def state_dict(self) -> dict:
"""Snapshot the dataset's consumption state.
The returned dict is topology-independent: at global step boundaries
every split has been consumed the same number of times (by the
round-robin design), so the per-split count is a single uniform value
that is identical across all ranks and DataLoader workers.
``positions_consumed_per_split`` records how far into each split's
permutation iteration has advanced. It only differs from
``samples_consumed_per_split`` when ``on_transform_error`` skipped
rows, in which case entries are exact for the splits this instance
iterated and a lower bound (the sample count) for splits owned by
other ranks or workers. Combine the state dicts from all ranks with
In row mode, the returned dict is topology-independent at global step
boundaries. ``positions_consumed_per_split`` records how far each
split's permutation has advanced, which can differ from the sample
count when ``on_transform_error`` skips rows. Combine state dicts from
every rank with
[merge_state_dicts][lancedb.streaming.StreamingDataset.merge_state_dicts]
to recover the exact value for every split before resuming on a
different topology.
before resuming on a different topology.
Packed state includes partial token buffers and emitted block counts
for every logical split. When packing is sharded, merge every rank
state with ``merge_state_dicts`` before loading it.
"""
if self._pack_sequences is not None:
return {
"shuffle_seed": self._shuffle_seed,
"num_splits": self._num_splits,
"epoch": self._epoch,
"pack_sequences": self._pack_sequences,
"eos_id": self._eos_id,
"pad_id": self._pad_id,
"blocks_per_epoch": self._blocks_per_epoch,
"samples_consumed_per_split": list(self._pack_consumed),
"blocks_emitted_per_split": list(self._pack_blocks_emitted),
"pack_buffers": deepcopy(self._pack_buffers),
}
positions = [
self._resume_positions.get(split, self._resume_offset)
for split in range(self._num_splits)
@@ -795,7 +1125,9 @@ class StreamingDataset(IterableDataset):
Raises ``ValueError`` if ``num_splits`` or ``shuffle_seed`` differ
from the checkpoint, since a different split structure or shuffle order
makes mid-epoch resumption meaningless.
makes mid-epoch resumption meaningless. Packed checkpoints
pin ``pack_sequences``, ``eos_id``, ``pad_id``,
``blocks_per_epoch``, and ``epoch``.
"""
if state["num_splits"] != self._num_splits:
raise ValueError(
@@ -807,6 +1139,31 @@ class StreamingDataset(IterableDataset):
f"shuffle_seed mismatch: checkpoint has {state['shuffle_seed']}, "
f"current dataset has {self._shuffle_seed}"
)
if "pack_buffers" in state or self._pack_sequences is not None:
for key in (
"pack_sequences",
"eos_id",
"pad_id",
"blocks_per_epoch",
"epoch",
):
ours = getattr(self, f"_{key}")
if state.get(key) != ours:
raise ValueError(
f"{key} mismatch: checkpoint has {state.get(key)}, "
f"current dataset has {ours}"
)
self._pack_consumed = [int(c) for c in state["samples_consumed_per_split"]]
self._pack_blocks_emitted = [
int(c) for c in state["blocks_emitted_per_split"]
]
self._pack_buffers = {
int(g): {"tokens": list(b["tokens"]), "starts": list(b["starts"])}
for g, b in state["pack_buffers"].items()
}
return
consumed = state["samples_consumed_per_split"]
# All entries are equal at step boundaries; use the first.
if isinstance(consumed, list):
@@ -828,25 +1185,22 @@ class StreamingDataset(IterableDataset):
def merge_state_dicts(states: list[dict]) -> dict:
"""Merge state dicts saved by different ranks into one exact state.
Only needed when ``on_transform_error`` skips rows in multi-rank
training: each rank then knows the exact permutation position only for
its own splits, and records a lower bound for the rest. Because
exactly one rank owns each split, the elementwise maximum across all
ranks' ``positions_consumed_per_split`` recovers the exact position of
every split. Without skipped rows every rank's state is already
identical and merging is a no-op.
For row mode, the elementwise maximum of permutation positions recovers
splits advanced by different ranks after transform failures. For packed
mode, the state that emitted the most blocks for each logical split
supplies that split's permutation position and partial token buffer. Packed
states must cover every rank at the same global step.
Raises ``ValueError`` if the states are empty or were not produced by
the same run (mismatched seed, split count, epoch, or sample counts).
Raises ``ValueError`` if the states are empty, were not produced by
the same run, or do not represent the same global step.
The merge is always all-to-all and topology-agnostic: collect the
``state_dict()`` from every rank of the *previous* run into one list,
merge that whole list, and hand the identical merged result to every
rank of the *next* run — regardless of whether the rank count grew,
shrank, or stayed the same. There is no pairwise or subset merging
step, because each split's exact position is only known to whichever
rank owned that split, and the elementwise maximum needs every rank's
contribution to be correct.
``state_dict()`` from every rank of the *previous* run into
one list, merge that whole list, and hand the identical merged result
to every rank of the *next* run — regardless of whether the
topology grew, shrank, or stayed the same. There is no pairwise or
subset merging step, because each split's exact state is only known to
whichever iterator owned that split.
For example, checkpointing 8 ranks and resuming on 4 (the same
pattern applies when growing, e.g. 4 ranks resuming on 8)::
@@ -879,13 +1233,73 @@ class StreamingDataset(IterableDataset):
if not states:
raise ValueError("merge_state_dicts requires at least one state dict")
first = states[0]
packed = "pack_buffers" in first
config_keys = ["shuffle_seed", "num_splits", "epoch"]
if packed:
config_keys.extend(
["pack_sequences", "eos_id", "pad_id", "blocks_per_epoch"]
)
for state in states[1:]:
for key in ("shuffle_seed", "num_splits", "epoch"):
if ("pack_buffers" in state) != packed:
raise ValueError("cannot merge packed and unpacked state dicts")
for key in config_keys:
if state[key] != first[key]:
raise ValueError(
f"{key} mismatch across state dicts: "
f"{state[key]} != {first[key]}"
)
if packed:
num_splits = first["num_splits"]
for state in states:
for key in (
"samples_consumed_per_split",
"blocks_emitted_per_split",
):
if len(state[key]) != num_splits:
raise ValueError(
f"{key} must contain one entry per logical split"
)
merged_consumed = []
merged_emitted = []
merged_buffers = {}
for split in range(num_splits):
owner = states[0]
owner_progress = (
owner["blocks_emitted_per_split"][split],
owner["samples_consumed_per_split"][split],
)
for state in states[1:]:
progress = (
state["blocks_emitted_per_split"][split],
state["samples_consumed_per_split"][split],
)
if progress > owner_progress:
owner = state
owner_progress = progress
merged_consumed.append(owner["samples_consumed_per_split"][split])
merged_emitted.append(owner["blocks_emitted_per_split"][split])
buffer = owner["pack_buffers"].get(
split, owner["pack_buffers"].get(str(split))
)
if buffer is not None:
merged_buffers[split] = deepcopy(buffer)
if len(set(merged_emitted)) > 1:
raise ValueError(
"packed state dicts were not captured at the same global "
"step or do not cover every rank"
)
merged = dict(first)
merged["samples_consumed_per_split"] = merged_consumed
merged["blocks_emitted_per_split"] = merged_emitted
merged["pack_buffers"] = merged_buffers
return merged
for state in states[1:]:
if (
state["samples_consumed_per_split"]
!= first["samples_consumed_per_split"]
python/python/lancedb/table.py
+43 -18
View File
@@ -40,7 +40,7 @@ from ._blob import (
from .types import BlobMode
from lancedb.arrow import peek_reader
from lancedb.background_loop import LOOP, embedding_executor
from lancedb.job import AsyncJob, Job
from lancedb.job import AsyncJob, Job, _typed_job
from .dependencies import (
_check_for_hugging_face,
_check_for_lance,
@@ -72,7 +72,10 @@ from .index import (
FTS,
)
from .expr import Expr
from .functions import FunctionApplication
from .functions import (
FunctionApplication,
RefreshColumnResult as RefreshColumnJobResult,
)
from .merge import LanceMergeInsertBuilder
from .pydantic import LanceModel, model_to_dict
from .query import (
@@ -2039,7 +2042,7 @@ class Table(ABC):
"""
@abstractmethod
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
@@ -2050,6 +2053,12 @@ class Table(ABC):
than failing the job. On local tables the job runs in-process; on
LanceDB Cloud and Enterprise it is the server's backfill job.
Returns
-------
Job[RefreshColumnResult]
A job whose successful ``wait`` returns row counts plus the source
and published table versions.
Examples
--------
>>> import lancedb
@@ -2058,7 +2067,9 @@ class Table(ABC):
>>> table.add_columns(computed={"doubled": "x * 2"})
AddColumnsResult(version=2)
>>> job = table.refresh_column_async("doubled")
>>> job.wait()
>>> result = job.wait()
>>> result.rows_assigned
2
>>> job.status()
'finished'
"""
@@ -4082,7 +4093,7 @@ class LanceTable(Table):
[`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]:
"""Fill a computed column's unfilled rows, returning a handle to the
refresh job. See
[`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
@@ -6122,7 +6133,9 @@ class AsyncTable:
"""
return await self._inner.refresh_column(column)
async def refresh_column_async(self, column: str) -> AsyncJob:
async def refresh_column_async(
self, column: str
) -> AsyncJob[RefreshColumnJobResult]:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
@@ -6134,6 +6147,12 @@ class AsyncTable:
in-process; on LanceDB Cloud and Enterprise it is the server's
backfill job.
Returns
-------
AsyncJob[RefreshColumnResult]
A job whose successful ``wait`` returns row counts plus the source
and published table versions.
Examples
--------
>>> import asyncio
@@ -6143,12 +6162,16 @@ class AsyncTable:
... table = await db.create_table("computed_job_async_demo", [{"x": 1}])
... await table.add_columns(computed={"doubled": "x * 2"})
... job = await table.refresh_column_async("doubled")
... await job.wait()
... result = await job.wait()
... assert result.rows_assigned == 1
... return await job.status()
>>> asyncio.run(refresh_in_background())
'finished'
"""
return AsyncJob(await self._inner.refresh_column_async(column))
return _typed_job(
await self._inner.refresh_column_async(column),
RefreshColumnJobResult.from_json,
)
async def alter_columns(
self, *alterations: Iterable[dict[str, Any]]
@@ -6804,21 +6827,21 @@ class Branches:
"""Diff a branch against main."""
return LOOP.run(self._table.branches.diff(from_branch))
def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Merge a branch into main, or dry-run.
def cherry_pick(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Cherry-pick a branch onto main, or dry-run.
Parameters
----------
from_branch: str
Branch to merge from.
Branch to cherry-pick from.
dry_run: bool, default False
When True, only preview. When False, attempt the merge.
When True, only preview. When False, attempt the cherry-pick.
Notes
-----
A rejected merge returns ``status="rejected"`` instead of raising.
A failed cherry-pick returns ``status="failed"`` instead of raising.
"""
return LOOP.run(self._table.branches.merge(from_branch, dry_run))
return LOOP.run(self._table.branches.cherry_pick(from_branch, dry_run))
def _wrap(
self, async_table: "AsyncTable", version: Optional[int] = None
@@ -6954,9 +6977,11 @@ class AsyncBranches:
"""Diff a branch against main."""
return await self._table.branches.diff(from_branch)
async def merge(self, from_branch: str, dry_run: bool = False) -> Dict[str, Any]:
"""Merge a branch into main, or dry-run.
async def cherry_pick(
self, from_branch: str, dry_run: bool = False
) -> Dict[str, Any]:
"""Cherry-pick a branch onto main, or dry-run.
A rejected merge returns ``status="rejected"`` instead of raising.
A failed cherry-pick returns ``status="failed"`` instead of raising.
"""
return await self._table.branches.merge(from_branch, dry_run)
return await self._table.branches.cherry_pick(from_branch, dry_run)
python/python/tests/test_db.py
+2 -2
View File
@@ -774,7 +774,7 @@ def test_drop_table_async(tmp_db: lancedb.DBConnection):
job = tmp_db.drop_table_async("test")
assert job.id is None
assert job.status() == "finished"
job.wait()
assert job.wait() is None
assert tmp_db.table_names() == []
tmp_db.create_table("test", data=data)
@@ -790,7 +790,7 @@ async def test_drop_table_async_connection(tmp_db_async: lancedb.AsyncConnection
job = await tmp_db_async.drop_table_async("test")
assert job.id is None
assert await job.status() == "finished"
await job.wait()
assert await job.wait() is None
assert await tmp_db_async.table_names() == []
python/python/tests/test_elastic_dataloader.py
+392 -2
View File
@@ -1374,6 +1374,188 @@ def test_transform_parallelism_must_be_positive(lance_table, transform_paralleli
)
# ---------------------------------------------------------------------------
# Backpressure / transform_queue_depth tests
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("transform_queue_depth", [0, -1])
def test_transform_queue_depth_must_be_positive(lance_table, transform_queue_depth):
"""transform_queue_depth=0 or negative must raise ValueError."""
with pytest.raises(
ValueError, match="transform_queue_depth must be greater than 0"
):
StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
transform_queue_depth=transform_queue_depth,
)
@pytest.mark.parametrize("transform_queue_depth", [1, 2, 4])
def test_transform_queue_depth_correctness(lance_table, transform_queue_depth):
"""With backpressure enabled, every row is still yielded exactly once."""
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=transform_queue_depth,
read_batch_size=8,
)
items = list(ds)
assert sorted(item["id"] for item in items) == list(range(NUM_ROWS))
def test_transform_queue_depth_matches_no_backpressure(lance_table):
"""With backpressure enabled the same samples are produced as without it."""
ds_unlimited = StreamingDataset(
lance_table, num_splits=NUM_SPLITS, shuffle_seed=SHUFFLE_SEED
)
ds_limited = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=1,
)
assert [item["id"] for item in ds_unlimited] == [
item["id"] for item in ds_limited
], "transform_queue_depth must not affect the sample ordering or set"
def test_transform_queue_depth_bounds_cooked_rows(lance_table):
"""prefetch_queue_depth stays within transform_queue_depth * read_batch_size
per split when observed from the main thread during iteration."""
n_splits = 4
batch_size = 8
cooked_depth = 2
# max cooked rows across all 4 splits: 4 * 2 * 8 = 64
max_allowed = n_splits * cooked_depth * batch_size
ds = StreamingDataset(
lance_table,
num_splits=n_splits,
shuffle_seed=SHUFFLE_SEED,
transform_queue_depth=cooked_depth,
read_batch_size=batch_size,
transform_parallelism=1,
world_size=1,
)
peak = 0
for _ in ds:
depth = ds.prefetch_queue_depth
if depth > peak:
peak = depth
# The main thread observes depth *after* popping a row, so the peak is at
# most max_allowed (one row already popped from the split just served).
assert peak <= max_allowed, (
f"prefetch_queue_depth peaked at {peak}, expected <= {max_allowed}"
)
def test_transform_queue_depth_does_not_admit_at_capacity_minus_one(tmp_path):
"""Admission requires a full read_batch_size of free space, not just one slot.
The test intercepts ThreadPoolExecutor.submit to make I/O calls execute
synchronously on the main thread. This ensures all raw batches land in
raw_batches (via _drain_io) before _try_submit_tx evaluates the admission
predicate for the first time. Without this, the I/O future for batch N+1
might still be in io_pending at the capacity-minus-one transition, leaving
raw_batches empty and causing _try_submit_tx to skip the admission check
entirely — so both the correct and the broken predicate produce depth=0
observations and the test cannot distinguish them.
With all raw batches pre-loaded in raw_batches the 4→3 cooked transition
(consuming one row from a full cooked queue) always triggers _try_submit_tx
against a non-empty raw_batches.
With transform_queue_depth=1 and batch_size=4, max_cooked_rows=4.
A transform may only be submitted when in_pipeline + batch_size <= 4, i.e.
when in_pipeline == 0 (cooked is completely empty). Under the old broken
predicate (in_pipeline >= max_cooked_rows) the second transform would be
admitted with cooked containing batch_size-1 rows still unconsumed.
"""
import concurrent.futures as cf
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import patch
db = lancedb.connect(tmp_path)
batch_size = 4
# Four full batches → four transform submissions to observe.
table = db.create_table("t", pa.table({"id": list(range(batch_size * 4))}))
cooked_at_submit: list[int] = []
original_submit = ThreadPoolExecutor.submit
def tracking_submit(self, fn, *args, **kwargs):
name = getattr(fn, "__name__", "")
if name == "_io_call":
# Run I/O synchronously on the calling (main) thread and return an
# already-completed Future. _drain_io checks fut.done(), so a
# completed Future is moved to raw_batches immediately on the next
# _advance call — making raw-batch readiness deterministic at the
# capacity-minus-one transition instead of depending on I/O thread
# scheduling.
fut = cf.Future()
try:
fut.set_result(fn(*args, **kwargs))
except Exception as exc:
fut.set_exception(exc)
return fut
if name == "_tx_call_guarded":
# Capture cooked depth synchronously on the main thread before the
# transform worker can drain the queue.
ref = ds._cooked_ref
cooked_at_submit.append(len(ref[0]) if ref is not None else -1)
return original_submit(self, fn, *args, **kwargs)
with patch.object(ThreadPoolExecutor, "submit", tracking_submit):
ds = StreamingDataset(
table,
num_splits=1,
shuffle_seed=42,
read_batch_size=batch_size,
transform_queue_depth=1,
transform_parallelism=1,
)
list(ds)
assert len(cooked_at_submit) == 4, (
f"Expected 4 transform submissions (one per batch), got {len(cooked_at_submit)}"
)
# With full-batch backpressure each transform is only admitted when the
# cooked queue is completely empty (depth == 0). The old broken predicate
# would admit at depth == batch_size - 1 == 3.
assert all(depth == 0 for depth in cooked_at_submit), (
"Transform admitted with non-empty cooked queue; full-batch backpressure "
"requires in_pipeline + batch_size <= max_cooked_rows before admission. "
f"Cooked depths at each submission: {cooked_at_submit}"
)
# ---------------------------------------------------------------------------
# Deprecated parameter name tests
# ---------------------------------------------------------------------------
def test_prefetch_batches_deprecated_warns(lance_table, caplog):
"""prefetch_batches logs a deprecation warning and behaves like io_queue_depth."""
with caplog.at_level(logging.WARNING, logger="lancedb.streaming"):
ds = StreamingDataset(
lance_table,
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
prefetch_batches=2,
)
messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING]
assert any("deprecated" in m.lower() and "io_queue_depth" in m for m in messages), (
f"Expected deprecation warning mentioning io_queue_depth; got: {messages}"
)
assert sorted(item["id"] for item in ds) == list(range(NUM_ROWS))
def test_filter_limits_rows(tmp_path):
"""A filter expression is applied to the permutation so only matching rows
are yielded. IDs 0..59 pass ``id < 60``; the other 60 are excluded."""
@@ -1931,6 +2113,214 @@ def test_shuffle_seed_none_generates_stable_seed(lance_table):
assert first == second, "Same resolved seed must produce the same ordering"
# Sequence packing tests
def _create_token_table(tmp_path, documents):
db = lancedb.connect(tmp_path)
tokens = pa.array(documents, type=pa.list_(pa.int64()))
return db.create_table("tokens", pa.table({"tokens": tokens}))
def _packed_dataset(table, pack_sequences, *, blocks_per_epoch, pad_id=0, **kwargs):
return StreamingDataset(
table,
shuffle=False,
columns=["tokens"],
pack_sequences=pack_sequences,
eos_id=9,
pad_id=pad_id,
blocks_per_epoch=blocks_per_epoch,
**kwargs,
)
def test_pack_sequences_emits_blocks_and_pads_final_tail(tmp_path):
table = _create_token_table(tmp_path, [[1, 2], [3, 4], [5]])
dataset = _packed_dataset(table, 6, blocks_per_epoch=2)
blocks = list(dataset)
assert len(blocks) == 2
assert blocks[0]["input_ids"].tolist() == [1, 2, 9, 3, 4, 9]
assert blocks[0]["doc_ids"].tolist() == [0, 0, 0, 1, 1, 1]
assert blocks[1]["input_ids"].tolist() == [5, 9, 0, 0, 0, 0]
assert blocks[1]["doc_ids"].tolist() == [0, 0, 0, 0, 0, 0]
assert blocks[0]["input_ids"].dtype == torch.int64
assert blocks[0]["doc_ids"].dtype == torch.int64
def test_pack_sequences_pads_lagging_splits(tmp_path):
table = _create_token_table(
tmp_path,
[[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]],
)
dataset = _packed_dataset(table, 5, blocks_per_epoch=6, num_splits=2)
input_ids = [block["input_ids"].tolist() for block in dataset]
# Split 0 has four real tokens including EOS markers, while split 1 has
# eleven. Packing must emit three complete two-split cycles.
assert input_ids == [
[1, 9, 2, 9, 0],
[10, 11, 12, 13, 14],
[0, 0, 0, 0, 0],
[15, 16, 17, 9, 20],
[0, 0, 0, 0, 0],
[9, 0, 0, 0, 0],
]
per_rank = []
for rank in range(2):
rank_dataset = _packed_dataset(
table,
5,
blocks_per_epoch=6,
num_splits=2,
world_size=2,
rank=rank,
)
per_rank.append([block["input_ids"].tolist() for block in rank_dataset])
assert [len(blocks) for blocks in per_rank] == [3, 3]
sharded = [block for cycle in zip(*per_rank) for block in cycle]
assert sharded == input_ids
def test_pack_sequences_auto_estimates_filtered_token_column(tmp_path):
db = lancedb.connect(tmp_path)
table = db.create_table(
"tokens",
pa.table(
{
"tokens": pa.array([[1] * 4, [2] * 9], type=pa.list_(pa.int64())),
"keep": [True, False],
}
),
)
table.add(
pa.table(
{
"tokens": pa.array([[3] * 4, [4] * 9], type=pa.list_(pa.int64())),
"keep": [True, False],
}
)
)
with pytest.warns(UserWarning, match="approximate token-count sample"):
dataset = _packed_dataset(
table,
5,
blocks_per_epoch="auto",
num_splits=2,
filter="keep",
)
# Two kept documents contain 8 tokens plus 2 EOS tokens: two blocks.
assert dataset.state_dict()["blocks_per_epoch"] == 2
def test_pack_sequences_checkpoint_resumes_on_new_topology(tmp_path):
table = _create_token_table(
tmp_path,
[[1], [2], [10, 11, 12, 13, 14, 15, 16, 17], [20]],
)
kwargs = dict(pack_sequences=5, blocks_per_epoch=6, num_splits=2)
reference = list(_packed_dataset(table, **kwargs))
datasets = [
_packed_dataset(table, world_size=2, rank=rank, **kwargs) for rank in range(2)
]
iterators = [iter(dataset) for dataset in datasets]
first_cycle = [next(iterator) for iterator in iterators]
checkpoint = StreamingDataset.merge_state_dicts(
[dataset.state_dict() for dataset in datasets]
)
for iterator in iterators:
iterator.close()
resumed = _packed_dataset(table, **kwargs)
resumed.load_state_dict(checkpoint)
actual_remaining = list(resumed)
assert [block["input_ids"].tolist() for block in first_cycle] == [
[1, 9, 2, 9, 0],
[10, 11, 12, 13, 14],
]
assert checkpoint["blocks_emitted_per_split"] == [1, 1]
assert [block["input_ids"].tolist() for block in actual_remaining] == [
block["input_ids"].tolist() for block in reference[2:]
]
assert [block["doc_ids"].tolist() for block in actual_remaining] == [
block["doc_ids"].tolist() for block in reference[2:]
]
def test_pack_sequences_validates_configuration_and_tokens(tmp_path):
table = _create_token_table(tmp_path, [[1, 2]])
with pytest.raises(ValueError, match="pad_id is required"):
StreamingDataset(
table,
shuffle=False,
columns=["tokens"],
pack_sequences=4,
eos_id=9,
)
with pytest.raises(ValueError, match="blocks_per_epoch is required"):
StreamingDataset(
table,
shuffle=False,
columns=["tokens"],
pack_sequences=4,
eos_id=9,
pad_id=0,
)
with pytest.raises(ValueError, match="must be divisible"):
_packed_dataset(table, 4, blocks_per_epoch=3, num_splits=2)
with pytest.raises(ValueError, match="positive integer or 'auto'"):
_packed_dataset(table, 4, blocks_per_epoch="estimate")
checkpoint = _packed_dataset(table, 4, blocks_per_epoch=1).state_dict()
resumed = _packed_dataset(table, 4, blocks_per_epoch=1, pad_id=8)
with pytest.raises(ValueError, match="pad_id mismatch"):
resumed.load_state_dict(checkpoint)
float_db = lancedb.connect(tmp_path / "float")
float_table = float_db.create_table(
"tokens",
pa.table({"tokens": pa.array([[1.5, 2.5]], type=pa.list_(pa.float64()))}),
)
with pytest.raises(ValueError, match="token column with integer values"):
_packed_dataset(float_table, 4, blocks_per_epoch=1)
null_db = lancedb.connect(tmp_path / "null")
null_table = null_db.create_table(
"tokens",
pa.table({"tokens": pa.array([None], type=pa.list_(pa.int64()))}),
)
with pytest.raises(ValueError, match="does not support null token lists"):
list(_packed_dataset(null_table, 4, blocks_per_epoch=1))
null_value_db = lancedb.connect(tmp_path / "null_value")
null_value_table = null_value_db.create_table(
"tokens",
pa.table(
{"tokens": pa.array([[1], [2, None], [3]], type=pa.list_(pa.int64()))}
),
)
blocks = list(
_packed_dataset(
null_value_table,
2,
blocks_per_epoch=2,
on_transform_error="skip",
)
)
assert [block["input_ids"].tolist() for block in blocks] == [[1, 9], [3, 9]]
# ---------------------------------------------------------------------------
# Doc examples — each test mirrors the code snippet in index.mdx so that
# broken doc examples are caught before they ship.
@@ -1954,7 +2344,7 @@ def test_doc_example_basic(tmp_path):
def test_doc_example_prefetch_params(tmp_path):
"""doc: Prefetching — read_batch_size and prefetch_batches still cover all rows."""
"""doc: Prefetching — read_batch_size and io_queue_depth still cover all rows."""
db = lancedb.connect(tmp_path)
table = db.create_table("t", pa.table({"id": list(range(NUM_ROWS))}))
@@ -1963,7 +2353,7 @@ def test_doc_example_prefetch_params(tmp_path):
num_splits=NUM_SPLITS,
shuffle_seed=SHUFFLE_SEED,
read_batch_size=8,
prefetch_batches=2,
io_queue_depth=2,
)
assert sorted(s["id"] for s in ds) == list(range(NUM_ROWS))
python/python/tests/test_first_class_function_slice2.py
+1 -1
View File
@@ -187,7 +187,7 @@ def _mock_remote_function_catalog():
"job_state": "DONE",
"result": state["version"],
}
elif self.path == "/v1/functions/get":
elif self.path == "/v1/functions/describe":
assert body == {
"name": "normalize_score",
"version": "fv_exact",
python/python/tests/test_index.py
+1 -1
View File
@@ -88,7 +88,7 @@ async def binary_table(db_async):
async def test_create_index_async_returns_done_job(some_table: AsyncTable):
job = await some_table.create_index_async("id", config=BTree())
assert job.id is None
await job.wait()
assert await job.wait() is None
assert len(await some_table.list_indices()) == 1
await job.cancel()
python/python/tests/test_materialized_views.py
+268
View File
@@ -0,0 +1,268 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import lancedb
import pytest
from lancedb.materialized_view import MaterializedViewDefinition
STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"}
def make_db(tmp_path):
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
db.create_table(
"people",
[
{"name": "ada", "age": 36},
{"name": "kid", "age": 7},
{"name": "grace", "age": 85},
],
)
return db
def test_create_refresh_and_query(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view(
"adults",
"people",
select=["name", ("shout", "upper(name)")],
where="age >= 18",
)
assert view.name == "adults"
assert view.table.count_rows() == 0
result = view.refresh()
assert result.mode == "rebuild"
assert result.rows_written == 2
rows = view.table.search().to_list()
assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"]
def test_definition_round_trips(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("adults", "people", where="age >= 18")
view = db.open_materialized_view("adults")
assert view.definition == MaterializedViewDefinition(
source_table="people",
projections=[("name", "`name`"), ("age", "`age`")],
filter="age >= 18",
inputs=["age", "name"],
)
def test_incremental_refresh_after_append(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").add([{"name": "alan", "age": 41}])
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
assert view.table.count_rows() == 4
assert view.refresh().mode == "no_op"
def test_incremental_refresh_after_update(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36, 85]
def test_legacy_storage_source_update_rebuilds(tmp_path):
db = lancedb.connect(
tmp_path,
storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"},
)
db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}])
view = db.create_materialized_view("copy", "people")
view.refresh()
db.open_table("people").update(where="name = 'kid'", values={"age": 8})
result = view.refresh()
assert result.mode == "rebuild"
rows = view.table.search().to_list()
assert sorted(row["age"] for row in rows) == [8, 36]
def test_list_and_not_a_view(tmp_path):
db = make_db(tmp_path)
db.create_materialized_view("adults", "people", where="age >= 18")
assert db.list_materialized_views() == ["adults"]
with pytest.raises(ValueError, match="not a materialized view"):
db.open_materialized_view("people")
def test_invalid_expression_fails_at_create(tmp_path):
db = make_db(tmp_path)
with pytest.raises(Exception, match="missing"):
db.create_materialized_view("bad", "people", select=[("x", "missing + 1")])
assert "bad" not in db.list_tables().tables
@pytest.mark.asyncio
async def test_async_create_refresh_and_open(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view(
"shouts", "people", select=[("shout", "upper(name)")]
)
result = await view.refresh()
assert result.mode == "rebuild"
assert result.rows_written == 1
reopened = await db.open_materialized_view("shouts")
definition = await reopened.definition()
assert definition.projections == [("shout", "upper(name)")]
assert await db.list_materialized_views() == ["shouts"]
@pytest.mark.asyncio
async def test_async_incremental(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("copy", "people")
await view.refresh()
table = await db.open_table("people")
await table.add([{"name": "alan", "age": 41}])
result = await view.refresh()
assert result.mode == "incremental"
assert result.rows_written == 1
def test_source_requires_stable_row_ids(tmp_path):
db = lancedb.connect(tmp_path)
db.create_table("plain", [{"x": 1}])
with pytest.raises(Exception, match="stable row ids"):
db.create_materialized_view("v", "plain")
def test_bare_select_names_are_quoted(tmp_path):
db = lancedb.connect(tmp_path, storage_options=STABLE_ROW_IDS)
db.create_table("odd_names", [{"order item": "widget", "select": 2}])
view = db.create_materialized_view(
"quoted", "odd_names", select=["order item", "select"]
)
result = view.refresh()
assert result.rows_written == 1
rows = view.table.search().to_list()
assert rows[0]["order item"] == "widget"
assert rows[0]["select"] == 2
@pytest.mark.asyncio
async def test_async_remote_is_refused_without_network():
db = await lancedb.connect_async(
"db://nowhere", api_key="sk_test", region="us-east-1"
)
with pytest.raises(NotImplementedError, match="local"):
await db.create_materialized_view("v", "src")
with pytest.raises(NotImplementedError, match="local"):
await db.open_materialized_view("v")
with pytest.raises(NotImplementedError, match="local"):
await db.list_materialized_views()
def test_scalar_select_is_one_column(tmp_path):
db = make_db(tmp_path)
view = db.create_materialized_view("just_name", "people", select="name")
view.refresh()
rows = view.table.search().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
assert sorted(row["name"] for row in rows) == ["ada", "grace", "kid"]
@pytest.mark.asyncio
async def test_async_scalar_select_is_one_column(tmp_path):
db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS)
await db.create_table("people", [{"name": "ada", "age": 36}])
view = await db.create_materialized_view("just_name", "people", select="name")
await view.refresh()
rows = await view.table.query().to_list()
assert set(rows[0]) - {"__source_row_id"} == {"name"}
def test_limit_above_i64_max_is_refused(tmp_path):
db = make_db(tmp_path)
with pytest.raises(ValueError, match="exceeds the maximum"):
db.create_materialized_view("too_big", "people", limit=2**63)
# The boundary is fine, and zero still means an empty view.
db.create_materialized_view("at_max", "people", limit=2**63 - 1)
empty = db.create_materialized_view("none", "people", limit=0)
empty.refresh()
assert empty.table.count_rows() == 0
def _namespace_db(tmp_path):
return lancedb.connect_namespace(
"dir",
{"root": str(tmp_path)},
storage_options=STABLE_ROW_IDS,
)
def test_namespace_connection_materialized_views(tmp_path):
db = _namespace_db(tmp_path)
db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options=STABLE_ROW_IDS,
)
view = db.create_materialized_view("adults", "people", where="age >= 18")
view.refresh()
assert view.table.count_rows() == 1
assert db.list_materialized_views() == ["adults"]
reopened = db.open_materialized_view("adults")
assert reopened.definition.source_table == "people"
with pytest.raises(ValueError, match="not a materialized view"):
db.open_materialized_view("people")
@pytest.mark.asyncio
async def test_async_namespace_connection_materialized_views(tmp_path):
db = lancedb.connect_namespace_async(
"dir",
{"root": str(tmp_path)},
storage_options=STABLE_ROW_IDS,
)
await db.create_table(
"people",
[{"name": "ada", "age": 36}, {"name": "kid", "age": 7}],
storage_options=STABLE_ROW_IDS,
)
view = await db.create_materialized_view("adults", "people", where="age >= 18")
await view.refresh()
assert await view.table.count_rows() == 1
assert await db.list_materialized_views() == ["adults"]
reopened = await db.open_materialized_view("adults")
assert (await reopened.definition()).source_table == "people"
# The view's table came through the namespace, not straight from the
# inner connection: a bare inner table carries no namespace context, so
# its pushdown routing differs from a table the namespace opened.
through_namespace = await db.open_table("adults")
for handle in (view.table, reopened.table):
assert (
handle._route_pushdown_to_rust == through_namespace._route_pushdown_to_rust
)
assert handle._namespace_path == through_namespace._namespace_path
python/python/tests/test_remote_db.py
+83 -10
View File
@@ -242,8 +242,8 @@ def test_remote_table_branches_sync():
table.branches.delete("exp")
def test_remote_table_branch_merge_defaults_to_execute():
merge_bodies = []
def test_remote_table_cherry_pick_defaults_to_execute():
cherry_pick_bodies = []
diff = {
"fromBranch": "exp",
"parentVersion": 1,
@@ -265,8 +265,7 @@ def test_remote_table_branch_merge_defaults_to_execute():
"changedColumns": [],
"addedIndexes": [],
"removedIndexes": [],
"mergeable": True,
"mergeBlockers": [],
"errors": [],
}
def handler(request):
@@ -276,11 +275,11 @@ def test_remote_table_branch_merge_defaults_to_execute():
else:
content_len = int(request.headers.get("Content-Length"))
request_body = json.loads(request.rfile.read(content_len))
merge_bodies.append(request_body)
cherry_pick_bodies.append(request_body)
dry_run = request_body["dry_run"]
status = 200 if dry_run else 409
body = {
"status": "ready" if dry_run else "rejected",
"status": "ready" if dry_run else "failed",
"diff": diff,
"preview": {"promotedColumns": []},
}
@@ -292,10 +291,10 @@ def test_remote_table_branch_merge_defaults_to_execute():
with mock_lancedb_connection(handler) as db:
branches = db.open_table("test").branches
assert branches.merge("exp")["status"] == "rejected"
assert branches.merge("exp", dry_run=True)["status"] == "ready"
assert branches.cherry_pick("exp")["status"] == "failed"
assert branches.cherry_pick("exp", dry_run=True)["status"] == "ready"
assert merge_bodies == [
assert cherry_pick_bodies == [
{"from_branch": "exp", "dry_run": False},
{"from_branch": "exp", "dry_run": True},
]
@@ -901,11 +900,85 @@ def test_remote_create_index_async_returns_job():
table = db.create_table("test", [{"id": 1}])
job = table.create_index_async("id", config=BTree())
assert job.id == "job-1"
job.wait(timeout=timedelta(seconds=30))
assert job.wait(timeout=timedelta(seconds=30)) is None
assert len(describe_calls) == 2
job.cancel()
def test_remote_refresh_async_returns_typed_terminal_result():
terminal_result = {
"rows_assigned": 12,
"rows_failed": 0,
"rows_remaining": 0,
"source_version": 7,
"published_version": 8,
}
def handler(request):
content_len = int(request.headers.get("Content-Length", 0))
body = request.rfile.read(content_len) if content_len > 0 else b""
if request.path == "/v1/table/test/backfill_column":
assert json.loads(body)["column"] == "derived"
request.send_response(202)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"job_id": "refresh-1"}')
elif request.path == "/v1/jobs/describe":
assert json.loads(body)["job_id"] == "refresh-1"
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
{
"job_id": "refresh-1",
"job_type": "function_refresh",
"job_state": "DONE",
"result": terminal_result,
}
).encode()
)
elif request.path == "/v1/table/test/create/?mode=create":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b"{}")
elif request.path == "/v1/table/test/describe/":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(
json.dumps(
{
"version": 1,
"schema": {
"fields": [
{
"name": "id",
"type": {"type": "int64"},
"nullable": False,
}
]
},
}
).encode()
)
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
table = db.create_table("test", [{"id": 1}])
job = table.refresh_column_async("derived")
assert job.id == "refresh-1"
result = job.wait(timeout=timedelta(seconds=30))
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.model_dump() == terminal_result
assert result.rows_filled == 12
assert result.version == 8
def test_remote_job_wait_raises_on_failure():
from lancedb.exceptions import JobFailedError
from lancedb.index import BTree
python/python/tests/test_table.py
+18 -3
View File
@@ -1467,7 +1467,7 @@ def test_create_index_async_returns_done_job(mem_db: DBConnection):
table = mem_db.create_table("job_test", [{"id": i} for i in range(10)])
job = table.create_index_async("id", config=BTree())
assert job.id is None
job.wait()
assert job.wait() is None
assert len(table.list_indices()) == 1
job.cancel()
@@ -3947,10 +3947,21 @@ def test_refresh_column_async_returns_job(tmp_path):
job = table.refresh_column_async("doubled")
assert job.id is None # in-process jobs have no server id
assert job.wait() is None
result = job.wait()
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.rows_assigned == 2
assert result.rows_failed == 0
assert result.rows_remaining == 0
assert result.source_version == 2
assert result.published_version == 3
assert job.status() == "finished"
assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
no_op = table.refresh_column_async("doubled").wait()
assert no_op.rows_assigned == 0
assert no_op.source_version == 3
assert no_op.published_version is None
# Bad input raises at the call, not through the job.
with pytest.raises(Exception, match="not a computed column"):
table.refresh_column_async("x")
@@ -3963,6 +3974,10 @@ async def test_refresh_column_async_job_async_table(tmp_path):
await table.add_columns(computed={"tripled": "x * 3"})
job = await table.refresh_column_async("tripled")
assert await job.wait() is None
result = await job.wait()
assert isinstance(result, lancedb.RefreshColumnResult)
assert result.rows_assigned == 1
assert result.source_version == 2
assert result.published_version == 3
assert await job.status() == "finished"
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
python/src/connection.rs
+35 -1
View File
@@ -333,6 +333,40 @@ impl Connection {
})
}
#[pyo3(signature = (name, source, projections=None, filter=None, limit=None))]
pub fn create_materialized_view(
self_: PyRef<'_, Self>,
name: String,
source: String,
projections: Option<Vec<(String, String)>>,
filter: Option<String>,
limit: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let mut builder = inner.create_materialized_view(name, source);
if let Some(projections) = projections {
builder = builder.select(projections);
}
if let Some(filter) = filter {
builder = builder.only_if(filter);
}
if let Some(limit) = limit {
builder = builder.limit(limit);
}
let view = builder.execute().await.infer_error()?;
Ok(Table::new(view.table().clone()))
})
}
pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.get_inner()?.clone();
future_into_py(self_.py(), async move {
let views = inner.list_materialized_views().await.infer_error()?;
Ok(views.into_iter().map(|view| view.name).collect::<Vec<_>>())
})
}
#[pyo3(signature = (name, namespace_path=None))]
pub fn drop_table(
self_: PyRef<'_, Self>,
@@ -575,7 +609,7 @@ impl Connection {
.create_function_async(request)
.await
.infer_error()
.map(crate::job::FunctionJob::new)
.map(crate::job::Job::new_typed)
})
}
python/src/job.rs
+18 -55
View File
@@ -5,72 +5,33 @@ use std::sync::Arc;
use crate::runtime::future_into_py;
use pyo3::{Bound, PyAny, PyRef, PyResult, pyclass, pymethods};
use serde::Serialize;
use crate::error::PythonErrorExt;
#[pyclass]
pub struct Job {
inner: Arc<lancedb::Job>,
}
/// Python bridge for a typed remote Function registration job.
///
/// The public Python layer decodes the canonical JSON returned by `wait`
/// into its immutable `FunctionVersion` model.
#[pyclass]
pub struct FunctionJob {
inner: Arc<lancedb::Job<lancedb::function::FunctionVersion>>,
}
impl FunctionJob {
pub(crate) fn new(inner: lancedb::Job<lancedb::function::FunctionVersion>) -> Self {
Self {
inner: Arc::new(inner),
}
}
inner: Arc<lancedb::Job<std::result::Result<Option<String>, String>>>,
}
impl Job {
pub(crate) fn new(inner: lancedb::Job) -> Self {
Self {
inner: Arc::new(inner),
inner: Arc::new(inner.map(|()| Ok(None))),
}
}
}
#[pymethods]
impl FunctionJob {
#[getter]
pub fn id(&self) -> Option<String> {
self.inner.id().map(str::to_string)
}
pub fn status(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(
self_.py(),
async move { inner.status().await.infer_error() },
)
}
pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner
.wait()
.await
.infer_error()?
.to_canonical_json()
.infer_error()
})
}
pub fn cancel(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.cancel().await.infer_error()?;
Ok(())
})
pub(crate) fn new_typed<T>(inner: lancedb::Job<T>) -> Self
where
T: Clone + Serialize + Send + Sync + 'static,
{
Self {
inner: Arc::new(inner.map(|result| {
serde_json::to_string(&result)
.map(Some)
.map_err(|error| format!("failed to serialize typed job result: {error}"))
})),
}
}
}
@@ -92,8 +53,10 @@ impl Job {
pub fn wait(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.wait().await.infer_error()?;
Ok(None::<()>)
let result = inner.wait().await.infer_error()?;
result
.map_err(|message| lancedb::Error::Runtime { message })
.infer_error()
})
}
python/src/lib.rs
+3 -3
View File
@@ -16,8 +16,8 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery};
use session::Session;
use table::{
AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken,
LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, Table, UpdateFieldMetadataResult,
UpdateResult,
LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult,
Table, UpdateFieldMetadataResult, UpdateResult,
};
pub mod arrow;
@@ -47,7 +47,6 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<Session>()?;
m.add_class::<Table>()?;
m.add_class::<crate::job::Job>()?;
m.add_class::<crate::job::FunctionJob>()?;
m.add_class::<crate::job::JobInfo>()?;
m.add_class::<crate::job::JobDescription>()?;
m.add_class::<crate::job::JobFailureInfo>()?;
@@ -60,6 +59,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<RecordBatchStream>()?;
m.add_class::<AddColumnsResult>()?;
m.add_class::<RefreshColumnResult>()?;
m.add_class::<RefreshMaterializedViewResult>()?;
m.add_class::<AlterColumnsResult>()?;
m.add_class::<UpdateFieldMetadataResult>()?;
m.add_class::<AddResult>()?;
python/src/table.rs
+58 -3
View File
@@ -441,6 +441,41 @@ impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RefreshMaterializedViewResult {
pub mode: String,
pub rows_written: u64,
pub source_version: u64,
pub version: u64,
}
#[pymethods]
impl RefreshMaterializedViewResult {
pub fn __repr__(&self) -> String {
format!(
"RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})",
self.mode, self.rows_written, self.source_version, self.version
)
}
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = match result.mode {
lancedb::RefreshMode::Rebuild => "rebuild",
lancedb::RefreshMode::Incremental => "incremental",
lancedb::RefreshMode::NoOp => "no_op",
};
Self {
mode: mode.to_string(),
rows_written: result.rows_written,
source_version: result.source_version,
version: result.version,
}
}
}
#[pymethods]
impl AddColumnsResult {
pub fn __repr__(&self) -> String {
@@ -1584,7 +1619,27 @@ impl Table {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let job = inner.refresh_column_async(column).await.infer_error()?;
Ok(crate::job::Job::new(job))
Ok(crate::job::Job::new_typed(job))
})
}
#[pyo3(signature = (full=false, source_version=None))]
pub fn refresh_materialized_view(
self_: PyRef<'_, Self>,
full: bool,
source_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
if let Some(version) = source_version {
builder = builder.source_version(version);
}
let result = builder.execute().await.infer_error()?;
Ok(RefreshMaterializedViewResult::from(result))
})
}
@@ -1885,7 +1940,7 @@ impl Branches {
}
#[pyo3(signature = (from_branch, dry_run=false))]
pub fn merge(
pub fn cherry_pick(
self_: PyRef<'_, Self>,
from_branch: String,
dry_run: bool,
@@ -1893,7 +1948,7 @@ impl Branches {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner
.merge_branch(&from_branch, dry_run)
.cherry_pick(&from_branch, dry_run)
.await
.infer_error()?;
Python::attach(|py| struct_to_wire_py(py, &result))
rust/lancedb/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb"
version = "0.38.0-beta.3"
version = "0.38.0-beta.6"
edition.workspace = true
description = "LanceDB: A serverless, low-latency vector database for AI applications"
license.workspace = true
rust/lancedb/src/function.rs
+12 -2
View File
@@ -1,7 +1,8 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Canonical values exchanged with the Enterprise Function service.
//! Canonical Function values exchanged with the Enterprise service, plus the
//! backend-neutral terminal result of a computed-column refresh.
//!
//! This module contains client/wire values only. Catalog persistence,
//! environment bake, secret resolution, and execution are owned by Sophon.
@@ -580,13 +581,22 @@ impl FunctionBinding {
impl_json!(FunctionBinding);
/// Stable terminal result of a remote Function-column refresh Job.
/// Stable terminal result of an expression-backed or Function-backed column
/// refresh [`crate::Job`].
///
/// Local refresh jobs produce this value in process. LanceDB Cloud and
/// Enterprise decode the same value from the durable job's terminal payload.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RefreshColumnResult {
/// Rows assigned a value by this refresh.
pub rows_assigned: u64,
/// Rows whose computation failed.
pub rows_failed: u64,
/// Rows that still need a value when the job completes.
pub rows_remaining: u64,
/// Exact table version the refresh read.
pub source_version: u64,
/// Table version made visible by the refresh, when one was published.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub published_version: Option<u64>,
}
rust/lancedb/src/job.rs
+115 -37
View File
@@ -6,7 +6,7 @@
use std::sync::Arc;
use async_trait::async_trait;
use serde::de::DeserializeOwned;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use tokio::sync::watch;
use tokio::task::{AbortHandle, JoinHandle};
@@ -26,20 +26,16 @@ pub(crate) trait JobHandle: Send + Sync {
}
/// A backend-neutral successful terminal result.
///
/// Local operations do not carry a value. Remote operations may carry JSON
/// that the public [`Job`] decodes according to its result type.
#[derive(Clone)]
pub(crate) struct TerminalResult {
#[allow(dead_code)] // Typed remote submit endpoints consume this after Slice 1.
value: Option<Value>,
#[allow(dead_code)] // Preserved so typed decode errors retain request correlation.
request_id: Option<String>,
}
impl TerminalResult {
pub(crate) fn local() -> Self {
fn local(value: Value) -> Self {
Self {
value: None,
value: Some(value),
request_id: None,
}
}
@@ -51,23 +47,31 @@ impl TerminalResult {
}
}
#[allow(dead_code)] // Exercised by the remote typed-result fixtures in Slice 1.
fn decode<T: DeserializeOwned>(self) -> Result<T> {
let request_id = self.request_id.unwrap_or_default();
let value = self.value.ok_or_else(|| Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
let value = self.value.ok_or_else(|| match &self.request_id {
Some(request_id) => Error::Http {
source: "successful typed job response did not contain a result".into(),
request_id: request_id.clone(),
status_code: None,
},
None => Error::Runtime {
message: "successful typed job did not contain a result".to_string(),
},
})?;
serde_json::from_value(value).map_err(|error| Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
serde_json::from_value(value).map_err(|error| match self.request_id {
Some(request_id) => Error::Http {
source: format!("failed to parse typed job result: {error}").into(),
request_id,
status_code: None,
},
None => Error::Runtime {
message: format!("failed to parse typed job result: {error}"),
},
})
}
}
type ResultDecoder<T> = fn(TerminalResult) -> Result<T>;
type ResultDecoder<T> = Arc<dyn Fn(TerminalResult) -> Result<T> + Send + Sync>;
enum JobInner<T> {
Handle {
@@ -79,7 +83,9 @@ enum JobInner<T> {
/// A handle to an operation that may still be running.
///
/// The operation may already be complete when the handle is created.
/// The operation may already be complete when the handle is created. `T` is
/// the endpoint's successful terminal result; unit-result operations use the
/// default `Job<()>`.
pub struct Job<T = ()>
where
T: Clone + Send + Sync + 'static,
@@ -111,15 +117,10 @@ impl Job<()> {
Self {
inner: JobInner::Handle {
handle,
decode: |_| Ok(()),
decode: Arc::new(|_| Ok(())),
},
}
}
/// A unit-result job running as a task in this process.
pub(crate) fn spawned(task: JoinHandle<Result<()>>) -> Self {
Self::new(Box::new(SpawnedJob::new(task)))
}
}
impl<T> Job<T>
@@ -131,12 +132,22 @@ where
Self {
inner: JobInner::Handle {
handle,
decode: TerminalResult::decode::<T>,
decode: Arc::new(TerminalResult::decode::<T>),
},
}
}
}
impl<T> Job<T>
where
T: Clone + Serialize + DeserializeOwned + Send + Sync + 'static,
{
/// A typed job running as a task in this process.
pub(crate) fn spawned(task: JoinHandle<Result<T>>) -> Self {
Self::new_typed(Box::new(SpawnedJob::new(task)))
}
}
impl<T> Job<T>
where
T: Clone + Send + Sync + 'static,
@@ -169,11 +180,13 @@ where
/// Waits until the operation reaches a terminal state.
///
/// Returns the endpoint's typed result. Unit-result jobs return `()`.
///
/// Returns [`crate::Error::JobFailed`] if the operation failed and
/// [`crate::Error::JobCancelled`] if it was cancelled.
pub async fn wait(&self) -> Result<T> {
match &self.inner {
JobInner::Handle { handle, decode } => decode(handle.wait().await?),
JobInner::Handle { handle, decode } => (decode)(handle.wait().await?),
JobInner::Completed(result) => Ok(result.clone()),
}
}
@@ -187,21 +200,53 @@ where
JobInner::Completed(_) => Ok(()),
}
}
/// Maps a successful terminal result without changing the job lifecycle.
/// The mapping may run once for each call to [`Job::wait`], so it should
/// be deterministic and free of externally visible side effects.
///
/// ```
/// use lancedb::{Job, function::RefreshColumnResult};
///
/// # async fn rows_assigned(
/// # job: Job<RefreshColumnResult>,
/// # ) -> lancedb::Result<u64> {
/// let job = job.map(|result| result.rows_assigned);
/// job.wait().await
/// # }
/// ```
pub fn map<U, F>(self, map: F) -> Job<U>
where
U: Clone + Send + Sync + 'static,
F: Fn(T) -> U + Send + Sync + 'static,
{
match self.inner {
JobInner::Handle { handle, decode } => Job {
inner: JobInner::Handle {
handle,
decode: Arc::new(move |result| Ok(map((decode)(result)?))),
},
},
JobInner::Completed(result) => Job {
inner: JobInner::Completed(map(result)),
},
}
}
}
/// How an in-process operation ended. Cloneable so every waiter can be given
/// the outcome; [`Error`] is not, so failures share one behind an [`Arc`].
#[derive(Clone)]
enum Outcome {
Succeeded,
Succeeded(TerminalResult),
Failed(Arc<Error>),
Cancelled,
}
impl Outcome {
fn into_result(self) -> Result<()> {
fn into_result(self) -> Result<TerminalResult> {
match self {
Self::Succeeded => Ok(()),
Self::Succeeded(result) => Ok(result),
Self::Failed(source) => Err(Error::JobFailed {
job_id: None,
failure: JobFailure::from_source(source),
@@ -220,12 +265,20 @@ struct SpawnedJob {
}
impl SpawnedJob {
fn new(task: JoinHandle<Result<()>>) -> Self {
fn new<T>(task: JoinHandle<Result<T>>) -> Self
where
T: Serialize + Send + 'static,
{
let abort = task.abort_handle();
let (tx, outcome) = watch::channel(None);
tokio::spawn(async move {
let outcome = match task.await {
Ok(Ok(())) => Outcome::Succeeded,
Ok(Ok(result)) => match serde_json::to_value(result) {
Ok(value) => Outcome::Succeeded(TerminalResult::local(value)),
Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
message: format!("failed to serialize job result: {err}"),
})),
},
Ok(Err(err)) => Outcome::Failed(Arc::new(err)),
Err(err) if err.is_cancelled() => Outcome::Cancelled,
Err(err) => Outcome::Failed(Arc::new(Error::Runtime {
@@ -243,7 +296,7 @@ impl JobHandle for SpawnedJob {
async fn status(&self) -> Result<String> {
let label = match &*self.outcome.borrow() {
None => "running",
Some(Outcome::Succeeded) => "finished",
Some(Outcome::Succeeded(_)) => "finished",
Some(Outcome::Failed(_)) => "failed",
Some(Outcome::Cancelled) => "cancelled",
};
@@ -256,12 +309,11 @@ impl JobHandle for SpawnedJob {
.wait_for(|outcome| outcome.is_some())
.await
.map_err(|_| Error::Runtime {
message: "index job outcome was dropped before it completed".to_string(),
message: "job outcome was dropped before it completed".to_string(),
})?
.clone()
.expect("wait_for returns once an outcome is set");
settled.into_result()?;
Ok(TerminalResult::local())
settled.into_result()
}
async fn cancel(&self) -> Result<()> {
@@ -269,3 +321,29 @@ impl JobHandle for SpawnedJob {
Ok(())
}
}
#[cfg(test)]
mod tests {
use std::future::pending;
use super::*;
#[tokio::test]
async fn mapped_spawned_job_reuses_outcome() {
let job = Job::spawned(tokio::spawn(async { Ok(41_u64) })).map(|value| value + 1);
assert_eq!(job.wait().await.unwrap(), 42);
assert_eq!(job.wait().await.unwrap(), 42);
assert_eq!(job.status().await.unwrap(), "finished");
}
#[tokio::test]
async fn mapped_spawned_job_preserves_cancellation() {
let job = Job::spawned(tokio::spawn(async { pending::<Result<u64>>().await }))
.map(|value| value.to_string());
job.cancel().await.unwrap();
assert!(matches!(job.wait().await, Err(Error::JobCancelled { .. })));
assert_eq!(job.status().await.unwrap(), "cancelled");
}
}
rust/lancedb/src/lib.rs
+3 -1
View File
@@ -211,7 +211,9 @@ pub use function::FunctionVersion;
pub use job::Job;
use lance_index::vector::ApproxMode as LanceApproxMode;
use lance_linalg::distance::DistanceType as LanceDistanceType;
pub use materialized_view::{MaterializedView, MaterializedViewDefinition};
pub use materialized_view::{
MaterializedView, MaterializedViewDefinition, RefreshMaterializedViewResult, RefreshMode,
};
/// Re-export of the [`metrics`](https://docs.rs/metrics) crate facade. Enable
/// the `metrics` feature to publish LanceDB's internal metrics; install any
/// `metrics`-compatible recorder to collect them. See also [`metrics_otel`] for
rust/lancedb/src/materialized_view.rs
+63 -1
View File
@@ -9,6 +9,11 @@
//! metadata; a kind added later reads back as unrefreshable, not as a plain
//! table. Queries, indexes and search work on the view unchanged.
pub mod refresh;
#[cfg(test)]
mod differential;
use std::collections::HashMap;
use std::sync::Arc;
@@ -27,6 +32,8 @@ use crate::table::refresh::quote_identifier;
use crate::table::{ColumnDefinition, ColumnKind};
use crate::{Error, Result};
pub use refresh::{RefreshMaterializedViewResult, RefreshMode};
/// Schema metadata key holding the view definition, as kind-tagged JSON.
pub const DEFINITION_META_KEY: &str = "mv.definition";
@@ -736,6 +743,12 @@ pub async fn prepare_declaration(
),
});
}
refresh::ensure_no_mem_wal(
native.dataset.get().await?.as_ref(),
"source table",
resolved.name(),
)
.await?;
let source_schema = resolved.schema().await?;
let source_metadata = source_schema.metadata().clone();
let (definition, mut fields, lineage) = plan(
@@ -909,6 +922,54 @@ impl MaterializedView {
pub fn definition(&self) -> &MaterializedViewDefinition {
&self.definition
}
/// Recompute the view from its source.
///
/// By default the refresh is incremental when the source's changes can be
/// reconciled into the view, and otherwise rebuilds; see
/// [`RefreshMaterializedViewBuilder`].
///
/// ```no_run
/// # #![recursion_limit = "256"]
/// # use lancedb::materialized_view::MaterializedView;
/// # async fn refresh(view: &MaterializedView) -> Result<(), Box<dyn std::error::Error>> {
/// let result = view.refresh().execute().await?;
/// println!("{:?}: {} rows", result.mode, result.rows_written);
/// # Ok(())
/// # }
/// ```
pub fn refresh(&self) -> RefreshMaterializedViewBuilder {
RefreshMaterializedViewBuilder {
view: self.clone(),
full: false,
source_version: None,
}
}
}
/// Builds a refresh. Created by [`MaterializedView::refresh`].
pub struct RefreshMaterializedViewBuilder {
view: MaterializedView,
full: bool,
source_version: Option<u64>,
}
impl RefreshMaterializedViewBuilder {
/// Rebuild the view even where an incremental refresh would do.
pub fn full(mut self, full: bool) -> Self {
self.full = full;
self
}
/// Refresh to this source table version instead of the latest.
pub fn source_version(mut self, version: u64) -> Self {
self.source_version = Some(version);
self
}
pub async fn execute(self) -> Result<RefreshMaterializedViewResult> {
refresh::execute_refresh(&self.view.table, self.full, self.source_version).await
}
}
impl Connection {
@@ -918,6 +979,7 @@ impl Connection {
/// metadata; refresh computes the rows. Local databases only.
///
/// ```no_run
/// # #![recursion_limit = "256"]
/// # use lancedb::Connection;
/// # async fn create(conn: &Connection) -> Result<(), Box<dyn std::error::Error>> {
/// let view = conn
@@ -926,7 +988,7 @@ impl Connection {
/// .only_if("age >= 18")
/// .execute()
/// .await?;
/// println!("{}", view.definition().source_table);
/// view.refresh().execute().await?;
/// # Ok(())
/// # }
/// ```
rust/lancedb/src/materialized_view/differential.rs
+730
View File
@@ -0,0 +1,730 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Differential refresh testing.
//!
//! The refresh contract is a property: after any sequence of source
//! mutations, a view maintained by default (incremental-where-possible)
//! refreshes equals the definition evaluated against the source directly,
//! and so does a forced rebuild. The oracle is an independent read of the
//! source -- plain column scan, filter applied in Rust -- so it shares
//! nothing with the refresh path it checks.
//!
//! The oracle runs after every step, not just at the end: a later mutation
//! that forces a rebuild would silently heal an incremental error, and those
//! transient errors are exactly the bugs this exists to catch.
use arrow_array::{Float32Array, Int32Array, RecordBatch};
use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
use futures::{StreamExt, TryStreamExt};
use lance::dataset::NewColumnTransform;
use std::sync::Arc;
use super::MaterializedView;
use super::refresh::RefreshMode;
use crate::connect;
use crate::connection::Connection;
use crate::query::{ExecutableQuery, QueryBase, Select};
use crate::table::{CompactionOptions, OptimizeAction, Table};
/// One source mutation, one per correctness-relevant class.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum SrcOp {
/// Fresh non-colliding ids of both parities, so every other op has
/// view-resident rows to act on: the only op that should refresh
/// incrementally.
AppendNew,
/// Deletion in surviving fragments must break the pure-append check.
DeleteEven,
/// An in-place update; on the filtered shape it crosses the predicate,
/// so rows must leave the view.
UpdateOddScore,
/// Fragment rewrite/renumber must break the pure-append check.
Compact,
/// A column the view does not read must NOT force a rebuild.
AddColumn,
/// merge_insert commits an Update whose by-source arm deletes rows, so a
/// classifier that reads Update as "changed only" loses those deletions.
MergeDropLargest,
/// merge_insert that both changes existing rows and inserts new ones in
/// one transaction.
MergeUpsert,
}
const ALL_OPS: [SrcOp; 7] = [
SrcOp::AppendNew,
SrcOp::DeleteEven,
SrcOp::UpdateOddScore,
SrcOp::Compact,
SrcOp::AddColumn,
SrcOp::MergeDropLargest,
SrcOp::MergeUpsert,
];
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum Shape {
/// SELECT id, score.
Identity,
/// SELECT id, score WHERE score > 50: additionally sensitive to rows
/// crossing the predicate.
Filtered,
/// SELECT id, score LIMIT 4. Which rows are held depends on the order
/// they were first materialized, so the oracle checks containment and
/// the cap rather than equality.
Limited,
}
impl Shape {
fn filter(&self) -> Option<&'static str> {
match self {
Self::Identity | Self::Limited => None,
Self::Filtered => Some("score > 50"),
}
}
fn matches(&self, score: f32) -> bool {
match self {
Self::Identity | Self::Limited => true,
Self::Filtered => score > 50.0,
}
}
fn limit(&self) -> Option<usize> {
match self {
Self::Limited => Some(4),
_ => None,
}
}
}
struct Case {
conn: Connection,
source: Table,
view: MaterializedView,
shape: Shape,
next_id: i32,
added_columns: u32,
}
fn rows_batch(ids: &[i32]) -> RecordBatch {
let scores: Vec<f32> = ids.iter().map(|id| (*id * 10) as f32).collect();
RecordBatch::try_new(
Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, true),
ArrowField::new("score", DataType::Float32, true),
])),
vec![
Arc::new(Int32Array::from(ids.to_vec())),
Arc::new(Float32Array::from(scores)),
],
)
.unwrap()
}
fn merge_batch(ids: &[i32]) -> RecordBatch {
let scores: Vec<f32> = ids.iter().map(|id| (*id * 10 + 5) as f32).collect();
RecordBatch::try_new(
Arc::new(ArrowSchema::new(vec![
ArrowField::new("id", DataType::Int32, true),
ArrowField::new("score", DataType::Float32, true),
])),
vec![
Arc::new(Int32Array::from(ids.to_vec())),
Arc::new(Float32Array::from(scores)),
],
)
.unwrap()
}
impl Case {
async fn new(shape: Shape) -> Self {
let conn = connect("memory://").execute().await.unwrap();
let source = conn
.create_table("src", rows_batch(&[1, 2, 3, 4]))
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let mut builder = conn
.create_materialized_view("view", "src")
.select([("id", "id"), ("score", "score")]);
if let Some(filter) = shape.filter() {
builder = builder.only_if(filter);
}
if let Some(limit) = shape.limit() {
builder = builder.limit(limit as u64);
}
let view = builder.execute().await.unwrap();
Self {
conn,
source,
view,
shape,
next_id: 100,
added_columns: 0,
}
}
async fn apply(&mut self, op: SrcOp) {
match op {
SrcOp::AppendNew => {
// Mixed parity: the middle id is odd, so UpdateOddScore always
// has a filter-matching appended row to evict.
let ids = vec![self.next_id, self.next_id + 101, self.next_id + 202];
self.next_id += 303;
self.source.add(rows_batch(&ids)).execute().await.unwrap();
}
SrcOp::DeleteEven => {
self.source.delete("id % 2 = 0").await.unwrap();
}
SrcOp::UpdateOddScore => {
self.source
.update()
.column("score", "-1.0")
.only_if("id % 2 = 1")
.execute()
.await
.unwrap();
}
SrcOp::Compact => {
self.source
.optimize(OptimizeAction::Compact {
options: CompactionOptions::default(),
remap_options: None,
})
.await
.unwrap();
}
SrcOp::MergeDropLargest => {
let mut ids = self.source_ids().await;
ids.sort_unstable();
ids.pop();
if ids.is_empty() {
return;
}
let batch = rows_batch(&ids);
let reader =
arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
let mut merge = self.source.merge_insert(&["id"]);
merge.when_not_matched_by_source_delete(None);
merge.execute(Box::new(reader)).await.unwrap();
}
SrcOp::MergeUpsert => {
let mut ids = self.source_ids().await;
ids.sort_unstable();
// One row that exists (updated in place) and one that does not.
let existing = ids.first().copied().unwrap_or(self.next_id);
let fresh = self.next_id;
self.next_id += 1;
let batch = merge_batch(&[existing, fresh]);
let reader =
arrow_array::RecordBatchIterator::new(vec![Ok(batch.clone())], batch.schema());
let mut merge = self.source.merge_insert(&["id"]);
merge
.when_matched_update_all(None)
.when_not_matched_insert_all();
merge.execute(Box::new(reader)).await.unwrap();
}
SrcOp::AddColumn => {
self.added_columns += 1;
let field = ArrowField::new(
format!("extra_{}", self.added_columns),
DataType::Int32,
true,
);
self.source
.add_columns()
.transform(NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(
vec![field],
))))
.execute()
.await
.unwrap();
}
}
}
async fn source_ids(&self) -> Vec<i32> {
read_rows(
self.source
.query()
.select(Select::columns(&["id", "score"])),
)
.await
.into_iter()
.map(|(id, _)| id)
.collect()
}
/// The definition's result, read independently of the refresh path:
/// plain column scan, filter applied here, sorted.
async fn oracle(&self) -> Vec<(i32, i32)> {
let mut rows = read_rows(
self.source
.query()
.select(Select::columns(&["id", "score"])),
)
.await
.into_iter()
.filter(|(_, score)| self.shape.matches(*score as f32))
.collect::<Vec<_>>();
rows.sort_unstable();
rows
}
async fn view_rows(&self) -> Vec<(i32, i32)> {
let mut rows = read_rows(
self.view
.table()
.query()
.select(Select::columns(&["id", "score"])),
)
.await;
rows.sort_unstable();
rows
}
async fn check(&self, label: &str) -> Result<(), String> {
let expected = self.oracle().await;
let actual = self.view_rows().await;
let Some(cap) = self.shape.limit() else {
if expected != actual {
return Err(format!(
"{label}: view diverged from oracle\n expected: {expected:?}\n actual: {actual:?}"
));
}
return Ok(());
};
// A capped view holds some subset of the definition's result, never
// more than the cap, and never the same row twice.
if actual.len() > cap {
return Err(format!(
"{label}: view holds {} rows, over its cap of {cap}: {actual:?}",
actual.len()
));
}
let mut unique = actual.clone();
unique.dedup();
if unique.len() != actual.len() {
return Err(format!("{label}: view holds a row twice: {actual:?}"));
}
if let Some(stray) = actual.iter().find(|row| !expected.contains(row)) {
return Err(format!(
"{label}: view holds {stray:?}, which the definition does not select: {expected:?}"
));
}
// Below the cap the view must be complete, or a row was lost.
if actual.len() < cap.min(expected.len()) {
return Err(format!(
"{label}: view holds {} of {} selectable rows under a cap of {cap}: {actual:?}",
actual.len(),
expected.len()
));
}
Ok(())
}
}
async fn read_rows(query: impl ExecutableQuery) -> Vec<(i32, i32)> {
let batches = query
.execute()
.await
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
batches
.iter()
.flat_map(|batch| {
let ids = batch["id"].as_any().downcast_ref::<Int32Array>().unwrap();
let scores = batch["score"]
.as_any()
.downcast_ref::<Float32Array>()
.unwrap();
// Scores are integer-valued by construction; compare exactly.
(0..batch.num_rows())
.map(|i| (ids.value(i), scores.value(i) as i32))
.collect::<Vec<_>>()
})
.collect()
}
/// Drive one mutation sequence: refresh + oracle-check after every step,
/// then a forced rebuild checked against the same oracle.
async fn run_sequence(ops: &[SrcOp], shape: Shape) -> Result<(), String> {
let label = format!("{shape:?} {ops:?}");
let mut case = Case::new(shape).await;
case.view
.refresh()
.execute()
.await
.map_err(|e| format!("{label}: initial refresh failed: {e}"))?;
case.check(&format!("{label} (initial)")).await?;
for (step, op) in ops.iter().enumerate() {
case.apply(*op).await;
case.view
.refresh()
.execute()
.await
.map_err(|e| format!("{label}: refresh at step {step} failed: {e}"))?;
case.check(&format!("{label} (step {step}, {op:?})"))
.await?;
}
case.view
.refresh()
.full(true)
.execute()
.await
.map_err(|e| format!("{label}: final full refresh failed: {e}"))?;
case.check(&format!("{label} (final rebuild)")).await?;
// Silence the unused-connection lint without dropping it mid-case.
let _ = &case.conn;
Ok(())
}
/// Every op sequence up to `max_len`.
fn all_sequences(max_len: u32) -> Vec<Vec<SrcOp>> {
let mut sequences = Vec::new();
for len in 1..=max_len {
for mut index in 0..ALL_OPS.len().pow(len) {
let mut ops = Vec::with_capacity(len as usize);
for _ in 0..len {
ops.push(ALL_OPS[index % ALL_OPS.len()]);
index /= ALL_OPS.len();
}
sequences.push(ops);
}
}
sequences
}
async fn run_exhaustive(max_len: u32) {
let mut cases = Vec::new();
for shape in [Shape::Identity, Shape::Filtered, Shape::Limited] {
for ops in all_sequences(max_len) {
cases.push((ops, shape));
}
}
let failures: Vec<String> = futures::stream::iter(cases)
.map(|(ops, shape)| async move { run_sequence(&ops, shape).await.err() })
.buffer_unordered(8)
.filter_map(|failure| async move { failure })
.collect()
.await;
assert!(
failures.is_empty(),
"{} sequences diverged; first: {}",
failures.len(),
failures[0]
);
}
#[tokio::test(flavor = "multi_thread")]
async fn differential_exhaustive() {
run_exhaustive(3).await;
}
#[tokio::test(flavor = "multi_thread")]
#[ignore = "longer sweep; run manually"]
async fn differential_exhaustive_deep() {
run_exhaustive(4).await;
}
/// Named interleavings that double as repro handles. The mode assertions pin
/// the classifier, which value comparison alone cannot: a wrongly rebuilt
/// view still matches the oracle.
#[tokio::test(flavor = "multi_thread")]
async fn differential_named_regressions() {
// An append is the one op that must stay incremental.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AppendNew).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
case.check("append stays incremental").await.unwrap();
// A column the view does not read must not force a rebuild.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AddColumn).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 0);
// Compaction rearranges rows without changing them: the watermark
// advances and nothing rebuilds.
let mut case = Case::new(Shape::Identity).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::AppendNew).await;
case.view.refresh().execute().await.unwrap();
case.apply(SrcOp::Compact).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 0);
case.check("compaction alone").await.unwrap();
// Fragment bookkeeping stays coherent across the compaction: the next
// append is separable and computed alone.
case.apply(SrcOp::AppendNew).await;
let result = case.view.refresh().execute().await.unwrap();
assert_eq!(result.mode, RefreshMode::Incremental);
assert_eq!(result.rows_written, 3);
case.check("compact then append").await.unwrap();
// A row updated to no longer match the filter must leave the view --
// and the fixture must prove the eviction happened, not merely that the
// end state matches: an update that never touched a view-resident row
// would also "match".
let mut case = Case::new(Shape::Filtered).await;
case.apply(SrcOp::AppendNew).await;
case.view.refresh().execute().await.unwrap();
let before = case.view_rows().await.len();
case.apply(SrcOp::UpdateOddScore).await;
case.view.refresh().execute().await.unwrap();
let after = case.view_rows().await.len();
assert!(
after < before,
"no view-resident row was evicted ({before} -> {after}); the fixture \
no longer exercises the filtered-update transition"
);
case.check("update crosses the filter").await.unwrap();
}
// ---------------------------------------------------------------------------
// Concurrency
// ---------------------------------------------------------------------------
//
// The sequential cases above cannot observe a cross-process race: the
// per-view refresh lock is process-local, so a second refresh in this
// process queues behind the first. What is missing is not more op
// sequences but a second process. These cases add one, and assert the same
// property the harness always asserts -- the view holds each row once.
/// Rows the definition selects from the source: every id but the first,
/// read straight from the source, sharing nothing with the refresh path.
async fn concurrency_oracle(conn: &Connection) -> Vec<i32> {
let batches: Vec<RecordBatch> = conn
.open_table("src")
.execute()
.await
.unwrap()
.query()
.select(Select::columns(&["id"]))
.execute()
.await
.unwrap()
.try_collect()
.await
.unwrap();
let mut ids = Vec::new();
for batch in &batches {
let column = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
for i in 0..batch.num_rows() {
if column.value(i) > 1 {
ids.push(column.value(i));
}
}
}
ids.sort_unstable();
ids
}
/// The view's ids, sorted.
async fn concurrency_view_ids(conn: &Connection) -> Vec<i32> {
let batches: Vec<RecordBatch> = conn
.open_table("mv")
.execute()
.await
.unwrap()
.query()
.select(Select::columns(&["id"]))
.execute()
.await
.unwrap()
.try_collect()
.await
.unwrap();
let mut ids = Vec::new();
for batch in &batches {
let column = batch
.column(0)
.as_any()
.downcast_ref::<Int32Array>()
.unwrap();
for i in 0..batch.num_rows() {
ids.push(column.value(i));
}
}
ids.sort_unstable();
ids
}
/// One refresh of the view at `MV_RACE_DIR`, in its own process.
///
/// Setup happens before the start barrier so warm-up does not stagger the
/// two processes. What makes the race certain rather than likely is the
/// second barrier inside `refresh()` itself, which holds every participant
/// between staging and commit.
#[tokio::test]
#[ignore = "spawned as a child process by the concurrency cases"]
async fn cross_process_refresh_child() {
let Ok(dir) = std::env::var("MV_RACE_DIR") else {
return;
};
let dir = std::path::PathBuf::from(dir);
let tag = std::env::var("MV_RACE_TAG").unwrap();
let conn = connect(dir.to_str().unwrap()).execute().await.unwrap();
let table = conn.open_table("mv").execute().await.unwrap();
let _ = table.schema().await.unwrap();
let _ = table.count_rows(None).await.unwrap();
let source = conn.open_table("src").execute().await.unwrap();
let _ = source.count_rows(None).await.unwrap();
let view = MaterializedView::from_table(table).await.unwrap();
std::fs::write(dir.join(format!("ready-{tag}")), b"1").unwrap();
while !dir.join("START").exists() {
std::thread::sleep(std::time::Duration::from_millis(2));
}
let outcome = match view.refresh().execute().await {
Ok(result) => format!("committed rows={}", result.rows_written),
Err(err) if is_commit_conflict(&err) => "conflicted".to_string(),
Err(err) => format!("failed {err}"),
};
std::fs::write(dir.join(format!("outcome-{tag}")), outcome).unwrap();
}
/// Whether a refresh lost its commit to a concurrent one, as opposed to
/// failing for any other reason.
fn is_commit_conflict(err: &crate::Error) -> bool {
let text = err.to_string();
text.contains("Retryable commit conflict") || text.contains("preempted by concurrent")
}
/// Two processes refreshing one view concurrently must leave the view
/// equal to the oracle: each selected row present exactly once.
///
/// Both plan the same incremental delta from one watermark. A refresh is
/// meant to land on the generation it planned or leave nothing behind, so
/// at most one of them may write.
#[tokio::test]
async fn concurrent_refreshes_hold_each_row_once() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().to_str().unwrap().to_string();
let conn = connect(&path).execute().await.unwrap();
conn.create_table("src", rows_batch(&[1, 2, 3, 4]))
.write_options(crate::materialized_view::tests::stable_row_ids())
.execute()
.await
.unwrap();
let view = conn
.create_materialized_view("mv", "src")
.select([("id", "id"), ("score", "score")])
.only_if("id > 1")
.execute()
.await
.unwrap();
// Seed the watermark so the racing refreshes are both incremental.
view.refresh().execute().await.unwrap();
// Large enough that a refresh is real work rather than a formality.
let ids: Vec<i32> = (100..200_100).collect();
conn.open_table("src")
.execute()
.await
.unwrap()
.add(rows_batch(&ids))
.execute()
.await
.unwrap();
let tags = ["a", "b"];
let exe = std::env::current_exe().unwrap();
let children: Vec<std::process::Child> = tags
.iter()
.map(|tag| {
std::process::Command::new(&exe)
.args([
"--exact",
"materialized_view::differential::cross_process_refresh_child",
"--ignored",
"--nocapture",
])
.env("MV_RACE_DIR", dir.path())
.env("MV_RACE_SYNC", dir.path())
.env("MV_RACE_PEERS", "2")
.env("MV_RACE_TAG", tag)
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.spawn()
.unwrap()
})
.collect();
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(180);
while tags
.iter()
.any(|tag| !dir.path().join(format!("ready-{tag}")).exists())
{
assert!(
std::time::Instant::now() < deadline,
"children never became ready"
);
std::thread::sleep(std::time::Duration::from_millis(10));
}
std::fs::write(dir.path().join("START"), b"1").unwrap();
for (tag, mut child) in tags.iter().zip(children) {
let status = loop {
match child.try_wait().unwrap() {
Some(status) => break status,
None if std::time::Instant::now() >= deadline => {
child.kill().unwrap();
panic!("child {tag} never finished");
}
None => std::thread::sleep(std::time::Duration::from_millis(10)),
}
};
assert!(status.success(), "child {tag} exited {status}");
}
// Both refreshes reached the commit boundary before either committed --
// the in-refresh barrier guarantees it -- so exactly one may win.
let outcomes: Vec<String> = tags
.iter()
.map(|tag| {
std::fs::read_to_string(dir.path().join(format!("outcome-{tag}")))
.unwrap_or_else(|_| panic!("child {tag} recorded no outcome"))
})
.collect();
for tag in tags {
assert!(
dir.path().join(format!("planned-{tag}")).exists(),
"child {tag} never reached the commit boundary, so nothing was synchronized"
);
}
let committed = outcomes.iter().filter(|o| o.contains("committed")).count();
let conflicted = outcomes.iter().filter(|o| o.contains("conflicted")).count();
assert_eq!(
(committed, conflicted),
(1, 1),
"exactly one refresh may win the generation both planned: {outcomes:?}"
);
let expected = concurrency_oracle(&conn).await;
let actual = concurrency_view_ids(&conn).await;
assert_eq!(
actual.len(),
expected.len(),
"the view holds {} rows, the oracle {}: a losing refresh left rows behind",
actual.len(),
expected.len()
);
assert_eq!(actual, expected, "the view does not match the oracle");
}
rust/lancedb/src/materialized_view/refresh.rs
+2757
View File
File diff suppressed because it is too large Load Diff
rust/lancedb/src/remote/db.rs
+2 -2
View File
@@ -513,7 +513,7 @@ impl<S: HttpSend> Database for RemoteDatabase<S> {
async fn get_function(&self, name: &str, version: &str) -> Result<FunctionVersion> {
let req = self
.client
.post("/v1/functions/get")
.post("/v1/functions/describe")
.json(&serde_json::json!({
"name": name,
"version": version,
@@ -2520,7 +2520,7 @@ mod tests {
);
let conn = Connection::new_with_handler(|request| {
assert_eq!(request.method(), &reqwest::Method::POST);
assert_eq!(request.url().path(), "/v1/functions/get");
assert_eq!(request.url().path(), "/v1/functions/describe");
let body: serde_json::Value =
serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap();
assert_eq!(
rust/lancedb/src/remote/table.rs
+69 -43
View File
@@ -21,11 +21,11 @@ use crate::remote::job::RemoteJob;
use crate::table::AddColumnsResult;
use crate::table::AddResult;
use crate::table::BranchDiff;
use crate::table::CherryPickResult;
use crate::table::DeleteResult;
use crate::table::DropColumnsResult;
use crate::table::LsmStats;
use crate::table::LsmWriteSpec;
use crate::table::MergeBranchResult;
use crate::table::MergeResult;
use crate::table::Tags;
use crate::table::UpdateResult;
@@ -2034,7 +2034,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
async fn diff_branch(&self, from_branch: &str) -> Result<BranchDiff> {
if from_branch.trim().is_empty() {
return Err(Error::InvalidInput {
message: "from_branch must be a non-empty string".into(),
message: "Branch name cannot be empty.".into(),
});
}
let request = self
@@ -2061,20 +2061,23 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
}
async fn merge_branch(&self, from_branch: &str, dry_run: bool) -> Result<MergeBranchResult> {
async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
if from_branch.trim().is_empty() {
return Err(Error::InvalidInput {
message: "from_branch must be a non-empty string".into(),
message: "Branch name cannot be empty.".into(),
});
}
let request = self
.client
.post(&format!("/v1/table/{}/branches/merge/", self.identifier))
.post(&format!(
"/v1/table/{}/branches/cherry_pick/",
self.identifier
))
.json(&serde_json::json!({
"from_branch": from_branch,
"dry_run": dry_run,
}));
// No retry. 409 rejected merge is final and carries a body.
// No retry. HTTP 409 is CherryPickStatus::Failed with a body, not a transport error.
let (request_id, response) = self.send(request, false).await?;
let status = response.status();
if status == StatusCode::NOT_FOUND {
@@ -2083,11 +2086,11 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
source: format!("branch '{}' does not exist", from_branch).into(),
});
}
// 200 and 409 both carry MergeBranchResult.
// 200 and 409 both carry CherryPickResult.
if status != StatusCode::OK && status != StatusCode::CONFLICT {
let body = response.text().await.unwrap_or_default();
return Err(Error::Http {
source: format!("unexpected status {status} from merge_branch: {body}").into(),
source: format!("unexpected status {status} from cherry_pick: {body}").into(),
request_id,
status_code: Some(status),
});
@@ -2095,7 +2098,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
let body = response.text().await.err_to_http(request_id.clone())?;
serde_json::from_str(&body).map_err(|err| Error::Http {
source: format!(
"Failed to parse merge_branch response: {}, body: {}",
"Failed to parse cherry_pick response: {}, body: {}",
err, body
)
.into(),
@@ -2864,7 +2867,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
})
}
async fn refresh_column_async(&self, column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
self.check_mutable().await?;
let mut body = serde_json::json!({ "column": column });
self.apply_branch_body(&mut body);
@@ -2885,7 +2891,7 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
status_code: None,
})?;
Ok(Job::new(Box::new(FreshnessJob {
Ok(Job::new_typed(Box::new(FreshnessJob {
inner: RemoteJob::new(self.client.clone(), response.job_id),
freshness: self.freshness.clone(),
version: self.version.clone(),
@@ -3280,6 +3286,21 @@ mod tests {
},
};
fn refresh_done(job_id: &str) -> String {
json!({
"job_id": job_id,
"job_state": "DONE",
"result": {
"rows_assigned": 12,
"rows_failed": 0,
"rows_remaining": 0,
"source_version": 7,
"published_version": 8,
}
})
.to_string()
}
#[tokio::test]
async fn test_not_found() {
let table = Table::new_with_handler("my_table", |_| {
@@ -6951,7 +6972,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-7", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-7"))
.unwrap(),
"/v1/table/my_table/count_rows/" => {
saw.store(
@@ -6967,7 +6988,9 @@ mod tests {
});
let job = table.refresh_column_async("doubled").await.unwrap();
job.wait().await.unwrap();
let result = job.wait().await.unwrap();
assert_eq!(result.rows_assigned, 12);
assert_eq!(result.published_version, Some(8));
table.count_rows(None).await.unwrap();
assert!(
saw_min_timestamp.load(std::sync::atomic::Ordering::SeqCst),
@@ -6989,7 +7012,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-8", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-8"))
.unwrap(),
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -7035,7 +7058,7 @@ mod tests {
.unwrap(),
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-9", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-9"))
.unwrap(),
"/v1/table/my_table/tags/version/" => http::Response::builder()
.status(200)
@@ -7099,7 +7122,7 @@ mod tests {
}
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-10", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-10"))
.unwrap(),
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new("x", DataType::Int32, true)]);
@@ -7172,7 +7195,7 @@ mod tests {
}
"/v1/jobs/describe" => http::Response::builder()
.status(200)
.body(r#"{"job_id": "j-11", "job_state": "DONE"}"#.to_string())
.body(refresh_done("j-11"))
.unwrap(),
"/v1/table/my_table/count_rows/" => {
*saw.lock().unwrap() = request
@@ -10575,8 +10598,7 @@ mod tests {
"changedColumns":[],
"addedIndexes":[],
"removedIndexes":[],
"mergeable":true,
"mergeBlockers":[]
"errors":[]
}"#
}
@@ -10594,15 +10616,18 @@ mod tests {
});
let diff = table.diff_branch("exp").await.unwrap();
assert_eq!(diff.from_branch, "exp");
assert!(diff.mergeable);
assert!(diff.errors.is_empty());
assert_eq!(diff.added_columns.len(), 1);
assert_eq!(diff.added_columns[0].name, "tag");
}
#[tokio::test]
async fn test_merge_branch_dry_run() {
async fn test_cherry_pick_dry_run() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
assert_eq!(
request.url().path(),
"/v1/table/my_table/branches/cherry_pick/"
);
let body = request_body_json(&request);
assert_eq!(body["from_branch"], "exp");
assert_eq!(body["dry_run"], true);
@@ -10612,27 +10637,29 @@ mod tests {
);
http::Response::builder().status(200).body(resp).unwrap()
});
let result = table.merge_branch("exp", true).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Ready);
let result = table.cherry_pick("exp", true).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Ready);
assert_eq!(result.preview.promoted_columns, vec!["tag".to_string()]);
assert!(result.main_version_after.is_none());
}
#[tokio::test]
async fn test_merge_branch_rejected_returns_ok_with_body() {
async fn test_cherry_pick_failed_returns_ok_with_body() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.url().path(), "/v1/table/my_table/branches/merge/");
assert_eq!(
request.url().path(),
"/v1/table/my_table/branches/cherry_pick/"
);
let body = request_body_json(&request);
assert_eq!(body["dry_run"], false);
let mut diff: serde_json::Value =
serde_json::from_str(sample_branch_diff_json()).unwrap();
diff["mergeable"] = serde_json::json!(false);
diff["mergeBlockers"] = serde_json::json!([{
diff["errors"] = serde_json::json!([{
"code": "baseMoved",
"message": "main has advanced"
}]);
let resp = serde_json::json!({
"status": "rejected",
"status": "failed",
"diff": diff,
"preview": { "promotedColumns": [] }
});
@@ -10641,24 +10668,23 @@ mod tests {
.body(resp.to_string())
.unwrap()
});
let result = table.merge_branch("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
assert!(!result.diff.mergeable);
assert_eq!(result.diff.merge_blockers.len(), 1);
let result = table.cherry_pick("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
assert!(!result.diff.errors.is_empty());
assert_eq!(result.diff.errors.len(), 1);
}
#[tokio::test]
async fn test_merge_branch_unknown_blocker_code_parses() {
async fn test_cherry_pick_unknown_error_code_parses() {
let table = Table::new_with_handler("my_table", |_| {
let mut diff: serde_json::Value =
serde_json::from_str(sample_branch_diff_json()).unwrap();
diff["mergeable"] = serde_json::json!(false);
diff["mergeBlockers"] = serde_json::json!([{
diff["errors"] = serde_json::json!([{
"code": "multipleCommits",
"message": "branch has more than one data commit"
}]);
let resp = serde_json::json!({
"status": "rejected",
"status": "failed",
"diff": diff,
"preview": { "operation": "append", "rowsAdded": 2 }
});
@@ -10667,24 +10693,24 @@ mod tests {
.body(resp.to_string())
.unwrap()
});
let result = table.merge_branch("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::MergeBranchStatus::Rejected);
let result = table.cherry_pick("exp", false).await.unwrap();
assert_eq!(result.status, crate::table::CherryPickStatus::Failed);
assert_eq!(
result.diff.merge_blockers[0].code,
crate::table::MergeBlockerCode::Unknown
result.diff.errors[0].code,
crate::table::CherryPickErrorCode::Unknown
);
assert!(result.preview.promoted_columns.is_empty());
}
#[tokio::test]
async fn test_merge_branch_unexpected_2xx_is_error() {
async fn test_cherry_pick_unexpected_2xx_is_error() {
let table = Table::new_with_handler("my_table", |_| {
http::Response::builder()
.status(204)
.body(String::new())
.unwrap()
});
let err = table.merge_branch("exp", false).await.unwrap_err();
let err = table.cherry_pick("exp", false).await.unwrap_err();
match err {
Error::Http {
status_code: Some(code),
rust/lancedb/src/table.rs
+30 -22
View File
@@ -66,8 +66,8 @@ use self::merge::MergeInsertBuilder;
pub mod add_columns;
mod add_data;
pub mod branch_merge;
pub mod checkpoint;
pub mod cherry_pick;
pub mod computed_columns;
mod create_index;
pub mod datafusion;
@@ -87,9 +87,9 @@ pub use add_columns::AddColumnsBuilder;
#[cfg(feature = "remote")]
pub(crate) use add_data::PreprocessingOutput;
pub use add_data::{AddDataBuilder, AddDataMode, AddResult, NaNVectorBehavior};
pub use branch_merge::{
BranchDiff, ColumnChange, ColumnSummary, IndexSummary, MergeBlocker, MergeBlockerCode,
MergeBranchResult, MergeBranchStatus, MergePreview, RowCountSummary,
pub use cherry_pick::{
BranchDiff, CherryPickError, CherryPickErrorCode, CherryPickPreview, CherryPickResult,
CherryPickStatus, ColumnChange, ColumnSummary, IndexSummary, RowCountSummary,
};
pub use chrono::Duration;
pub use computed_columns::{
@@ -779,7 +779,10 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
}
/// Fill a computed column's unfilled rows, returning a [`Job`] tracking
/// the operation.
async fn refresh_column_async(&self, _column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
_column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
Err(Error::NotSupported {
message: "computed columns are supported only on local tables".into(),
})
@@ -840,14 +843,14 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync {
/// Diff a branch against main. Remote only.
async fn diff_branch(&self, _from_branch: &str) -> Result<BranchDiff> {
Err(Error::NotSupported {
message: "diff_branch is only supported on remote tables".into(),
message: "Branch diffs are only supported on Enterprise tables.".into(),
})
}
/// Merge a branch into main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
async fn merge_branch(&self, _from_branch: &str, _dry_run: bool) -> Result<MergeBranchResult> {
/// Cherry-pick a branch onto main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
async fn cherry_pick(&self, _from_branch: &str, _dry_run: bool) -> Result<CherryPickResult> {
Err(Error::NotSupported {
message: "merge_branch is only supported on remote tables".into(),
message: "Cherry-picking branches is only supported on Enterprise tables.".into(),
})
}
/// The branch this handle is scoped to, or `None` for `main`.
@@ -1716,7 +1719,9 @@ impl Table {
/// operation instead of blocking until it completes.
///
/// The job may already be complete when returned, and callers must not
/// assume the column is filled until [`Job::wait`] returns. Invalid input
/// assume the column is filled until [`Job::wait`] returns. A successful
/// wait returns the durable [`crate::function::RefreshColumnResult`] for
/// both expression-backed and Function-backed columns. Invalid input
/// -- an unknown column, or one that is not computed -- is reported by
/// this call rather than by the job. On local tables the job runs as an
/// in-process task; on LanceDB Cloud and Enterprise it is the server's
@@ -1727,11 +1732,15 @@ impl Table {
/// # async fn refresh_in_background(table: &Table) -> Result<(), Box<dyn std::error::Error>> {
/// let job = table.refresh_column_async("doubled").await?;
/// println!("refresh running: {:?}", job.status().await?);
/// job.wait().await?;
/// let result = job.wait().await?;
/// println!("assigned {} rows", result.rows_assigned);
/// # Ok(())
/// # }
/// ```
pub async fn refresh_column_async(&self, column: impl AsRef<str>) -> Result<Job> {
pub async fn refresh_column_async(
&self,
column: impl AsRef<str>,
) -> Result<Job<crate::function::RefreshColumnResult>> {
self.inner.refresh_column_async(column.as_ref()).await
}
@@ -2271,14 +2280,10 @@ impl Table {
self.inner.diff_branch(from_branch).await
}
/// Merge a branch into main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`MergeBranchStatus::Rejected`].
pub async fn merge_branch(
&self,
from_branch: &str,
dry_run: bool,
) -> Result<MergeBranchResult> {
self.inner.merge_branch(from_branch, dry_run).await
/// Cherry-pick a branch onto main, or dry-run. Remote only.
/// HTTP 409 still returns [`Ok`] with [`CherryPickStatus::Failed`].
pub async fn cherry_pick(&self, from_branch: &str, dry_run: bool) -> Result<CherryPickResult> {
self.inner.cherry_pick(from_branch, dry_run).await
}
/// The branch this handle is scoped to, or `None` for `main`.
@@ -3437,7 +3442,10 @@ impl BaseTable for NativeTable {
Ok(result)
}
async fn refresh_column_async(&self, column: &str) -> Result<Job> {
async fn refresh_column_async(
&self,
column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
refresh::execute_refresh_column_async(self, column).await
}
rust/lancedb/src/table/branch_merge.rs → rust/lancedb/src/table/cherry_pick.rs
+13 -14
View File
@@ -1,7 +1,7 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
//! Types for remote branch diff / merge against main.
//! Types for remote branch diff / cherry-pick onto main.
use serde::{Deserialize, Serialize};
@@ -44,13 +44,13 @@ pub struct RowCountSummary {
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum MergeBlockerCode {
pub enum CherryPickErrorCode {
BaseMoved,
RowCountMismatch,
RowsChanged,
ColumnRemoved,
ColumnChanged,
NoMergeableChanges,
NothingToApply,
NoColumnChanges,
InputColumnDependency,
ParentNotMain,
@@ -60,8 +60,8 @@ pub enum MergeBlockerCode {
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergeBlocker {
pub code: MergeBlockerCode,
pub struct CherryPickError {
pub code: CherryPickErrorCode,
pub message: String,
}
@@ -81,34 +81,33 @@ pub struct BranchDiff {
pub changed_columns: Vec<ColumnChange>,
pub added_indexes: Vec<IndexSummary>,
pub removed_indexes: Vec<IndexSummary>,
pub mergeable: bool,
pub merge_blockers: Vec<MergeBlocker>,
pub errors: Vec<CherryPickError>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergePreview {
pub struct CherryPickPreview {
#[serde(default)]
pub promoted_columns: Vec<String>,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub enum MergeBranchStatus {
pub enum CherryPickStatus {
Ready,
Rejected,
Failed,
NotImplemented,
Merged,
CherryPicked,
#[serde(other)]
Unknown,
}
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
#[serde(rename_all = "camelCase")]
pub struct MergeBranchResult {
pub status: MergeBranchStatus,
pub struct CherryPickResult {
pub status: CherryPickStatus,
pub diff: BranchDiff,
pub preview: MergePreview,
pub preview: CherryPickPreview,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub main_version_after: Option<u64>,
}
rust/lancedb/src/table/merge/lsm.rs
+7
View File
@@ -104,6 +104,13 @@ pub(crate) async fn set_lsm_write_spec(table: &NativeTable, spec: LsmWriteSpec)
.into(),
});
}
if crate::materialized_view::materialized_view_kind(&dataset.schema().metadata)?.is_some() {
return Err(Error::NotSupported {
message: "an LSM write spec cannot be installed on a materialized view: \
rows in un-compacted tiers are invisible to refresh"
.into(),
});
}
let mut builder = dataset.initialize_mem_wal();
let writer_config_defaults = match spec {
LsmWriteSpec::Bucket {
rust/lancedb/src/table/refresh.rs
+60 -13
View File
@@ -49,11 +49,25 @@ pub struct RefreshColumnResult {
pub version: u64,
}
struct RefreshExecution {
result: RefreshColumnResult,
source_version: u64,
}
/// Internal implementation of the refresh logic.
pub(crate) async fn execute_refresh_column(
table: &NativeTable,
column: &str,
) -> Result<RefreshColumnResult> {
Ok(execute_refresh_column_with_source(table, column)
.await?
.result)
}
async fn execute_refresh_column_with_source(
table: &NativeTable,
column: &str,
) -> Result<RefreshExecution> {
table.dataset.ensure_mutable()?;
ensure_no_lsm_write_spec(table).await?;
let dataset = table.dataset.get().await?;
@@ -87,9 +101,13 @@ pub(crate) async fn execute_refresh_column(
}
if replacements.is_empty() {
return Ok(RefreshColumnResult {
rows_filled: 0,
version: dataset.version().version,
let source_version = dataset.version().version;
return Ok(RefreshExecution {
result: RefreshColumnResult {
rows_filled: 0,
version: source_version,
},
source_version,
});
}
@@ -110,14 +128,20 @@ pub(crate) async fn execute_refresh_column(
let version = new_dataset.version().version;
table.dataset.update(new_dataset);
Ok(RefreshColumnResult {
rows_filled,
version,
Ok(RefreshExecution {
result: RefreshColumnResult {
rows_filled,
version,
},
source_version: read_version,
})
}
/// Run the refresh as a [`Job`] in this process.
pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &str) -> Result<Job> {
pub(crate) async fn execute_refresh_column_async(
table: &NativeTable,
column: &str,
) -> Result<Job<crate::function::RefreshColumnResult>> {
// Validate before spawning so bad input is reported by this call rather
// than only by the job.
table.dataset.ensure_mutable()?;
@@ -129,9 +153,16 @@ pub(crate) async fn execute_refresh_column_async(table: &NativeTable, column: &s
let table = table.clone();
let column = column.to_string();
Ok(Job::spawned(tokio::spawn(async move {
execute_refresh_column(&table, &column).await?;
let execution = execute_refresh_column_with_source(&table, &column).await?;
table.bump_freshness();
Ok(())
Ok(crate::function::RefreshColumnResult {
rows_assigned: execution.result.rows_filled,
rows_failed: 0,
rows_remaining: 0,
source_version: execution.source_version,
published_version: (execution.result.rows_filled > 0)
.then_some(execution.result.version),
})
})))
}
@@ -396,6 +427,17 @@ mod tests {
read(&table, "doubled").await,
vec![Some(2), Some(4), Some(6)]
);
let no_op = table
.refresh_column_async("doubled")
.await
.unwrap()
.wait()
.await
.unwrap();
assert_eq!(no_op.rows_assigned, 0);
assert_eq!(no_op.source_version, 3);
assert_eq!(no_op.published_version, None);
}
/// Values written after the last refresh must be reachable by another one.
@@ -597,7 +639,7 @@ mod tests {
/// A fragment spanning several scan batches exercises the streamed fill:
/// the probe buffers only until the first gained value and the rest flows
/// through write_column a batch at a time.
/// through write_columns a batch at a time.
#[tokio::test]
async fn test_refresh_streams_a_multi_batch_fragment() {
let values: Vec<i32> = (0..20_000).collect();
@@ -646,7 +688,12 @@ mod tests {
let job = table.refresh_column_async("doubled").await.unwrap();
assert!(job.id().is_none(), "in-process jobs have no server id");
job.wait().await.unwrap();
let result = job.wait().await.unwrap();
assert_eq!(result.rows_assigned, 3);
assert_eq!(result.rows_failed, 0);
assert_eq!(result.rows_remaining, 0);
assert_eq!(result.source_version, 2);
assert_eq!(result.published_version, Some(3));
assert_eq!(job.status().await.unwrap(), "finished");
assert_eq!(
read(&table, "doubled").await,
@@ -672,9 +719,9 @@ mod tests {
declare_doubled(&table).await.unwrap();
let job = table.refresh_column_async("doubled").await.unwrap();
job.wait().await.unwrap();
let first = job.wait().await.unwrap();
// A second wait after completion observes the same outcome.
job.wait().await.unwrap();
assert_eq!(job.wait().await.unwrap(), first);
assert_eq!(job.status().await.unwrap(), "finished");
}